From cba3cc8cc54e677698f23a616ea303b23cc503d2 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 16 May 2025 19:07:50 +0100 Subject: [PATCH 001/127] v0.1 validator, and specify platform --- docker-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d4e88c8..8605c0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,8 @@ version: '3.8' services: flask: - image: "ghcr.io/esciencelab/cratey-validator:latest" + platform: linux/x86_64 + image: "ghcr.io/esciencelab/cratey-validator:0.1" ports: - "5001:5000" environment: @@ -19,7 +20,8 @@ services: - minio celery_worker: - image: "ghcr.io/esciencelab/cratey-validator:latest" + platform: linux/x86_64 + image: "ghcr.io/esciencelab/cratey-validator:0.1" command: celery -A app.celery_worker.celery worker --loglevel=info -E environment: - CELERY_BROKER_URL=redis://redis:6379/0 From 6116484b77469040c0d974e6b9fbbd80c5feaaf0 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 27 Jun 2025 21:11:12 +0100 Subject: [PATCH 002/127] validate_metadata api --- app/ro_crates/routes/post_routes.py | 40 ++++++++++++++++- app/services/validation_service.py | 34 +++++++++++++++ app/tasks/validation_tasks.py | 67 ++++++++++++++++++++++++++++- app/utils/file_utils.py | 54 +++++++++++++++++++++++ 4 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 app/utils/file_utils.py diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 12f849b..f73326e 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -8,7 +8,10 @@ from apiflask.fields import Integer, String from flask import request, Response -from app.services.validation_service import queue_ro_crate_validation_task +from app.services.validation_service import ( + queue_ro_crate_validation_task, + queue_ro_crate_metadata_validation_task +) post_routes_bp = APIBlueprint("post_routes", __name__) @@ -17,6 +20,10 @@ class validate_data(Schema): profile_name = String(required=False) webhook_url = String(required=False) +class validate_json(Schema): + crate_json = String(required=True) + profile_name = String(required=False) + @post_routes_bp.post("/validate_by_id") @post_routes_bp.input(validate_data(partial=True), location='json') @@ -80,3 +87,34 @@ def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: profile_name = None return queue_ro_crate_validation_task(crate_id, profile_name) + + +@post_routes_bp.post("/validate_metadata") +@post_routes_bp.input(validate_json(partial=True), location='json') # -> json_data +def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: + """ + Endpoint to validate an RO-Crate JSON file uploaded to the Service. + + Parameters: + - **crate_json**: The RO-Crate JSON-LD, as a string. _Required_ + - **profile_name**: The profile name for validation. _Optional_. + + Returns: + - A tuple containing the validation task's response and an HTTP status code. + + Raises: + - KeyError: If required parameters (`crate_json`) are missing. + """ + + try: + crate_json = json_data['crate_json'] + except: + raise KeyError("Missing required parameter: 'crate_json'") + + try: + profile_name = json_data['profile_name'] + except: + profile_name = None + + return queue_ro_crate_metadata_validation_task(crate_json, profile_name) + diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 669ab38..357b10e 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -10,6 +10,7 @@ from app.tasks.validation_tasks import ( process_validation_task_by_id, + process_validation_task_by_metadata, return_ro_crate_validation ) @@ -42,6 +43,39 @@ def queue_ro_crate_validation_task( return jsonify({"error": str(e)}), 500 +def queue_ro_crate_metadata_validation_task( + crate_json, profile_name=None, webhook_url=None +) -> tuple[Response, int]: + """ + Queues an RO-Crate for validation with Celery. + + :param crate_id: The ID of the RO-Crate to validate. + :param profile_name: The profile to validate against. + :param webhook_url: The URL to POST the validation results to. + :return: A tuple containing a JSON response and an HTTP status code. + :raises: Exception: If an error occurs whilst queueing the task. + """ + + logging.info(f"Processing: {crate_json}, {profile_name}, {webhook_url}") + + if not crate_json: + return jsonify({"error": "Missing required parameter: crate_json"}), 400 + + try: + result = process_validation_task_by_metadata.delay( + crate_json, + profile_name, + webhook_url + ) + if webhook_url: + return jsonify({"message": "Validation in progress"}), 202 + else: + return jsonify({"result": result.get()}), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + + def get_ro_crate_validation_task( crate_id ) -> tuple[Response, int]: diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index c58799e..261aa67 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -6,6 +6,8 @@ import logging import os +import shutil +from typing import Optional from rocrate_validator import services from rocrate_validator.models import ValidationResult @@ -17,6 +19,7 @@ get_validation_status_from_minio ) from app.utils.webhook_utils import send_webhook_notification +from app.utils.file_utils import build_metadata_only_rocrate logger = logging.getLogger(__name__) @@ -78,8 +81,67 @@ def process_validation_task_by_id( os.remove(file_path) +@celery.task +def process_validation_task_by_metadata( + crate_json: str, profile_name: str | None, webhook_url: str | None +) -> ValidationResult | str: + """ + Background task to process the RO-Crate validation for a given json metadata string. + + :param crate_json: A string containing the RO-Crate JSON metadata to validate. + :param profile_name: The name of the validation profile to use. Defaults to None. + :param webhook_url: The webhook URL to send notifications to. Defaults to None. + :raises Exception: If an error occurs during the validation process. + + :todo: Replace the Crate ID with a more comprehensive system, and replace profile name with URI. + """ + + skip_checks_list = ['ro-crate-1.1_12.1'] + file_path = None + + try: + # Fetch the RO-Crate from MinIO using the provided ID: + file_path = build_metadata_only_rocrate(crate_json) + + logging.info(f"Processing validation task for {file_path}") + + # Perform validation: + validation_result = perform_ro_crate_validation(file_path, + profile_name, + skip_checks_list + ) + + if isinstance(validation_result, str): + logging.error(f"Validation failed: {validation_result}") + # TODO: Send webhook with failure notification + raise Exception(f"Validation failed: {validation_result}") + + if not validation_result.has_issues(): + logging.info(f"RO Crate {file_path} is valid.") + else: + logging.info(f"RO Crate {file_path} is invalid.") + + if webhook_url: + send_webhook_notification(webhook_url, validation_result.to_json()) + + except Exception as e: + logging.error(f"Error processing validation task: {e}") + + # Send failure notification via webhook + error_data = {"profile_name": profile_name, "error": str(e)} + send_webhook_notification(webhook_url, error_data) + + finally: + # Clean up the temporary file if it was created: + if file_path and os.path.exists(file_path): + shutil.rmtree(file_path) + + return validation_result.to_json() + + + def perform_ro_crate_validation( - file_path: str, profile_name: str | None + file_path: str, profile_name: str | None, skip_checks_list: Optional[list] = None ) -> ValidationResult | str: """ Validates an RO-Crate using the provided file path and profile name. @@ -87,6 +149,7 @@ def perform_ro_crate_validation( :param file_path: The path to the RO-Crate file to validate :param profile_name: The name of the validation profile to use. Defaults to None. If None, the CRS4 validator will attempt to determine the profile. + :param skip_checks_list: A list of checks to skip, if needed :return: The validation result. :raises Exception: If an error occurs during the validation process. """ @@ -102,8 +165,8 @@ def perform_ro_crate_validation( ) settings = services.ValidationSettings( rocrate_uri=full_file_path, - # Only include profile_identifier if the profile_name is provided: **({"profile_identifier": profile_name} if profile_name else {}), + **({"skip_checks": skip_checks_list} if skip_checks_list else {}) ) return services.validate(settings) diff --git a/app/utils/file_utils.py b/app/utils/file_utils.py new file mode 100644 index 0000000..0b20eac --- /dev/null +++ b/app/utils/file_utils.py @@ -0,0 +1,54 @@ +"""Utility methods for interacting with the File System.""" + +# Author: Douglas Lowe, Alexander Hambley +# License: MIT +# Copyright (c) 2025 eScience Lab, The University of Manchester + +import json +import logging +import os +import tempfile + +from dotenv import load_dotenv + + +logger = logging.getLogger(__name__) + + +def build_metadata_only_rocrate(crate_json: str) -> str: + """ + Creates a temporary directory for an empty RO-Crate, + and saves the JSON string as a metadata file. + + :param crate_json: The metadata string. + :return: The local file path where the RO-Crate is saved. + :raises ValueError: If the required environment variables are not set. + :raises Exception: If an unexpected error occurs during the operation. + """ + + load_dotenv() + + try: + # Prepare temporary file path to store RO Crate for validation: + temp_dir = tempfile.mkdtemp() + file_path = os.path.join(temp_dir, 'ro-crate-metadata.json') + + logging.info( + f"Creating RO-Crate Metadata file. File path: {file_path}" + ) + with open(file_path, 'w') as f: + f.write(crate_json) + logging.info( + f"RO-Crate metadata successfully saved to {file_path}." + ) + + return temp_dir + + except ValueError as value_error: + logging.error(f"Configuration Error: {value_error}") + raise + + except Exception as e: + logging.error(f"Unexpected error creating RO-Crate metadata: {e}") + raise + From b92649ecececd957ca3f4d9dd1120ba71e725586 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 1 Jul 2025 22:35:36 +0100 Subject: [PATCH 003/127] metadata api integration test --- requirements.testing.txt | 10 +++ tests/data/ro-crate-metadata.json | 101 ++++++++++++++++++++++++++++++ tests/metadata_integration.py | 53 ++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 requirements.testing.txt create mode 100644 tests/data/ro-crate-metadata.json create mode 100644 tests/metadata_integration.py diff --git a/requirements.testing.txt b/requirements.testing.txt new file mode 100644 index 0000000..6bec2e6 --- /dev/null +++ b/requirements.testing.txt @@ -0,0 +1,10 @@ +celery~=5.4.0 +minio~=7.2.10 +requests~=2.32.3 +Flask~=3.0.3 +Werkzeug~=3.0.4 +redis~=5.2.0 +python-dotenv~=1.0.1 +apiflask~=2.4.0 +roc-validator~=0.6.5 +pytest~=8.4.1 diff --git a/tests/data/ro-crate-metadata.json b/tests/data/ro-crate-metadata.json new file mode 100644 index 0000000..957ddea --- /dev/null +++ b/tests/data/ro-crate-metadata.json @@ -0,0 +1,101 @@ +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "name": "example dataset", + "description": "crate that should conform with validator requirements", + "datePublished": "2024-03-17T15:55:54+00:00", + "license": { + "@id": "http://spdx.org/licenses/MIT" + }, + "hasPart": [ + { + "@id": "paper.pdf" + }, + { + "@id": "results.csv" + }, + { + "@id": "images/figure.svg" + }, + { + "@id": "logs/" + } + ] + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "./" + }, + "conformsTo": { + "@id": "https://w3id.org/ro/crate/1.1" + } + }, + { + "@id": "paper.pdf", + "@type": "File", + "author": [ + { + "@id": "https://orcid.org/0000-0000-0000-0000" + }, + { + "@id": "https://orcid.org/0000-0000-0000-0001" + } + ], + "encodingFormat": "application/pdf", + "name": "manuscript" + }, + { + "@id": "results.csv", + "@type": "File", + "author": { + "@id": "https://orcid.org/0000-0000-0000-0000" + }, + "encodingFormat": "text/csv", + "name": "experimental data" + }, + { + "@id": "images/figure.svg", + "@type": "File", + "author": { + "@id": "https://orcid.org/0000-0000-0000-0001" + }, + "encodingFormat": "image/svg+xml", + "name": "bar chart" + }, + { + "@id": "logs/", + "@type": "Dataset" + }, + { + "@id": "https://orcid.org/0000-0000-0000-0000", + "@type": "Person", + "affiliation": "University of Flatland", + "name": "Alice Doe" + }, + { + "@id": "https://orcid.org/0000-0000-0000-0001", + "@type": "Person", + "affiliation": "University of Flatland", + "name": "Bob Doe" + }, + { + "@id": "#hdr2024", + "@type": "Project Meeting", + "endDate": "2024-04-16", + "location": "Swansea, UK", + "name": "HDR-UK 2024", + "startDate": "2024-04-15" + }, + { + "@id": "https://spdx.org/licenses/MIT", + "@type": "CreativeWork", + "name": "MIT Licence", + "identifier": "MIT" + } + ] +} \ No newline at end of file diff --git a/tests/metadata_integration.py b/tests/metadata_integration.py new file mode 100644 index 0000000..2196ddd --- /dev/null +++ b/tests/metadata_integration.py @@ -0,0 +1,53 @@ +import pytest +import subprocess +import time +import requests +import json +import os + + +@pytest.fixture(scope="session", autouse=True) +def docker_compose(): + """Start Docker Compose before tests, shut down after.""" + print("Starting Docker Compose...") + subprocess.run( + ["docker", "compose", "-f", "docker-compose-develop.yml", "up", "-d"], + check=True + ) + time.sleep(10) # Wait for services to start — adjust as needed + + yield # Run the tests + + print("Stopping Docker Compose...") + subprocess.run(["docker", "compose", "down"], check=True) + + + +def test_validate_metadata(): + url = "http://localhost:5001/ro_crates/validate_metadata" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # Load the JSON from file + filepath = os.path.join("tests/data", "ro-crate-metadata.json") + with open(filepath, "r", encoding="utf-8") as f: + crate_json_data = json.load(f) + + # The API expects the JSON to be passed as a string + payload = { + "crate_json": json.dumps(crate_json_data) + } + + response = requests.post(url, json=payload, headers=headers) + + response_result = json.loads(response.json()['result']) + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions — update based on expected API behavior + assert response.status_code == 200 + assert response_result['passed'] == True \ No newline at end of file From 13b4bedb264eed79c3d9a64f2bc943f539fbc71a Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 1 Jul 2025 22:44:55 +0100 Subject: [PATCH 004/127] gh action for integration test --- .github/workflows/test_docker.yml | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/test_docker.yml diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml new file mode 100644 index 0000000..db06678 --- /dev/null +++ b/.github/workflows/test_docker.yml @@ -0,0 +1,39 @@ +name: Integration Tests + +on: + pull_request: + branches: [ develop ] + +jobs: + test: + runs-on: ubuntu-latest + + services: + docker: + image: docker:24.0.6 + options: --privileged + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest requests + + - name: Build Docker Compose Containers + run: | + docker compose -f docker-compose-develop.yml build + + - name: Spin Up Docker Compose and Run Tests + run: pytest tests/metadata_integration.py -v + + - name: Ensure that Docker Compose is Shutdown + if: always() + run: docker compose down From f1f3dadf8cc4bce750a3f661030b371b3d71773e Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 1 Jul 2025 22:49:27 +0100 Subject: [PATCH 005/127] load example env for tests --- .github/workflows/test_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index db06678..d9f5ba6 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -29,6 +29,7 @@ jobs: - name: Build Docker Compose Containers run: | + cp example.env .env docker compose -f docker-compose-develop.yml build - name: Spin Up Docker Compose and Run Tests From 91314527ca6c255c2c52e70909f15d0752df5ed7 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 4 Jul 2025 10:05:25 +0100 Subject: [PATCH 006/127] PEP8 conformance for validation classes --- app/ro_crates/routes/post_routes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index f73326e..2b98d1a 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -15,18 +15,18 @@ post_routes_bp = APIBlueprint("post_routes", __name__) -class validate_data(Schema): +class ValidateData(Schema): crate_id = String(required=True) profile_name = String(required=False) webhook_url = String(required=False) -class validate_json(Schema): +class ValidateJSON(Schema): crate_json = String(required=True) profile_name = String(required=False) @post_routes_bp.post("/validate_by_id") -@post_routes_bp.input(validate_data(partial=True), location='json') +@post_routes_bp.input(ValidateData(partial=True), location='json') def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate using its ID from MinIO. @@ -60,7 +60,7 @@ def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url) @post_routes_bp.post("/validate_by_id_no_webhook") -@post_routes_bp.input(validate_data(partial=True), location='json') # -> json_data +@post_routes_bp.input(ValidateData(partial=True), location='json') # -> json_data def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate using its ID from MinIO. @@ -90,7 +90,7 @@ def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: @post_routes_bp.post("/validate_metadata") -@post_routes_bp.input(validate_json(partial=True), location='json') # -> json_data +@post_routes_bp.input(ValidateJSON(partial=True), location='json') # -> json_data def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate JSON file uploaded to the Service. From e004fda41b6575eab91c2c780494dc889719576f Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 4 Jul 2025 14:21:06 +0100 Subject: [PATCH 007/127] tests without loading docker service --- .github/workflows/test_docker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index d9f5ba6..bce7a70 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -8,11 +8,6 @@ jobs: test: runs-on: ubuntu-latest - services: - docker: - image: docker:24.0.6 - options: --privileged - steps: - name: Checkout code uses: actions/checkout@v4 From 252434af6fd1e4f8642e650a8da00144a8af5905 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:30:13 +0100 Subject: [PATCH 008/127] rename metadata_integration to test_integration --- .github/workflows/test_docker.yml | 2 +- tests/{metadata_integration.py => test_integration.py} | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename tests/{metadata_integration.py => test_integration.py} (96%) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index bce7a70..9befbba 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -28,7 +28,7 @@ jobs: docker compose -f docker-compose-develop.yml build - name: Spin Up Docker Compose and Run Tests - run: pytest tests/metadata_integration.py -v + run: pytest tests/test_integration.py -v - name: Ensure that Docker Compose is Shutdown if: always() diff --git a/tests/metadata_integration.py b/tests/test_integration.py similarity index 96% rename from tests/metadata_integration.py rename to tests/test_integration.py index 2196ddd..d426403 100644 --- a/tests/metadata_integration.py +++ b/tests/test_integration.py @@ -22,7 +22,6 @@ def docker_compose(): subprocess.run(["docker", "compose", "down"], check=True) - def test_validate_metadata(): url = "http://localhost:5001/ro_crates/validate_metadata" headers = { @@ -50,4 +49,4 @@ def test_validate_metadata(): # Assertions — update based on expected API behavior assert response.status_code == 200 - assert response_result['passed'] == True \ No newline at end of file + assert response_result['passed'] is True From 54fe23b1cf2105b6634f9bf21a5895631806e811 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:52:22 +0100 Subject: [PATCH 009/127] tests for metadata api --- tests/test_metadata.py | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/test_metadata.py diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 0000000..4744606 --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,63 @@ +from flask.testing import FlaskClient +import pytest +from unittest.mock import patch +from app import create_app + + +@pytest.fixture +def client(): + app = create_app() + return app.test_client() + + +def test_validate_metadata_with_all_fields(client: FlaskClient): + test_data = { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "profile_name": "default" + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: + mock_queue.return_value = ({"status": "success"}, 200) + + response = client.post("/ro_crates/validate_metadata", json=test_data) + print(response.json) + mock_queue.assert_called_once_with( + test_data["crate_json"], + test_data["profile_name"] + ) + assert response.status_code == 200 + assert response.json == {"status": "success"} + + +def test_validate_metadata_without_profile_name(client: FlaskClient): + """ + If the profile name is not specified then : + - 422 error + - Error message which includes 'Missing data for required field' + """ + test_data = { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}' + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: + mock_queue.return_value = ({"status": "success"}, 200) + + response = client.post("/ro_crates/validate_metadata", json=test_data) + mock_queue.assert_called_once_with(test_data["crate_json"], None) + assert response.status_code == 200 + assert response.json == {"status": "success"} + + +def test_validate_metadata_missing_crate_json(client: FlaskClient): + """ + If the RO-Crate is missing APIFlask should return: + - 422 error + - Error message which includes 'Missing data for required field' + """ + test_data = { + "profile_name": "default" + } + + response = client.post("/ro_crates/validate_metadata", json=test_data) + assert response.status_code == 422 + assert "Missing data for required field" in response.get_data(as_text=True) From 463345e06a6de47908ed956a3eb67515f6d02fd0 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 8 Jul 2025 11:30:11 +0100 Subject: [PATCH 010/127] test docstrings --- tests/test_metadata.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 4744606..f39f36f 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -11,6 +11,12 @@ def client(): def test_validate_metadata_with_all_fields(client: FlaskClient): + """ + If both profile name and crate json are provided then processing + of the RO-Crate metadata should continue, and return: + - 200 status code + - JSON data object which includes "status" which is "success" + """ test_data = { "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', "profile_name": "default" @@ -31,9 +37,10 @@ def test_validate_metadata_with_all_fields(client: FlaskClient): def test_validate_metadata_without_profile_name(client: FlaskClient): """ - If the profile name is not specified then : - - 422 error - - Error message which includes 'Missing data for required field' + If the profile name is not specified then processing + of the RO-Crate metadata should continue, and return: + - 200 status code + - JSON data object which includes "status" which is "success" """ test_data = { "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}' @@ -51,7 +58,7 @@ def test_validate_metadata_without_profile_name(client: FlaskClient): def test_validate_metadata_missing_crate_json(client: FlaskClient): """ If the RO-Crate is missing APIFlask should return: - - 422 error + - 422 status code - Error message which includes 'Missing data for required field' """ test_data = { From 1aaa885a7b399834af6440e4efe8497c963000cc Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 8 Jul 2025 11:47:52 +0100 Subject: [PATCH 011/127] switch API root from /ro_crates/ to /v1/ --- app/__init__.py | 6 +++--- app/ro_crates/routes/__init__.py | 10 ++-------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index cee8c91..184aaa1 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -9,7 +9,7 @@ from apiflask import APIFlask from app.celery_worker import make_celery, celery -from app.ro_crates.routes import ro_crates_post_bp, ro_crates_get_bp +from app.ro_crates.routes import v1_post_bp, v1_get_bp from app.utils.config import DevelopmentConfig, ProductionConfig, make_celery @@ -20,8 +20,8 @@ def create_app() -> APIFlask: :return: Flask: A configured Flask application instance. """ app = APIFlask(__name__) - app.register_blueprint(ro_crates_post_bp, url_prefix="/ro_crates") - app.register_blueprint(ro_crates_get_bp, url_prefix="/ro_crates") + app.register_blueprint(v1_post_bp, url_prefix="/v1") + app.register_blueprint(v1_get_bp, url_prefix="/v1") # Load configuration: if os.getenv("FLASK_ENV") == "production": diff --git a/app/ro_crates/routes/__init__.py b/app/ro_crates/routes/__init__.py index 0a20b53..da8a903 100644 --- a/app/ro_crates/routes/__init__.py +++ b/app/ro_crates/routes/__init__.py @@ -4,14 +4,8 @@ # License: MIT # Copyright (c) 2025 eScience Lab, The University of Manchester -from apiflask import APIBlueprint - from app.ro_crates.routes.post_routes import post_routes_bp from app.ro_crates.routes.get_routes import get_routes_bp -#ro_crates_bp = APIBlueprint("ro_crates", __name__) - -#ro_crates_bp.register_blueprint(post_routes_bp, url_prefix="/post") - -ro_crates_post_bp = post_routes_bp -ro_crates_get_bp = get_routes_bp \ No newline at end of file +v1_post_bp = post_routes_bp +v1_get_bp = get_routes_bp From b0166fa72bea0edc092f6ca25f5d3ebc1b6c8f71 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 8 Jul 2025 11:48:18 +0100 Subject: [PATCH 012/127] switch API root to /v1/ in tests --- tests/test_integration.py | 2 +- tests/test_metadata.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index d426403..fda2b14 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -23,7 +23,7 @@ def docker_compose(): def test_validate_metadata(): - url = "http://localhost:5001/ro_crates/validate_metadata" + url = "http://localhost:5001/v1/validate_metadata" headers = { "accept": "application/json", "Content-Type": "application/json" diff --git a/tests/test_metadata.py b/tests/test_metadata.py index f39f36f..64401dc 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -25,7 +25,7 @@ def test_validate_metadata_with_all_fields(client: FlaskClient): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: mock_queue.return_value = ({"status": "success"}, 200) - response = client.post("/ro_crates/validate_metadata", json=test_data) + response = client.post("/v1/validate_metadata", json=test_data) print(response.json) mock_queue.assert_called_once_with( test_data["crate_json"], @@ -49,7 +49,7 @@ def test_validate_metadata_without_profile_name(client: FlaskClient): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: mock_queue.return_value = ({"status": "success"}, 200) - response = client.post("/ro_crates/validate_metadata", json=test_data) + response = client.post("/v1/validate_metadata", json=test_data) mock_queue.assert_called_once_with(test_data["crate_json"], None) assert response.status_code == 200 assert response.json == {"status": "success"} @@ -65,6 +65,6 @@ def test_validate_metadata_missing_crate_json(client: FlaskClient): "profile_name": "default" } - response = client.post("/ro_crates/validate_metadata", json=test_data) + response = client.post("/v1/validate_metadata", json=test_data) assert response.status_code == 422 assert "Missing data for required field" in response.get_data(as_text=True) From e4e1812a511cd0e20b942d47718493a01ba2ac9b Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 14 Jul 2025 16:16:45 +0100 Subject: [PATCH 013/127] update path in metadata tests --- tests/test_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 64401dc..83efe04 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -25,7 +25,7 @@ def test_validate_metadata_with_all_fields(client: FlaskClient): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: mock_queue.return_value = ({"status": "success"}, 200) - response = client.post("/v1/validate_metadata", json=test_data) + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) print(response.json) mock_queue.assert_called_once_with( test_data["crate_json"], @@ -49,7 +49,7 @@ def test_validate_metadata_without_profile_name(client: FlaskClient): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: mock_queue.return_value = ({"status": "success"}, 200) - response = client.post("/v1/validate_metadata", json=test_data) + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) mock_queue.assert_called_once_with(test_data["crate_json"], None) assert response.status_code == 200 assert response.json == {"status": "success"} @@ -65,6 +65,6 @@ def test_validate_metadata_missing_crate_json(client: FlaskClient): "profile_name": "default" } - response = client.post("/v1/validate_metadata", json=test_data) + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) assert response.status_code == 422 assert "Missing data for required field" in response.get_data(as_text=True) From e5651ebfa92f0f1782b260cf591b1ef3cd8f4dd6 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 14 Jul 2025 16:23:15 +0100 Subject: [PATCH 014/127] integration test for metadata updated to /vi/ro_crates path --- tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index fda2b14..bdf1030 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -23,7 +23,7 @@ def docker_compose(): def test_validate_metadata(): - url = "http://localhost:5001/v1/validate_metadata" + url = "http://localhost:5001/v1/ro_crates/validate_metadata" headers = { "accept": "application/json", "Content-Type": "application/json" From 0a32d983bf25bd823a77b54db5e77d3b644b01f3 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 14 Jul 2025 16:23:47 +0100 Subject: [PATCH 015/127] app path changed to /v1/ro_crates --- app/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 184aaa1..b67f34b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -20,8 +20,8 @@ def create_app() -> APIFlask: :return: Flask: A configured Flask application instance. """ app = APIFlask(__name__) - app.register_blueprint(v1_post_bp, url_prefix="/v1") - app.register_blueprint(v1_get_bp, url_prefix="/v1") + app.register_blueprint(v1_post_bp, url_prefix="/v1/ro_crates") + app.register_blueprint(v1_get_bp, url_prefix="/v1/ro_crates") # Load configuration: if os.getenv("FLASK_ENV") == "production": From 12bb41c20f9121e095a94a81bb4505ce0a11c9b1 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 14 Jul 2025 22:55:03 +0100 Subject: [PATCH 016/127] minio util tests --- tests/test_minio.py | 304 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 tests/test_minio.py diff --git a/tests/test_minio.py b/tests/test_minio.py new file mode 100644 index 0000000..18938e1 --- /dev/null +++ b/tests/test_minio.py @@ -0,0 +1,304 @@ +import os +import tempfile +import json +import pytest +from io import BytesIO +from minio import Minio +from minio.error import S3Error +from unittest.mock import MagicMock +from unittest import mock + + +@pytest.fixture +def mock_minio_response(): + response = MagicMock() + response.data.decode.return_value = json.dumps({"status": "valid"}) + return response + +# @pytest.fixture +# def mock_minio_response(): +# mock_response = mock.Mock() +# return mock_response +# mock_response.data.decode.return_value = json.dumps({"status": "valid"}) + +# Testing function: get_minio_client_and_bucket + +def test_get_minio_client_success(monkeypatch): + # Set required env vars + monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000") + monkeypatch.setenv("MINIO_ROOT_USER", "admin") + monkeypatch.setenv("MINIO_ROOT_PASSWORD", "password123") + monkeypatch.setenv("MINIO_BUCKET_NAME", "test-bucket") + + from app.utils.minio_utils import get_minio_client_and_bucket + client, bucket = get_minio_client_and_bucket() + + assert isinstance(client, Minio) + assert bucket == "test-bucket" + assert client._base_url.host == "localhost:9000" + + +def test_get_minio_client_missing_bucket_name(monkeypatch): + # Set all except MINIO_BUCKET_NAME + monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000") + monkeypatch.setenv("MINIO_ROOT_USER", "admin") + monkeypatch.setenv("MINIO_ROOT_PASSWORD", "password123") + monkeypatch.setenv("MINIO_BUCKET_NAME", "") + + from app.utils.minio_utils import get_minio_client_and_bucket + with pytest.raises(ValueError, match="MINIO_BUCKET_NAME is not set"): + get_minio_client_and_bucket() + + +# Testing function: get_validation_status_from_minio + +def test_successful_retrieval(mocker, mock_minio_response): + mock_client = MagicMock() + mock_bucket = "test-bucket" + mock_client.get_object.return_value = mock_minio_response + mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) + + from app.utils.minio_utils import get_validation_status_from_minio + result = get_validation_status_from_minio("crate123") + + assert result == {"status": "valid"} + mock_minio_response.close.assert_called_once() + mock_minio_response.release_conn.assert_called_once() + + +def test_s3_error_raised(mocker): + mock_client = MagicMock() + mock_bucket = "test-bucket" + mock_client.get_object.side_effect = S3Error( + code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None + ) + mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) + + from app.utils.minio_utils import get_validation_status_from_minio + with pytest.raises(S3Error): + get_validation_status_from_minio("crate123") + + +def test_value_error_raised(mocker): + mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing env var")) + + from app.utils.minio_utils import get_validation_status_from_minio + with pytest.raises(ValueError): + get_validation_status_from_minio("crate123") + + +def test_generic_exception_raised(mocker): + mock_client = MagicMock() + mock_bucket = "test-bucket" + mock_client.get_object.side_effect = Exception("Unexpected failure") + mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) + + from app.utils.minio_utils import get_validation_status_from_minio + with pytest.raises(Exception): + get_validation_status_from_minio("crate123") + + +# Testing function: fetch_ro_crate_from_minio + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +@mock.patch("app.utils.minio_utils.tempfile.mkdtemp") +@mock.patch("app.utils.minio_utils.os.path.join", side_effect=os.path.join) +def test_fetch_ro_crate_success(mock_join, mock_mkdtemp, mock_get_client, mock_load_dotenv): + # Setup mocks + mock_minio_client = mock.Mock() + mock_get_client.return_value = (mock_minio_client, "test-bucket") + mock_mkdtemp.return_value = "/tmp/testdir" + + crate_id = "test_crate" + expected_path = os.path.join("/tmp/testdir", f"{crate_id}.zip") + + from app.utils.minio_utils import fetch_ro_crate_from_minio + + result = fetch_ro_crate_from_minio(crate_id) + + mock_load_dotenv.assert_called_once() + mock_get_client.assert_called_once() + mock_minio_client.fget_object.assert_called_once_with( + "test-bucket", f"{crate_id}.zip", expected_path + ) + + assert result == expected_path + + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +@mock.patch("app.utils.minio_utils.tempfile.mkdtemp", return_value="/tmp/testdir") +def test_fetch_ro_crate_s3_error(mock_mkdtemp, mock_get_client, mock_load_dotenv): + mock_minio_client = mock.Mock() + mock_get_client.return_value = (mock_minio_client, "test-bucket") + + crate_id = "bad_crate" + mock_minio_client.fget_object.side_effect = S3Error( + code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None + ) + + from app.utils.minio_utils import fetch_ro_crate_from_minio + with pytest.raises(S3Error): + fetch_ro_crate_from_minio(crate_id) + + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing config")) +def test_fetch_ro_crate_value_error(mock_get_client, mock_load_dotenv): + from app.utils.minio_utils import fetch_ro_crate_from_minio + with pytest.raises(ValueError): + fetch_ro_crate_from_minio("any_crate") + + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +@mock.patch("app.utils.minio_utils.tempfile.mkdtemp", return_value="/tmp/testdir") +def test_fetch_ro_crate_unexpected_error(mock_mkdtemp, mock_get_client, mock_load_dotenv): + mock_minio_client = mock.Mock() + mock_get_client.return_value = (mock_minio_client, "test-bucket") + mock_minio_client.fget_object.side_effect = RuntimeError("Unexpected failure") + + from app.utils.minio_utils import fetch_ro_crate_from_minio + with pytest.raises(RuntimeError): + fetch_ro_crate_from_minio("any_crate") + + +# Testing function: update_validation_status_in_minio + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_update_validation_status_success(mock_get_client, mock_load_dotenv): + mock_minio_client = mock.Mock() + mock_get_client.return_value = (mock_minio_client, "test-bucket") + + crate_id = "crate123" + validation_status = json.dumps({"status": "valid", "errors": []}) + + from app.utils.minio_utils import update_validation_status_in_minio + update_validation_status_in_minio(crate_id, validation_status) + + expected_object_name = f"{crate_id}/validation_status.txt" + expected_data = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") + + mock_minio_client.put_object.assert_called_once() + args, kwargs = mock_minio_client.put_object.call_args + + # FIXME: Original suggested test expected 4 values in args, but returned only 2. + # Solution was to check both args and kwargs for the 'data' and 'length' objects. + # Do we need to chose one format of call_args for our tests, or is this ambiguity okay? + bucket_name = args[0] if args else kwargs["bucket_name"] + object_name = args[1] if len(args) > 1 else kwargs["object_name"] + actual_data_stream = args[2] if len(args) > 2 else kwargs["data"] + length = args[3] if len(args) > 3 else kwargs["length"] + + assert bucket_name == "test-bucket" + assert object_name == expected_object_name + assert isinstance(actual_data_stream, BytesIO) + actual_data_stream.seek(0) + assert actual_data_stream.read() == expected_data + assert length == len(expected_data) + assert kwargs["content_type"] == "application/json" + + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_update_validation_status_s3_error(mock_get_client, mock_load_dotenv): + mock_minio_client = mock.Mock() + mock_get_client.return_value = (mock_minio_client, "test-bucket") + mock_minio_client.put_object.side_effect = S3Error( + code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None + ) + + from app.utils.minio_utils import update_validation_status_in_minio + with pytest.raises(S3Error): + update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing env vars")) +def test_update_validation_status_value_error(mock_get_client, mock_load_dotenv): + from app.utils.minio_utils import update_validation_status_in_minio + with pytest.raises(ValueError): + update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + + +@mock.patch("app.utils.minio_utils.load_dotenv") +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_update_validation_status_unexpected_error(mock_get_client, mock_load_dotenv): + mock_minio_client = mock.Mock() + mock_get_client.return_value = (mock_minio_client, "test-bucket") + mock_minio_client.put_object.side_effect = RuntimeError("Unexpected failure") + + from app.utils.minio_utils import update_validation_status_in_minio + with pytest.raises(RuntimeError): + update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + + +# Testing function: get_validation_status_from_minio + +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_get_validation_status_success(mock_get_client, mock_minio_response): + mock_minio_client = mock.Mock() + mock_minio_client.get_object.return_value = mock_minio_response + mock_get_client.return_value = (mock_minio_client, "test-bucket") + + crate_id = "crate123" + from app.utils.minio_utils import get_validation_status_from_minio + result = get_validation_status_from_minio(crate_id) + + assert result == {"status": "valid"} + mock_minio_client.get_object.assert_called_once_with("test-bucket", f"{crate_id}/validation_status.txt") + mock_minio_response.close.assert_called_once() + mock_minio_response.release_conn.assert_called_once() + + +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_get_validation_status_s3_error(mock_get_client): + mock_minio_client = mock.Mock() + mock_minio_client.get_object.side_effect = S3Error( + code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None + ) + mock_get_client.return_value = (mock_minio_client, "test-bucket") + + from app.utils.minio_utils import get_validation_status_from_minio + with pytest.raises(S3Error): + get_validation_status_from_minio("crate123") + + +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing config")) +def test_get_validation_status_value_error(mock_get_client): + from app.utils.minio_utils import get_validation_status_from_minio + with pytest.raises(ValueError): + get_validation_status_from_minio("crate123") + + +@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_get_validation_status_generic_exception(mock_get_client): + mock_minio_client = mock.Mock() + mock_minio_client.get_object.side_effect = RuntimeError("Unexpected failure") + mock_get_client.return_value = (mock_minio_client, "test-bucket") + + from app.utils.minio_utils import get_validation_status_from_minio + with pytest.raises(RuntimeError): + get_validation_status_from_minio("crate123") From bef24d379d008d2a1e7f18a5942c4576ca119bdb Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 14:20:00 +0100 Subject: [PATCH 017/127] remove unneeded commented code --- tests/test_minio.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 18938e1..0da8a0b 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -15,12 +15,6 @@ def mock_minio_response(): response.data.decode.return_value = json.dumps({"status": "valid"}) return response -# @pytest.fixture -# def mock_minio_response(): -# mock_response = mock.Mock() -# return mock_response -# mock_response.data.decode.return_value = json.dumps({"status": "valid"}) - # Testing function: get_minio_client_and_bucket def test_get_minio_client_success(monkeypatch): From 0bdad8f5711517c25c93176c00a56293c55b28a7 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 14:27:00 +0100 Subject: [PATCH 018/127] add unit tests to GH workflow --- .github/workflows/unit_tests.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/unit_tests.yml diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 0000000..d53568a --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,28 @@ +name: Unit Tests + +on: + pull_request: + branches: [ develop ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest + + - name: Run tests (excluding integration tests) + run: | + pytest --ignore=tests/test_integration.py From 3b50b9f02762e26e511fe5650500c87471169b36 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 14:46:24 +0100 Subject: [PATCH 019/127] try running pytest as python module --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d53568a..2912dd2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -25,4 +25,4 @@ jobs: - name: Run tests (excluding integration tests) run: | - pytest --ignore=tests/test_integration.py + python -m pytest --ignore=tests/test_integration.py From dc20f575651de7bae10e3b6428de45970bd37b48 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 14:54:58 +0100 Subject: [PATCH 020/127] add pytest-mock to pip install --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 2912dd2..7c0a996 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -21,7 +21,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - pip install pytest + pip install pytest pytest-mock - name: Run tests (excluding integration tests) run: | From c700fb36925ed0f2d4a293b86d9c43de7caf59b4 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 15:01:14 +0100 Subject: [PATCH 021/127] remove partial option for validating json --- app/ro_crates/routes/post_routes.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 2b98d1a..9f9f909 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -5,25 +5,27 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester from apiflask import APIBlueprint, Schema -from apiflask.fields import Integer, String -from flask import request, Response +from apiflask.fields import String +from flask import Response from app.services.validation_service import ( - queue_ro_crate_validation_task, + queue_ro_crate_validation_task, queue_ro_crate_metadata_validation_task ) post_routes_bp = APIBlueprint("post_routes", __name__) + class ValidateData(Schema): crate_id = String(required=True) profile_name = String(required=False) webhook_url = String(required=False) + class ValidateJSON(Schema): crate_json = String(required=True) profile_name = String(required=False) - + @post_routes_bp.post("/validate_by_id") @post_routes_bp.input(ValidateData(partial=True), location='json') @@ -59,8 +61,9 @@ def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url) + @post_routes_bp.post("/validate_by_id_no_webhook") -@post_routes_bp.input(ValidateData(partial=True), location='json') # -> json_data +@post_routes_bp.input(ValidateData(partial=True), location='json') # -> json_data def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate using its ID from MinIO. @@ -90,7 +93,7 @@ def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: @post_routes_bp.post("/validate_metadata") -@post_routes_bp.input(ValidateJSON(partial=True), location='json') # -> json_data +@post_routes_bp.input(ValidateJSON(partial=False), location='json') # -> json_data def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate JSON file uploaded to the Service. @@ -117,4 +120,3 @@ def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: profile_name = None return queue_ro_crate_metadata_validation_task(crate_json, profile_name) - From 3c6e3ba59fddf885c4793132d7cffe963b518d13 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:50:39 +0100 Subject: [PATCH 022/127] tests for validation tasks --- tests/test_validation_tasks.py | 306 +++++++++++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 tests/test_validation_tasks.py diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py new file mode 100644 index 0000000..b6bf70e --- /dev/null +++ b/tests/test_validation_tasks.py @@ -0,0 +1,306 @@ +import os +import pytest +from unittest import mock +from rocrate_validator.models import ValidationResult + +from app.tasks.validation_tasks import process_validation_task_by_id +from app.tasks.validation_tasks import perform_ro_crate_validation +from app.tasks.validation_tasks import return_ro_crate_validation +from app.tasks.validation_tasks import process_validation_task_by_metadata + + +# Test function: process_validation_task_by_id + +@mock.patch("app.tasks.validation_tasks.os.remove") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") +def test_process_validation_success( + mock_fetch, + mock_validate, + mock_update, + mock_webhook, + mock_exists, + mock_remove +): + mock_fetch.return_value = "/tmp/crate.zip" + + mock_validation_result = mock.Mock() + mock_validation_result.has_issues.return_value = False + mock_validation_result.to_json.return_value = '{"status": "valid"}' + mock_validate.return_value = mock_validation_result + + process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + + mock_fetch.assert_called_once_with("crate123") + mock_validate.assert_called_once_with("/tmp/crate.zip", "profileA") + mock_update.assert_called_once_with("crate123", '{"status": "valid"}') + mock_webhook.assert_called_once_with("https://example.com/hook", '{"status": "valid"}') + mock_remove.assert_called_once_with("/tmp/crate.zip") + + +@mock.patch("app.tasks.validation_tasks.os.remove") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") +def test_process_validation_fails_with_message( + mock_fetch, + mock_validate, + mock_update, + mock_webhook, + mock_exists, + mock_remove +): + mock_fetch.return_value = "/tmp/crate.zip" + mock_validate.return_value = "Validation failed" + + process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + + mock_update.assert_not_called() + mock_webhook.assert_called_once() + args, kwargs = mock_webhook.call_args + assert args[0] == "https://example.com/hook" + assert "Validation failed" in args[1]["error"] + mock_remove.assert_called_once_with("/tmp/crate.zip") + + +@mock.patch("app.tasks.validation_tasks.os.remove") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation", side_effect=Exception("Unexpected error")) +@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") +def test_process_validation_exception( + mock_fetch, + mock_validate, + mock_update, + mock_webhook, + mock_exists, + mock_remove +): + mock_fetch.return_value = "/tmp/crate.zip" + + process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + + mock_update.assert_not_called() + mock_webhook.assert_called_once() + args, kwargs = mock_webhook.call_args + assert args[0] == "https://example.com/hook" + assert "Unexpected error" in args[1]["error"] + mock_remove.assert_called_once_with("/tmp/crate.zip") + + +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=False) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio", side_effect=Exception("MinIO fetch failed")) +def test_process_validation_fetch_error( + mock_fetch, + mock_validate, + mock_update, + mock_webhook, + mock_exists +): + process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + + mock_validate.assert_not_called() + mock_update.assert_not_called() + mock_webhook.assert_called_once() + args, kwargs = mock_webhook.call_args + assert args[0] == "https://example.com/hook" + assert "MinIO fetch failed" in args[1]["error"] + + +# Test function: process_validation_task_by_metadata + +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") +def test_validation_success_no_issues( + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree +): + crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' + profile_name = "test-profile" + webhook_url = "https://example.com/webhook" + + mock_path = "/tmp/crate" + mock_build.return_value = mock_path + + mock_result = mock.Mock() + mock_result.has_issues.return_value = False + mock_result.to_json.return_value = '{"status": "valid"}' + mock_validate.return_value = mock_result + + result = process_validation_task_by_metadata(crate_json, profile_name, webhook_url) + + assert result == '{"status": "valid"}' + mock_build.assert_called_once_with(crate_json) + mock_validate.assert_called_once() + mock_webhook.assert_called_once_with(webhook_url, '{"status": "valid"}') + mock_rmtree.assert_called_once_with(mock_path) + + +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") +def test_validation_success_with_issues( + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree +): + crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' + mock_path = "/tmp/crate" + mock_build.return_value = mock_path + + mock_result = mock.Mock() + mock_result.has_issues.return_value = True + mock_result.to_json.return_value = '{"status": "invalid"}' + mock_validate.return_value = mock_result + + result = process_validation_task_by_metadata(crate_json, None, None) + + assert result == '{"status": "invalid"}' + mock_webhook.assert_not_called() + mock_rmtree.assert_called_once_with(mock_path) + + +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation", return_value="Validation error") +@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") +def test_validation_fails_and_sends_error_notification_to_webhook( + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree +): + crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' + mock_path = "/tmp/crate" + mock_build.return_value = mock_path + + result = process_validation_task_by_metadata(crate_json, "profileX", "https://webhook.test") + + assert isinstance(result, str) + assert "Validation error" in result + + # Error webhook should be sent + mock_webhook.assert_called_once() + args, kwargs = mock_webhook.call_args + assert kwargs is None or "error" in args[1] + + mock_rmtree.assert_called_once_with(mock_path) + + +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation", return_value="Validation error") +@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") +def test_validation_fails_with_no_webhook( + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree +): + crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' + mock_path = "/tmp/crate" + mock_build.return_value = mock_path + + result = process_validation_task_by_metadata(crate_json, "profileX", None) + + assert isinstance(result, str) + assert "Validation error" in result + + # Make sure webhook not sent + mock_webhook.assert_not_called() + mock_rmtree.assert_called_once_with(mock_path) + + +# Test function: perform_ro_crate_validation + +@mock.patch("app.tasks.validation_tasks.services.validate") +@mock.patch("app.tasks.validation_tasks.services.ValidationSettings") +def test_validation_success_with_all_args(mock_validation_settings, mock_validate): + mock_result = mock.Mock() + mock_validate.return_value = mock_result + + file_path = "crates/test_crate" + profile_name = "ro_profile" + skip_checks = ["check1", "check2"] + + result = perform_ro_crate_validation(file_path, profile_name, skip_checks) + + # Assert that result was returned + assert result == mock_result + + # Validate proper construction of ValidationSettings + mock_validation_settings.assert_called_once() + args, kwargs = mock_validation_settings.call_args + assert kwargs["rocrate_uri"].endswith(file_path) + assert kwargs["profile_identifier"] == profile_name + assert kwargs["skip_checks"] == skip_checks + + mock_validate.assert_called_once_with(mock_validation_settings.return_value) + + +@mock.patch("app.tasks.validation_tasks.services.validate") +@mock.patch("app.tasks.validation_tasks.services.ValidationSettings") +def test_validation_success_with_minimal_args(mock_validation_settings, mock_validate): + mock_result = mock.Mock() + mock_validate.return_value = mock_result + + file_path = "crates/test_crate" + result = perform_ro_crate_validation(file_path, None) + + assert result == mock_result + + args, kwargs = mock_validation_settings.call_args + assert "profile_identifier" not in kwargs + assert "skip_checks" not in kwargs + + +@mock.patch("app.tasks.validation_tasks.services.validate", side_effect=RuntimeError("Validation error")) +@mock.patch("app.tasks.validation_tasks.services.ValidationSettings") +def test_validation_raises_exception_and_returns_string(mock_validation_settings, mock_validate): + file_path = "crates/test_crate" + result = perform_ro_crate_validation(file_path, "profile", skip_checks_list=None) + + assert isinstance(result, str) + assert "Validation error" in result + mock_validate.assert_called_once() + + +@mock.patch("app.tasks.validation_tasks.services.validate") +@mock.patch("app.tasks.validation_tasks.services.ValidationSettings", side_effect=ValueError("Bad config")) +def test_validation_settings_error(mock_validation_settings, mock_validate): + file_path = "crates/test_crate" + result = perform_ro_crate_validation(file_path, None) + + assert isinstance(result, str) + assert "Bad config" in result + mock_validate.assert_not_called() + + +# Test function: return_ro_crate_validation + +@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") +def test_return_validation_success(mock_get_validation): + mock_get_validation.return_value = {"status": "valid"} + + crate_id = "crate-123" + result = return_ro_crate_validation(crate_id) + + assert result == {"status": "valid"} + mock_get_validation.assert_called_once_with(crate_id) + + +@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio", side_effect=Exception("Fetch failed")) +def test_return_validation_exception(mock_get_validation): + crate_id = "crate-123" + result = return_ro_crate_validation(crate_id) + + assert isinstance(result, str) + assert "Fetch failed" in result + mock_get_validation.assert_called_once_with(crate_id) From 1c548c1c69de2c0ee71968d2b89a21ddf16e85ad Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:51:49 +0100 Subject: [PATCH 023/127] correct edgecases for process_validation_task_by_metadata --- app/tasks/validation_tasks.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index 261aa67..f8295e7 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -109,7 +109,7 @@ def process_validation_task_by_metadata( validation_result = perform_ro_crate_validation(file_path, profile_name, skip_checks_list - ) + ) if isinstance(validation_result, str): logging.error(f"Validation failed: {validation_result}") @@ -129,15 +129,18 @@ def process_validation_task_by_metadata( # Send failure notification via webhook error_data = {"profile_name": profile_name, "error": str(e)} - send_webhook_notification(webhook_url, error_data) + if webhook_url: + send_webhook_notification(webhook_url, error_data) finally: # Clean up the temporary file if it was created: if file_path and os.path.exists(file_path): shutil.rmtree(file_path) - return validation_result.to_json() - + if isinstance(validation_result, str): + return validation_result + else: + return validation_result.to_json() def perform_ro_crate_validation( From 9cd24d7abc912fb26b5aca96d4dddd0f9d25564f Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 15:19:14 +0100 Subject: [PATCH 024/127] tidy up function loading in validation tests --- tests/test_validation_tasks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index b6bf70e..ae49c03 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -3,10 +3,12 @@ from unittest import mock from rocrate_validator.models import ValidationResult -from app.tasks.validation_tasks import process_validation_task_by_id -from app.tasks.validation_tasks import perform_ro_crate_validation -from app.tasks.validation_tasks import return_ro_crate_validation -from app.tasks.validation_tasks import process_validation_task_by_metadata +from app.tasks.validation_tasks import ( + process_validation_task_by_id, + perform_ro_crate_validation, + return_ro_crate_validation, + process_validation_task_by_metadata +) # Test function: process_validation_task_by_id From 7c554027f28cd77b96663fd75eb1ac7681605289 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 15:24:35 +0100 Subject: [PATCH 025/127] removed unneeded module loads in validation tests --- tests/test_validation_tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index ae49c03..78b786c 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -1,7 +1,4 @@ -import os -import pytest from unittest import mock -from rocrate_validator.models import ValidationResult from app.tasks.validation_tasks import ( process_validation_task_by_id, From f14f595440e04feb88e461da080638ae5750bfba Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 17 Jul 2025 15:26:59 +0100 Subject: [PATCH 026/127] tidy small parts of minio test code --- tests/test_minio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 0da8a0b..3c9f836 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -1,5 +1,4 @@ import os -import tempfile import json import pytest from io import BytesIO @@ -17,6 +16,7 @@ def mock_minio_response(): # Testing function: get_minio_client_and_bucket + def test_get_minio_client_success(monkeypatch): # Set required env vars monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000") @@ -190,7 +190,7 @@ def test_update_validation_status_success(mock_get_client, mock_load_dotenv): # FIXME: Original suggested test expected 4 values in args, but returned only 2. # Solution was to check both args and kwargs for the 'data' and 'length' objects. - # Do we need to chose one format of call_args for our tests, or is this ambiguity okay? + # Do we need to chose one format of call_args for our tests, or is this ambiguity okay? bucket_name = args[0] if args else kwargs["bucket_name"] object_name = args[1] if len(args) > 1 else kwargs["object_name"] actual_data_stream = args[2] if len(args) > 2 else kwargs["data"] From 647d2e5bbbe5ac2fd3aa8af1caea333c56aec8cd Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 18 Jul 2025 13:52:47 +0100 Subject: [PATCH 027/127] tests for validation services functions --- tests/test_services.py | 116 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tests/test_services.py diff --git a/tests/test_services.py b/tests/test_services.py new file mode 100644 index 0000000..9aa24e5 --- /dev/null +++ b/tests/test_services.py @@ -0,0 +1,116 @@ +import pytest +from unittest.mock import patch, MagicMock +from flask import Flask + +from app.services.validation_service import ( + queue_ro_crate_validation_task, + queue_ro_crate_metadata_validation_task, + get_ro_crate_validation_task +) + + +@pytest.fixture +def flask_app(): + app = Flask(__name__) + with app.app_context(): + yield app + + +# Test function: queue_ro_crate_validation_task + +def test_queue_task_success(flask_app): + with patch("app.services.validation_service.process_validation_task_by_id.delay") as mock_delay: + response, status_code = queue_ro_crate_validation_task("crate123", "profileA", "http://webhook.com") + + mock_delay.assert_called_once_with("crate123", "profileA", "http://webhook.com") + assert status_code == 202 + assert response.json == {"message": "Validation in progress"} + + +def test_queue_task_missing_crate_id(flask_app): + response, status_code = queue_ro_crate_validation_task(None) + + assert status_code == 400 + assert response.json == {"error": "Missing required parameter: crate_id"} + + +def test_queue_task_exception(flask_app): + with patch("app.services.validation_service.process_validation_task_by_id.delay", side_effect=Exception("Celery down")): + response, status_code = queue_ro_crate_validation_task("crate123") + + assert status_code == 500 + assert response.json == {"error": "Celery down"} + + +# Test function: queue_ro_crate_metadata_validation_task + +def test_queue_metadata_with_webhook(flask_app): + with patch("app.services.validation_service.process_validation_task_by_metadata.delay") as mock_delay: + mock_result = MagicMock() + mock_delay.return_value = mock_result + + crate_json = {"@context": "https://w3id.org/ro/crate/1.1/context"} + response, status = queue_ro_crate_metadata_validation_task(crate_json, "profile", "http://webhook") + + mock_delay.assert_called_once_with(crate_json, "profile", "http://webhook") + assert status == 202 + assert response.json == {"message": "Validation in progress"} + + +def test_queue_metadata_without_webhook(flask_app): + with patch("app.services.validation_service.process_validation_task_by_metadata.delay") as mock_delay: + mock_result = MagicMock() + mock_result.get.return_value = {"status": "ok"} + mock_delay.return_value = mock_result + + crate_json = {"@context": "https://w3id.org/ro/crate/1.1/context"} + response, status = queue_ro_crate_metadata_validation_task(crate_json, "profile", None) + + mock_delay.assert_called_once_with(crate_json, "profile", None) + assert status == 200 + assert response.json == {"result": {"status": "ok"}} + + +def test_queue_metadata_missing_json(flask_app): + response, status = queue_ro_crate_metadata_validation_task(None) + + assert status == 400 + assert response.json == {"error": "Missing required parameter: crate_json"} + + +def test_queue_metadata_exception(flask_app): + with patch("app.services.validation_service.process_validation_task_by_metadata.delay", + side_effect=Exception("Celery error")): + crate_json = {"@context": "https://w3id.org/ro/crate/1.1/context"} + response, status = queue_ro_crate_metadata_validation_task(crate_json) + + assert status == 500 + assert response.json == {"error": "Celery error"} + + +# Test function: get_ro_crate_validation_task + +def test_get_validation_success(flask_app): + with patch("app.services.validation_service.return_ro_crate_validation") as mock_return: + mock_return.return_value = {"status": "valid"} + + response, status = get_ro_crate_validation_task("crate123") + + mock_return.assert_called_once_with("crate123") + assert status == 200 + assert response == {"status": "valid"} + + +def test_get_validation_missing_id(flask_app): + response, status = get_ro_crate_validation_task(None) + + assert status == 400 + assert response.json == {"error": "Missing required parameter: crate_id"} + + +def test_get_validation_exception(flask_app): + with patch("app.services.validation_service.return_ro_crate_validation", side_effect=Exception("DB error")): + response, status = get_ro_crate_validation_task("crate123") + + assert status == 500 + assert response.json == {"service error": "DB error"} From 3a3888f46202a24e0f0305ff8f4a2749f881da7b Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 20 Jul 2025 21:23:47 +0100 Subject: [PATCH 028/127] remove redundant celery definitions --- app/__init__.py | 1 - app/celery_worker.py | 46 -------------------------------------------- 2 files changed, 47 deletions(-) delete mode 100644 app/celery_worker.py diff --git a/app/__init__.py b/app/__init__.py index b67f34b..3f56fdf 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -8,7 +8,6 @@ from apiflask import APIFlask -from app.celery_worker import make_celery, celery from app.ro_crates.routes import v1_post_bp, v1_get_bp from app.utils.config import DevelopmentConfig, ProductionConfig, make_celery diff --git a/app/celery_worker.py b/app/celery_worker.py deleted file mode 100644 index d3be21a..0000000 --- a/app/celery_worker.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Configures and initialises a Celery instance for use with a Flask application.""" - -# Author: Alexander Hambley -# License: MIT -# Copyright (c) 2025 eScience Lab, The University of Manchester - -from celery import Celery -from flask import Flask - - -def make_celery(app: Flask = None) -> Celery: - """ - Create and configure a Celery instance using Flask configuration. - - :param app: The Flask application. Configuration will be used to initialise Celery. - :return: Celery: A configured Celery instance. - :raises ValueError: If the Flask configuration values are not provided. - """ - if not app: - raise ValueError( - "A Flask application instance must be provided to configure Celery." - ) - - if "CELERY_RESULT_BACKEND" not in app.config: - raise ValueError( - "Missing configuration: 'CELERY_RESULT_BACKEND' is not defined in the Flask app config." - ) - - if "CELERY_BROKER_URL" not in app.config: - raise ValueError( - "Missing configuration: 'CELERY_BROKER_URL' is not defined in the Flask app config." - ) - - celery_instance = Celery( - app.import_name, - backend=app.config["CELERY_RESULT_BACKEND"], - broker=app.config["CELERY_BROKER_URL"], - ) - - if app: - celery_instance.conf.update(app.config) - - return celery_instance - - -celery = Celery() From dc670e1f1bebb70961294381631819760edc5340 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 20 Jul 2025 21:49:15 +0100 Subject: [PATCH 029/127] Revert "remove redundant celery definitions" This reverts commit 83a728f07b4e73aca53bdac85d7de806a7402362. --- app/__init__.py | 1 + app/celery_worker.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 app/celery_worker.py diff --git a/app/__init__.py b/app/__init__.py index 3f56fdf..b67f34b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -8,6 +8,7 @@ from apiflask import APIFlask +from app.celery_worker import make_celery, celery from app.ro_crates.routes import v1_post_bp, v1_get_bp from app.utils.config import DevelopmentConfig, ProductionConfig, make_celery diff --git a/app/celery_worker.py b/app/celery_worker.py new file mode 100644 index 0000000..d3be21a --- /dev/null +++ b/app/celery_worker.py @@ -0,0 +1,46 @@ +"""Configures and initialises a Celery instance for use with a Flask application.""" + +# Author: Alexander Hambley +# License: MIT +# Copyright (c) 2025 eScience Lab, The University of Manchester + +from celery import Celery +from flask import Flask + + +def make_celery(app: Flask = None) -> Celery: + """ + Create and configure a Celery instance using Flask configuration. + + :param app: The Flask application. Configuration will be used to initialise Celery. + :return: Celery: A configured Celery instance. + :raises ValueError: If the Flask configuration values are not provided. + """ + if not app: + raise ValueError( + "A Flask application instance must be provided to configure Celery." + ) + + if "CELERY_RESULT_BACKEND" not in app.config: + raise ValueError( + "Missing configuration: 'CELERY_RESULT_BACKEND' is not defined in the Flask app config." + ) + + if "CELERY_BROKER_URL" not in app.config: + raise ValueError( + "Missing configuration: 'CELERY_BROKER_URL' is not defined in the Flask app config." + ) + + celery_instance = Celery( + app.import_name, + backend=app.config["CELERY_RESULT_BACKEND"], + broker=app.config["CELERY_BROKER_URL"], + ) + + if app: + celery_instance.conf.update(app.config) + + return celery_instance + + +celery = Celery() From e760dc36d9b364ea036ce0fdc3249a04c133c48b Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 20 Jul 2025 21:50:33 +0100 Subject: [PATCH 030/127] remove only redundant make_celery function --- app/__init__.py | 1 - app/celery_worker.py | 36 ------------------------------------ 2 files changed, 37 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index b67f34b..3f56fdf 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -8,7 +8,6 @@ from apiflask import APIFlask -from app.celery_worker import make_celery, celery from app.ro_crates.routes import v1_post_bp, v1_get_bp from app.utils.config import DevelopmentConfig, ProductionConfig, make_celery diff --git a/app/celery_worker.py b/app/celery_worker.py index d3be21a..778407c 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -5,42 +5,6 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester from celery import Celery -from flask import Flask - - -def make_celery(app: Flask = None) -> Celery: - """ - Create and configure a Celery instance using Flask configuration. - - :param app: The Flask application. Configuration will be used to initialise Celery. - :return: Celery: A configured Celery instance. - :raises ValueError: If the Flask configuration values are not provided. - """ - if not app: - raise ValueError( - "A Flask application instance must be provided to configure Celery." - ) - - if "CELERY_RESULT_BACKEND" not in app.config: - raise ValueError( - "Missing configuration: 'CELERY_RESULT_BACKEND' is not defined in the Flask app config." - ) - - if "CELERY_BROKER_URL" not in app.config: - raise ValueError( - "Missing configuration: 'CELERY_BROKER_URL' is not defined in the Flask app config." - ) - - celery_instance = Celery( - app.import_name, - backend=app.config["CELERY_RESULT_BACKEND"], - broker=app.config["CELERY_BROKER_URL"], - ) - - if app: - celery_instance.conf.update(app.config) - - return celery_instance celery = Celery() From 6dc565eecfbbbace97e81e31807f12ac3cf9cad5 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:37:21 +0100 Subject: [PATCH 031/127] dependabot GH action --- .github/workflows/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/workflows/dependabot.yml diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml new file mode 100644 index 0000000..c75da03 --- /dev/null +++ b/.github/workflows/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" \ No newline at end of file From 552143ef17616fa6a774049a450b6aaa3defd8fa Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:53:07 +0100 Subject: [PATCH 032/127] tidy yaml file --- .github/workflows/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml index c75da03..6a7695c 100644 --- a/.github/workflows/dependabot.yml +++ b/.github/workflows/dependabot.yml @@ -3,4 +3,4 @@ updates: - package-ecosystem: "pip" directory: "/" schedule: - interval: "weekly" \ No newline at end of file + interval: "weekly" From 28ddb327cf383fd64c576a400f8307234557d883 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:57:24 +0100 Subject: [PATCH 033/127] move dependabot config to correct location --- .github/{workflows => }/dependabot.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => }/dependabot.yml (100%) diff --git a/.github/workflows/dependabot.yml b/.github/dependabot.yml similarity index 100% rename from .github/workflows/dependabot.yml rename to .github/dependabot.yml From 1fc7c69205531d3a8efae0e00d0f0d3af90ef5ab Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 20 Jul 2025 21:25:47 +0100 Subject: [PATCH 034/127] create InvalidAPIUsage class --- app/__init__.py | 7 ++++++- app/utils/config.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 3f56fdf..95a3d98 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -9,7 +9,8 @@ from apiflask import APIFlask from app.ro_crates.routes import v1_post_bp, v1_get_bp -from app.utils.config import DevelopmentConfig, ProductionConfig, make_celery +from app.utils.config import DevelopmentConfig, ProductionConfig, InvalidAPIUsage, make_celery +from flask import jsonify def create_app() -> APIFlask: @@ -22,6 +23,10 @@ def create_app() -> APIFlask: app.register_blueprint(v1_post_bp, url_prefix="/v1/ro_crates") app.register_blueprint(v1_get_bp, url_prefix="/v1/ro_crates") + @app.errorhandler(InvalidAPIUsage) + def invalid_api_usage(e): + return jsonify(e.to_dict()), e.status_code + # Load configuration: if os.getenv("FLASK_ENV") == "production": app.config.from_object(ProductionConfig) diff --git a/app/utils/config.py b/app/utils/config.py index 2258931..a57b63f 100644 --- a/app/utils/config.py +++ b/app/utils/config.py @@ -40,6 +40,22 @@ class ProductionConfig(Config): ENV = "production" +class InvalidAPIUsage(Exception): + status_code = 400 + + def __init__(self, message, status_code=None, payload=None): + super().__init__() + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv + + def make_celery(app: Flask = None) -> Celery: """ Initialises and configures a Celery instance with the Flask application. From 767b5e67932615f708630896374df1a3d43018cf Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 20 Jul 2025 22:22:20 +0100 Subject: [PATCH 035/127] InvalidAPIUsage class and cleaner API error messages --- app/ro_crates/routes/post_routes.py | 47 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 9f9f909..50803fc 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -12,6 +12,8 @@ queue_ro_crate_validation_task, queue_ro_crate_metadata_validation_task ) +from app.utils.config import InvalidAPIUsage + post_routes_bp = APIBlueprint("post_routes", __name__) @@ -45,18 +47,19 @@ def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id` or `webhook_url`) are missing. """ - try: + if "crate_id" not in json_data or json_data["crate_id"] is None: + raise InvalidAPIUsage("Missing required parameter: 'crate_id'") + else: crate_id = json_data["crate_id"] - except: - raise KeyError("Missing required parameter: 'crate_id'") - try: + + if "webhook_url" not in json_data or json_data["webhook_url"] is None: + raise InvalidAPIUsage("Missing required parameter: 'webhook_url'") + else: webhook_url = json_data["webhook_url"] - except: - raise KeyError("Missing required parameter: 'webhook_url'") - try: + if "profile_name" in json_data: profile_name = json_data["profile_name"] - except: + else: profile_name = None return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url) @@ -79,14 +82,14 @@ def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id`) are missing. """ - try: - crate_id = json_data['crate_id'] - except: - raise KeyError("Missing required parameter: 'id'") + if "crate_id" not in json_data or json_data["crate_id"] is None: + raise InvalidAPIUsage("Missing required parameter: 'crate_id'") + else: + crate_id = json_data["crate_id"] - try: - profile_name = json_data['profile_name'] - except: + if "profile_name" in json_data: + profile_name = json_data["profile_name"] + else: profile_name = None return queue_ro_crate_validation_task(crate_id, profile_name) @@ -109,14 +112,14 @@ def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_json`) are missing. """ - try: - crate_json = json_data['crate_json'] - except: - raise KeyError("Missing required parameter: 'crate_json'") + if "crate_json" not in json_data or json_data["crate_json"] is None: + raise InvalidAPIUsage("Missing required parameter: 'crate_json'") + else: + crate_json = json_data["crate_json"] - try: - profile_name = json_data['profile_name'] - except: + if "profile_name" in json_data: + profile_name = json_data["profile_name"] + else: profile_name = None return queue_ro_crate_metadata_validation_task(crate_json, profile_name) From ed396044f8c1292b7b0de166165f06c853f258e3 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 20 Jul 2025 22:23:00 +0100 Subject: [PATCH 036/127] add post_routes tests, including metadata tests --- tests/test_post_routes.py | 128 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/test_post_routes.py diff --git a/tests/test_post_routes.py b/tests/test_post_routes.py new file mode 100644 index 0000000..366e196 --- /dev/null +++ b/tests/test_post_routes.py @@ -0,0 +1,128 @@ +from flask.testing import FlaskClient +import pytest +from unittest.mock import patch +from app import create_app +from flask import jsonify + + +@pytest.fixture +def client(): + app = create_app() + return app.test_client() + + +def test_validate_by_id_success(client): + payload = { + "crate_id": "crate-123", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: + mock_queue.return_value = ({"message": "Validation in progress"}, 202) + + response = client.post("/v1/ro_crates/validate_by_id", json=payload) + + assert response.status_code == 202 + assert response.json == {"message": "Validation in progress"} + mock_queue.assert_called_once_with("crate-123", "default", "https://webhook.example.com") + + +def test_validate_by_id_missing_crate_id(client): + payload = { + "webhook_url": "https://webhook.example.com" + } + + response = client.post("/v1/ro_crates/validate_by_id", json=payload) + + assert response.status_code == 400 + assert b"Missing required parameter: 'crate_id'" in response.data + + +def test_validate_by_id_missing_webhook_url(client): + payload = { + "crate_id": "crate-123" + } + + response = client.post("/v1/ro_crates/validate_by_id", json=payload) + + assert response.status_code == 400 + assert b"Missing required parameter: 'webhook_url'" in response.data + + +def test_validate_by_id_missing_profile_name(client): + payload = { + "crate_id": "crate-123", + "webhook_url": "https://webhook.example.com" + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: + mock_queue.return_value = ({"message": "Validation in progress"}, 202) + + response = client.post("/v1/ro_crates/validate_by_id", json=payload) + + assert response.status_code == 202 + assert response.json == {"message": "Validation in progress"} + mock_queue.assert_called_once_with("crate-123", None, "https://webhook.example.com") + + +# Test API: /v1/ro_crates/validate_metadata + +def test_validate_metadata_with_all_fields(client: FlaskClient): + """ + If both profile name and crate json are provided then processing + of the RO-Crate metadata should continue, and return: + - 200 status code + - JSON data object which includes "status" which is "success" + """ + test_data = { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "profile_name": "default" + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: + mock_queue.return_value = ({"status": "success"}, 200) + + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) + print(response.json) + mock_queue.assert_called_once_with( + test_data["crate_json"], + test_data["profile_name"] + ) + assert response.status_code == 200 + assert response.json == {"status": "success"} + + +def test_validate_metadata_without_profile_name(client: FlaskClient): + """ + If the profile name is not specified then processing + of the RO-Crate metadata should continue, and return: + - 200 status code + - JSON data object which includes "status" which is "success" + """ + test_data = { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}' + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: + mock_queue.return_value = ({"status": "success"}, 200) + + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) + mock_queue.assert_called_once_with(test_data["crate_json"], None) + assert response.status_code == 200 + assert response.json == {"status": "success"} + + +def test_validate_metadata_missing_crate_json(client: FlaskClient): + """ + If the RO-Crate is missing APIFlask should return: + - 422 status code + - Error message which includes 'Missing data for required field' + """ + test_data = { + "profile_name": "default" + } + + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) + assert response.status_code == 422 + assert "Missing data for required field" in response.get_data(as_text=True) From c828f251f926f69e5ec8dcb27d29457efbd90fc8 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 20 Jul 2025 22:23:36 +0100 Subject: [PATCH 037/127] test_metadata.py moved to test_post_routes.py --- tests/test_metadata.py | 70 ------------------------------------------ 1 file changed, 70 deletions(-) delete mode 100644 tests/test_metadata.py diff --git a/tests/test_metadata.py b/tests/test_metadata.py deleted file mode 100644 index 83efe04..0000000 --- a/tests/test_metadata.py +++ /dev/null @@ -1,70 +0,0 @@ -from flask.testing import FlaskClient -import pytest -from unittest.mock import patch -from app import create_app - - -@pytest.fixture -def client(): - app = create_app() - return app.test_client() - - -def test_validate_metadata_with_all_fields(client: FlaskClient): - """ - If both profile name and crate json are provided then processing - of the RO-Crate metadata should continue, and return: - - 200 status code - - JSON data object which includes "status" which is "success" - """ - test_data = { - "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', - "profile_name": "default" - } - - with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: - mock_queue.return_value = ({"status": "success"}, 200) - - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - print(response.json) - mock_queue.assert_called_once_with( - test_data["crate_json"], - test_data["profile_name"] - ) - assert response.status_code == 200 - assert response.json == {"status": "success"} - - -def test_validate_metadata_without_profile_name(client: FlaskClient): - """ - If the profile name is not specified then processing - of the RO-Crate metadata should continue, and return: - - 200 status code - - JSON data object which includes "status" which is "success" - """ - test_data = { - "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}' - } - - with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: - mock_queue.return_value = ({"status": "success"}, 200) - - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - mock_queue.assert_called_once_with(test_data["crate_json"], None) - assert response.status_code == 200 - assert response.json == {"status": "success"} - - -def test_validate_metadata_missing_crate_json(client: FlaskClient): - """ - If the RO-Crate is missing APIFlask should return: - - 422 status code - - Error message which includes 'Missing data for required field' - """ - test_data = { - "profile_name": "default" - } - - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - assert response.status_code == 422 - assert "Missing data for required field" in response.get_data(as_text=True) From 04b302b5ca1b5e93bef4ee0a0dd5b12e7b4f9e24 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:13:41 +0100 Subject: [PATCH 038/127] specify str type for metadata service function and add checks for this --- app/services/validation_service.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 357b10e..e123995 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -5,6 +5,7 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester import logging +import json from flask import jsonify, Response @@ -44,7 +45,7 @@ def queue_ro_crate_validation_task( def queue_ro_crate_metadata_validation_task( - crate_json, profile_name=None, webhook_url=None + crate_json: str, profile_name=None, webhook_url=None ) -> tuple[Response, int]: """ Queues an RO-Crate for validation with Celery. @@ -61,6 +62,14 @@ def queue_ro_crate_metadata_validation_task( if not crate_json: return jsonify({"error": "Missing required parameter: crate_json"}), 400 + try: + json_dict = json.loads(crate_json) + except json.decoder.JSONDecodeError as err: + return jsonify({"error": f"Required parameter crate_json is not valid JSON: {err}"}), 400 + else: + if len(json_dict) == 0: + return jsonify({"error": "Required parameter crate_json is empty"}), 400 + try: result = process_validation_task_by_metadata.delay( crate_json, From 8862cfac0d23364524c354684d7e60fe17c7cb43 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:14:31 +0100 Subject: [PATCH 039/127] adapt and add tests for crate metadata json strings --- tests/test_services.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_services.py b/tests/test_services.py index 9aa24e5..3784ace 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -49,7 +49,7 @@ def test_queue_metadata_with_webhook(flask_app): mock_result = MagicMock() mock_delay.return_value = mock_result - crate_json = {"@context": "https://w3id.org/ro/crate/1.1/context"} + crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context"}' response, status = queue_ro_crate_metadata_validation_task(crate_json, "profile", "http://webhook") mock_delay.assert_called_once_with(crate_json, "profile", "http://webhook") @@ -63,7 +63,7 @@ def test_queue_metadata_without_webhook(flask_app): mock_result.get.return_value = {"status": "ok"} mock_delay.return_value = mock_result - crate_json = {"@context": "https://w3id.org/ro/crate/1.1/context"} + crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context"}' response, status = queue_ro_crate_metadata_validation_task(crate_json, "profile", None) mock_delay.assert_called_once_with(crate_json, "profile", None) @@ -78,6 +78,24 @@ def test_queue_metadata_missing_json(flask_app): assert response.json == {"error": "Missing required parameter: crate_json"} +def test_queue_metadata_invalid_json(flask_app): + + crate_json = '{' + response, status = queue_ro_crate_metadata_validation_task(crate_json) + + assert status == 400 + assert 'not valid JSON' in response.json['error'] + + +def test_queue_metadata_empty_json(flask_app): + + crate_json = '{}' + response, status = queue_ro_crate_metadata_validation_task(crate_json) + + assert status == 400 + assert response.json == {"error": "Required parameter crate_json is empty"} + + def test_queue_metadata_exception(flask_app): with patch("app.services.validation_service.process_validation_task_by_metadata.delay", side_effect=Exception("Celery error")): From 89d17472c5fa08c01b7e21d088feea78605a7420 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:19:17 +0100 Subject: [PATCH 040/127] 422 error for malformed/missing crate string --- app/services/validation_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index e123995..bb3bc4a 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -60,15 +60,15 @@ def queue_ro_crate_metadata_validation_task( logging.info(f"Processing: {crate_json}, {profile_name}, {webhook_url}") if not crate_json: - return jsonify({"error": "Missing required parameter: crate_json"}), 400 + return jsonify({"error": "Missing required parameter: crate_json"}), 422 try: json_dict = json.loads(crate_json) except json.decoder.JSONDecodeError as err: - return jsonify({"error": f"Required parameter crate_json is not valid JSON: {err}"}), 400 + return jsonify({"error": f"Required parameter crate_json is not valid JSON: {err}"}), 422 else: if len(json_dict) == 0: - return jsonify({"error": "Required parameter crate_json is empty"}), 400 + return jsonify({"error": "Required parameter crate_json is empty"}), 422 try: result = process_validation_task_by_metadata.delay( From aaeb5e22abe898daf4c3718121bd42ac3e5b95e1 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:19:54 +0100 Subject: [PATCH 041/127] 422 errors and make crate_json string for celery test --- tests/test_services.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_services.py b/tests/test_services.py index 3784ace..978f9c8 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -74,7 +74,7 @@ def test_queue_metadata_without_webhook(flask_app): def test_queue_metadata_missing_json(flask_app): response, status = queue_ro_crate_metadata_validation_task(None) - assert status == 400 + assert status == 422 assert response.json == {"error": "Missing required parameter: crate_json"} @@ -83,7 +83,7 @@ def test_queue_metadata_invalid_json(flask_app): crate_json = '{' response, status = queue_ro_crate_metadata_validation_task(crate_json) - assert status == 400 + assert status == 422 assert 'not valid JSON' in response.json['error'] @@ -92,14 +92,14 @@ def test_queue_metadata_empty_json(flask_app): crate_json = '{}' response, status = queue_ro_crate_metadata_validation_task(crate_json) - assert status == 400 + assert status == 422 assert response.json == {"error": "Required parameter crate_json is empty"} def test_queue_metadata_exception(flask_app): with patch("app.services.validation_service.process_validation_task_by_metadata.delay", side_effect=Exception("Celery error")): - crate_json = {"@context": "https://w3id.org/ro/crate/1.1/context"} + crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context"}' response, status = queue_ro_crate_metadata_validation_task(crate_json) assert status == 500 From 45beaa6f795568ac17b05aa359407de3ed2e7df7 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:25:54 +0100 Subject: [PATCH 042/127] cleaned out unused test check in validate metadata api --- app/ro_crates/routes/post_routes.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 50803fc..32e6f78 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -112,10 +112,7 @@ def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_json`) are missing. """ - if "crate_json" not in json_data or json_data["crate_json"] is None: - raise InvalidAPIUsage("Missing required parameter: 'crate_json'") - else: - crate_json = json_data["crate_json"] + crate_json = json_data["crate_json"] if "profile_name" in json_data: profile_name = json_data["profile_name"] From 963e22e5f35d0426034b163d0ad84fd22fbe1632 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:26:42 +0100 Subject: [PATCH 043/127] api tests for metadata errors --- tests/test_post_routes.py | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_post_routes.py b/tests/test_post_routes.py index 366e196..04713ba 100644 --- a/tests/test_post_routes.py +++ b/tests/test_post_routes.py @@ -126,3 +126,51 @@ def test_validate_metadata_missing_crate_json(client: FlaskClient): response = client.post("/v1/ro_crates/validate_metadata", json=test_data) assert response.status_code == 422 assert "Missing data for required field" in response.get_data(as_text=True) + + +def test_validate_metadata_emptystring_crate_json(client: FlaskClient): + """ + If the RO-Crate is missing APIFlask should return: + - 422 status code + - Error message which includes 'Missing required parameter' + """ + test_data = { + "crate_json": "", + "profile_name": "default" + } + + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) + assert response.status_code == 422 + assert "Missing required parameter" in response.get_data(as_text=True) + + +def test_validate_metadata_malformed_crate_json(client: FlaskClient): + """ + If the RO-Crate is missing APIFlask should return: + - 422 status code + - Error message which includes 'not valid JSON' + """ + test_data = { + "crate_json": "{", + "profile_name": "default" + } + + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) + assert response.status_code == 422 + assert "not valid JSON" in response.get_data(as_text=True) + + +def test_validate_metadata_emptydict_crate_json(client: FlaskClient): + """ + If the RO-Crate is missing APIFlask should return: + - 422 status code + - Error message which includes 'Required parameter crate_json is empty' + """ + test_data = { + "crate_json": "{}", + "profile_name": "default" + } + + response = client.post("/v1/ro_crates/validate_metadata", json=test_data) + assert response.status_code == 422 + assert "Required parameter crate_json is empty" in response.get_data(as_text=True) From 9c10ef05706fbd352f0f50f616ab1ce304c1746a Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:27:55 +0100 Subject: [PATCH 044/127] remove jsonify from test routes --- tests/test_post_routes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_post_routes.py b/tests/test_post_routes.py index 04713ba..1da6661 100644 --- a/tests/test_post_routes.py +++ b/tests/test_post_routes.py @@ -2,7 +2,6 @@ import pytest from unittest.mock import patch from app import create_app -from flask import jsonify @pytest.fixture From cb4bc520774ef4716d150ede594bc50e507c1f3f Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:36:45 +0100 Subject: [PATCH 045/127] clean validate_by_id function to use data class for validating inputs --- app/ro_crates/routes/post_routes.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 32e6f78..a8a4338 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -30,7 +30,7 @@ class ValidateJSON(Schema): @post_routes_bp.post("/validate_by_id") -@post_routes_bp.input(ValidateData(partial=True), location='json') +@post_routes_bp.input(ValidateData(partial=False), location='json') def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate using its ID from MinIO. @@ -47,10 +47,7 @@ def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id` or `webhook_url`) are missing. """ - if "crate_id" not in json_data or json_data["crate_id"] is None: - raise InvalidAPIUsage("Missing required parameter: 'crate_id'") - else: - crate_id = json_data["crate_id"] + crate_id = json_data["crate_id"] if "webhook_url" not in json_data or json_data["webhook_url"] is None: raise InvalidAPIUsage("Missing required parameter: 'webhook_url'") From e82dc46f67f3c5b2fd183ca071b5fafe6ea93445 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:37:10 +0100 Subject: [PATCH 046/127] tests updated for cleaner validate_by_id interface --- tests/test_post_routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_post_routes.py b/tests/test_post_routes.py index 1da6661..c742b3b 100644 --- a/tests/test_post_routes.py +++ b/tests/test_post_routes.py @@ -34,8 +34,8 @@ def test_validate_by_id_missing_crate_id(client): response = client.post("/v1/ro_crates/validate_by_id", json=payload) - assert response.status_code == 400 - assert b"Missing required parameter: 'crate_id'" in response.data + assert response.status_code == 422 + assert "Missing data for required field" in response.get_data(as_text=True) def test_validate_by_id_missing_webhook_url(client): From 456b538cb438757f5dbdb6006ff63a01d7df06c1 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:15:15 +0100 Subject: [PATCH 047/127] post route now /v1/ro_crates/{crate_id}/validation --- app/ro_crates/routes/post_routes.py | 50 +++++------------------------ 1 file changed, 8 insertions(+), 42 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index a8a4338..1e15c3d 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -18,8 +18,7 @@ post_routes_bp = APIBlueprint("post_routes", __name__) -class ValidateData(Schema): - crate_id = String(required=True) +class ValidateCrate(Schema): profile_name = String(required=False) webhook_url = String(required=False) @@ -29,16 +28,15 @@ class ValidateJSON(Schema): profile_name = String(required=False) -@post_routes_bp.post("/validate_by_id") -@post_routes_bp.input(ValidateData(partial=False), location='json') -def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: +@post_routes_bp.post("/validation") +@post_routes_bp.input(ValidateCrate(partial=False), location='json') +def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate using its ID from MinIO. Parameters: - - **crate_id**: The ID of the RO-Crate to validate. _Required_. - **profile_name**: The profile name for validation. _Optional_. - - **webhook_url**: The webhook URL where validation results will be sent. _Required_. + - **webhook_url**: The webhook URL where validation results will be sent. _Optional_. Returns: - A tuple containing the validation task's response and an HTTP status code. @@ -47,49 +45,17 @@ def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id` or `webhook_url`) are missing. """ - crate_id = json_data["crate_id"] - - if "webhook_url" not in json_data or json_data["webhook_url"] is None: - raise InvalidAPIUsage("Missing required parameter: 'webhook_url'") - else: + if "webhook_url" in json_data: webhook_url = json_data["webhook_url"] - - if "profile_name" in json_data: - profile_name = json_data["profile_name"] - else: - profile_name = None - - return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url) - - -@post_routes_bp.post("/validate_by_id_no_webhook") -@post_routes_bp.input(ValidateData(partial=True), location='json') # -> json_data -def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: - """ - Endpoint to validate an RO-Crate using its ID from MinIO. - - Parameters: - - **crate_id**: The ID of the RO-Crate to validate. _Required_. - - **profile_name**: The profile name for validation. _Optional_. - - Returns: - - A tuple containing the validation task's response and an HTTP status code. - - Raises: - - KeyError: If required parameters (`crate_id`) are missing. - """ - - if "crate_id" not in json_data or json_data["crate_id"] is None: - raise InvalidAPIUsage("Missing required parameter: 'crate_id'") else: - crate_id = json_data["crate_id"] + webhook_url = None if "profile_name" in json_data: profile_name = json_data["profile_name"] else: profile_name = None - return queue_ro_crate_validation_task(crate_id, profile_name) + return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url) @post_routes_bp.post("/validate_metadata") From 0101e71445c251610ff79dfbb99d51ae0135b477 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:15:54 +0100 Subject: [PATCH 048/127] updated tests to reflect post route change --- tests/test_post_routes.py | 45 ++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tests/test_post_routes.py b/tests/test_post_routes.py index c742b3b..c593a70 100644 --- a/tests/test_post_routes.py +++ b/tests/test_post_routes.py @@ -10,9 +10,11 @@ def client(): return app.test_client() +# Test API: /v1/ro_crates/{crate_id}/validation + def test_validate_by_id_success(client): + crate_id = "crate-123" payload = { - "crate_id": "crate-123", "webhook_url": "https://webhook.example.com", "profile_name": "default" } @@ -20,7 +22,7 @@ def test_validate_by_id_success(client): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: mock_queue.return_value = ({"message": "Validation in progress"}, 202) - response = client.post("/v1/ro_crates/validate_by_id", json=payload) + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) assert response.status_code == 202 assert response.json == {"message": "Validation in progress"} @@ -32,39 +34,58 @@ def test_validate_by_id_missing_crate_id(client): "webhook_url": "https://webhook.example.com" } - response = client.post("/v1/ro_crates/validate_by_id", json=payload) + response = client.post("/v1/ro_crates//validation", json=payload) - assert response.status_code == 422 - assert "Missing data for required field" in response.get_data(as_text=True) + assert response.status_code == 404 -def test_validate_by_id_missing_webhook_url(client): +def test_validate_by_id_missing_profile_name_and_webhook_url(client): + crate_id = "crate-123" payload = { - "crate_id": "crate-123" } - response = client.post("/v1/ro_crates/validate_by_id", json=payload) + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: + mock_queue.return_value = ({"message": "Validation in progress"}, 202) - assert response.status_code == 400 - assert b"Missing required parameter: 'webhook_url'" in response.data + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 202 + assert response.json == {"message": "Validation in progress"} + mock_queue.assert_called_once_with("crate-123", None, None) def test_validate_by_id_missing_profile_name(client): + crate_id = "crate-123" payload = { - "crate_id": "crate-123", "webhook_url": "https://webhook.example.com" } with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: mock_queue.return_value = ({"message": "Validation in progress"}, 202) - response = client.post("/v1/ro_crates/validate_by_id", json=payload) + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) assert response.status_code == 202 assert response.json == {"message": "Validation in progress"} mock_queue.assert_called_once_with("crate-123", None, "https://webhook.example.com") +def test_validate_by_id_missing_webhook_url(client): + crate_id = "crate-123" + payload = { + "profile_name": "default" + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: + mock_queue.return_value = ({"message": "Validation in progress"}, 202) + + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 202 + assert response.json == {"message": "Validation in progress"} + mock_queue.assert_called_once_with("crate-123", "default", None) + + # Test API: /v1/ro_crates/validate_metadata def test_validate_metadata_with_all_fields(client: FlaskClient): From e3ee7daf97fb8416f918fe81d4a2cf8bfdc1bd56 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:31:05 +0100 Subject: [PATCH 049/127] get route changed to /v1/ro_crates/{crate_id}/validation --- app/ro_crates/routes/get_routes.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/app/ro_crates/routes/get_routes.py b/app/ro_crates/routes/get_routes.py index 39d6064..30d6730 100644 --- a/app/ro_crates/routes/get_routes.py +++ b/app/ro_crates/routes/get_routes.py @@ -12,19 +12,12 @@ get_routes_bp = APIBlueprint("get_routes", __name__) -class validate_data(Schema): - crate_id = String(required=True) - -@get_routes_bp.get("/get_validation_by_id") -@get_routes_bp.input(validate_data(partial=True), location='json') -def get_ro_crate_validation_by_id(json_data) -> tuple[Response, int]: +@get_routes_bp.get("/validation") +def get_ro_crate_validation_by_id(crate_id) -> tuple[Response, int]: """ Endpoint to obtain an RO-Crate validation result using its ID from MinIO. - Parameters: - - **crate_id**: The ID of the RO-Crate to validate. _Required_. - Returns: - A tuple containing the validation result and an HTTP status code. @@ -32,9 +25,4 @@ def get_ro_crate_validation_by_id(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id`) are missing. """ - try: - crate_id = json_data["crate_id"] - except: - raise KeyError("Missing required parameter: 'crate_id'") - return get_ro_crate_validation_task(crate_id) From 56120a20b46dfb144e0d2fde11839a0015cae133 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:38:27 +0100 Subject: [PATCH 050/127] clean get route library imports --- app/ro_crates/routes/get_routes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/ro_crates/routes/get_routes.py b/app/ro_crates/routes/get_routes.py index 30d6730..3b43144 100644 --- a/app/ro_crates/routes/get_routes.py +++ b/app/ro_crates/routes/get_routes.py @@ -4,9 +4,8 @@ # License: MIT # Copyright (c) 2025 eScience Lab, The University of Manchester -from apiflask import APIBlueprint, Schema -from apiflask.fields import Integer, String -from flask import request, Response +from apiflask import APIBlueprint +from flask import Response from app.services.validation_service import get_ro_crate_validation_task From f8b42e67800c4d0931ae80d9c1b526b1f16c98ce Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:24:41 +0100 Subject: [PATCH 051/127] validate post and get examples --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 36a89a0..c4c7a2f 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,8 @@ For testing locally developed containers use the alternate Docker Compose file: ## Example Usage -``` +Validation of RO-Crate with the ID of `1`. No webhook is used here: +```bash curl -X 'POST' \ 'http://localhost:5001/ro_crates/validate_by_id_no_webhook' \ -H 'accept: application/json' \ @@ -79,3 +80,14 @@ curl -X 'POST' \ "crate_id": "1" }' ``` + +Retrieval of validation result for RO-Crate `1`: +```bash +curl -X 'GET' \ + 'http://localhost:5001/ro_crates/get_validation_by_id' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "crate_id": "1" +}' +``` From dffc061b3781b71b0678159fd8b18834300e7406 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:14:03 +0100 Subject: [PATCH 052/127] remove try to allow pass through of InvalidAPIUsage in get_ro_crate_validation_task --- app/services/validation_service.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index bb3bc4a..726988e 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -100,7 +100,4 @@ def get_ro_crate_validation_task( if not crate_id: return jsonify({"error": "Missing required parameter: crate_id"}), 400 - try: - return return_ro_crate_validation(crate_id), 200 - except Exception as e: - return jsonify({"service error": str(e)}), 500 + return return_ro_crate_validation(crate_id), 200 From b0afaa8b4b57a87a60afff8eea22d0bd4c7cdcbf Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:14:59 +0100 Subject: [PATCH 053/127] remove try to allow pass through of InvalidAPIUsage in return_ro_crate_validation --- app/tasks/validation_tasks.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index f8295e7..4d5d9dc 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -190,11 +190,6 @@ def return_ro_crate_validation( :raises Exception: If an error occurs in the retrieving the validation result """ - try: - logging.info(f"Fetching validation result for RO-Crate {crate_id}") - - return get_validation_status_from_minio(crate_id) + logging.info(f"Fetching validation result for RO-Crate {crate_id}") - except Exception as e: - logging.error(f"Unexpected error when retrieving validation: {e}") - return str(e) + return get_validation_status_from_minio(crate_id) From d774ee948abc442397edfb2f5e491cdcbe86a4be Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:15:38 +0100 Subject: [PATCH 054/127] file_utils whitespace cleanup --- app/utils/file_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/utils/file_utils.py b/app/utils/file_utils.py index 0b20eac..15c16e4 100644 --- a/app/utils/file_utils.py +++ b/app/utils/file_utils.py @@ -51,4 +51,3 @@ def build_metadata_only_rocrate(crate_json: str) -> str: except Exception as e: logging.error(f"Unexpected error creating RO-Crate metadata: {e}") raise - From 6aaa244421697a4a5212b0371566835d3e4b9695 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:17:53 +0100 Subject: [PATCH 055/127] Major updates to minio_utils.py Switch to InvalidAPIUsage function for capturing Exceptions. The catches for these have been combined where possible. Functions have been added to cover: get_minio_object_list find_rocrate_object_on_minio find_validation_object_on_minio download_file_from_minio The function fetch_ro_crate_from_minio can now cope with both zip archives and ro-crate directories. --- app/utils/minio_utils.py | 249 ++++++++++++++++++++++++++++++--------- 1 file changed, 196 insertions(+), 53 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index a4ac42a..5620215 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -12,6 +12,7 @@ from dotenv import load_dotenv from io import BytesIO from minio import Minio, S3Error +from app.utils.config import InvalidAPIUsage logger = logging.getLogger(__name__) @@ -23,43 +24,41 @@ def fetch_ro_crate_from_minio(crate_id: str) -> str: :param crate_id: The ID of the RO-Crate to fetch from MinIO. :return: The local file path where the RO-Crate is saved. - :raises S3Error: If an error occurs during the MinIO operation. - :raises ValueError: If the required environment variables are not set. - :raises Exception: If an unexpected error occurs during the operation. """ - load_dotenv() + minio_client, bucket_name = get_minio_client_and_bucket() - try: - minio_client, bucket_name = get_minio_client_and_bucket() + rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, bucket_name) - object_name = f"{crate_id}.zip" + rocrate_path = rocrate_object.object_name + rocrate_name = rocrate_path.split('/')[-1] - # Prepare temporary file path to store RO Crate for validation: - temp_dir = tempfile.mkdtemp() - file_path = os.path.join(temp_dir, object_name) + temp_dir = tempfile.mkdtemp() + root_path = os.path.join(temp_dir, rocrate_name) - logging.info( - f"Fetching RO-Crate {object_name} from MinIO bucket {bucket_name}. File path: {file_path}" - ) - minio_client.fget_object(bucket_name, object_name, file_path) - logging.info( - f"RO-Crate {object_name} fetched successfully and saved to {file_path}." - ) + logging.info( + f"Fetching RO-Crate {rocrate_name} from MinIO bucket {bucket_name}. File path {root_path}" + ) - return file_path + if rocrate_object.is_dir: + os.makedirs(os.path.dirname(root_path), exist_ok=True) - except S3Error as s3_error: - logging.error(f"MinIO S3 Error: {s3_error}") - raise + objects_list = get_minio_object_list(rocrate_path, minio_client, bucket_name, recursive=True) + for obj in objects_list: + relative_path = obj.object_name[len(rocrate_path):].lstrip("/") + local_file_path = os.path.join(root_path, relative_path) + os.makedirs(os.path.dirname(local_file_path), exist_ok=True) + download_file_from_minio(minio_client, bucket_name, obj.object_name, local_file_path) - except ValueError as value_error: - logging.error(f"Configuration Error: {value_error}") - raise + else: + file_path = root_path + download_file_from_minio(minio_client, bucket_name, rocrate_path, file_path) - except Exception as e: - logging.error(f"Unexpected error fetching RO-Crate from MinIO: {e}") - raise + logging.info( + f"RO-Crate {rocrate_name} fetched successfully and saved to {root_path}." + ) + + return root_path def update_validation_status_in_minio(crate_id: str, validation_status: str) -> None: @@ -73,16 +72,14 @@ def update_validation_status_in_minio(crate_id: str, validation_status: str) -> :raises Exception: If an unexpected error occurs """ - load_dotenv() + object_name = f"{crate_id}_validation/validation_status.txt" + + # convert pretty string to dictionary, then back to plain utf-8 encoded string + validation_string = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") try: - minio_client, bucket_name = get_minio_client_and_bucket() - # The object in MinIO is /validation_status.txt - object_name = f"{crate_id}/validation_status.txt" - - # convert pretty string to dictionary, then back to plain utf-8 encoded string - validation_string = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") + minio_client, bucket_name = get_minio_client_and_bucket() minio_client.put_object( bucket_name, @@ -92,43 +89,41 @@ def update_validation_status_in_minio(crate_id: str, validation_status: str) -> content_type="application/json", ) - logging.info( - f"Validation status file uploaded to {bucket_name}/{object_name} successfully." - ) - except S3Error as s3_error: logging.error(f"MinIO S3 Error: {s3_error}") - raise + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) except ValueError as value_error: logging.error(f"Configuration Error: {value_error}") - raise + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) except Exception as e: logging.error(f"Unexpected error updating validation status in MinIO: {e}") - raise + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) + logging.info( + f"Validation status file uploaded to {bucket_name}/{object_name} successfully." + ) -def get_validation_status_from_minio(crate_id: str) -> dict | str: + +def get_validation_status_from_minio(crate_id: str) -> dict: """ Checks for the existence of a validation report for the given RO-Crate in the MinIO bucket. Returns validation message if it exists, or notification that it is missing if not. :param crate_id: The ID of the RO-Crate in MinIO :return validation_status: Either the validation status, or note that this does not exist - :raises S3Error: If an error occurs during the MinIO operation - :raises ValueError: If the required environment variables are not set - :raises Exception: If an unexpected error occurs """ - try: - minio_client, bucket_name = get_minio_client_and_bucket() + # The object in MinIO is _validation/validation_status.txt + object_name = f"{crate_id}_validation/validation_status.txt" + + logging.info(f"Getting object {object_name}") - # The object in MinIO is /validation_status.txt - object_name = f"{crate_id}/validation_status.txt" + try: - logging.info(f"Getting object {object_name}") + minio_client, bucket_name = get_minio_client_and_bucket() response = minio_client.get_object( bucket_name, @@ -141,20 +136,168 @@ def get_validation_status_from_minio(crate_id: str) -> dict | str: except S3Error as s3_error: logging.error(f"MinIO S3 Error: {s3_error}") - raise + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) except ValueError as value_error: logging.error(f"Configuration Error: {value_error}") - raise + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) except Exception as e: logging.error(f"Unexpected error retrieving validation status from MinIO: {e}") - raise + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) else: return validation_message +def download_file_from_minio(minio_client: object, bucket_name: str, object_path: str, file_path: str) -> None: + """ + Downloads a file from MinIO + + :param minio_client: MinIO object + :param bucket_name: name of MinIO bucket, string + :param object_path: path to object on MinIO, string + :param file_path: local path, string + :raises S3Error: If an error occurs during the MinIO operation + :raises ValueError: If the required environment variables are not set + :raises Exception: If an unexpected error occurs + """ + + try: + minio_client.fget_object(bucket_name, object_path, file_path) + + except S3Error as s3_error: + logging.error(f"MinIO S3 Error: {s3_error}") + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) + + except ValueError as value_error: + logging.error(f"Configuration Error: {value_error}") + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) + + except Exception as e: + logging.error(f"Unexpected error retrieving file from MinIO: {e}") + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) + + +def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: str, storage_path: str = None) -> object: + """ + Checks that the requested object exists on the MinIO instance. + + If it does not exist then a False value is returned. + If it does exist then the minio.datatypes.Object is returned. + + Inputs: + :param rocrate_id: string containing the name of ro-crate + :param storage_path: string containing the path within which the ro-crate should be + :param minio_client: minio object + :param bucket_name: string containing bucket on minio + :return return_object: rocrate object we require + :raise Exception: If validation result can't be found, 400 + """ + + logging.info(f"Finding Validation result: {rocrate_id}_validation/validation_status.txt") + + if storage_path: + file_path = f"{storage_path}/{rocrate_id}_validation/validation_status.txt" + else: + file_path = f"{rocrate_id}_validation/validation_status.txt" + + file_list = get_minio_object_list(file_path, minio_client, bucket_name) + + return_object = False + for obj in file_list: + if obj.object_name == file_path: + return_object = obj + break + + if not return_object: + logging.error(f"No validation result yet for RO-Crate: {rocrate_id}") + raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {rocrate_id}", 400) + else: + return return_object + + +def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str, storage_path: str = None) -> object: + """ + Checks that the requested object exists on the MinIO instance. + + If it does not exist then a False value is returned. + If it does exist then the minio.datatypes.Object is returned. + + Inputs: + :param rocrate_id: string containing the name of ro-crate + :param storage_path: string containing the path within which the ro-crate should be + :param minio_client: minio object + :param bucket_name: string containing bucket on minio + :return return_object: rocrate object we require + :raise Exception: If RO-Crate can't be found, 400 + """ + + logging.info(f"Finding RO-Crate: {rocrate_id}") + + if storage_path: + rocrate_path = f"{storage_path}/{rocrate_id}" + else: + rocrate_path = rocrate_id + + rocrate_list = get_minio_object_list(rocrate_path, minio_client, bucket_name) + + return_object = False + for obj in rocrate_list: + # TODO: We should be checking here for the existence of the ro-crate metadata file within this object too + if (obj.object_name == rocrate_path and obj.is_dir) or obj.object_name == f"{rocrate_path}.zip": + return_object = obj + break + + if not return_object: + logging.error(f"No RO-Crate with prefix: {rocrate_path}") + raise InvalidAPIUsage(f"No RO-Crate with prefix: {rocrate_path}", 400) + else: + return return_object + + +def get_minio_object_list(object_path: str, minio_client, bucket_name: str, recursive: bool = False) -> list: + """ + Creates a list of objects which match the object_id and path_prefix + + :param object_path: The object ID, string + :param path_prefix: Path prefix, string, optional + :param minio_client: MinIO client object + :param bucket_name: string + :param recursive: boolean, default = False + :return object_list: List containing objects of type minio.datatypes.Object + :raises S3Error: If an error occurs during the MinIO operation, 500 + :raises ValueError: If the required environment variables are not set, 500 + :raises Exception: If an unexpected error occurs, 500 + """ + + try: + response = minio_client.list_objects( + bucket_name, + object_path, + recursive=recursive + ) + object_list = [obj for obj in response] + + response.close() + response.release_conn() + + except S3Error as s3_error: + logging.error(f"MinIO S3 Error: {s3_error}") + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) + + except ValueError as value_error: + logging.error(f"Configuration Error: {value_error}") + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) + + except Exception as e: + logging.error(f"Unexpected error getting object list from MinIO: {e}") + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) + + else: + return object_list + + def get_minio_client_and_bucket() -> [Minio, str]: """ Initialises the MinIO client and retrieves the bucket name from environment variables. From 8b49c03dbe121c0b18b1caf28cbaea8a94abfc6d Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:23:33 +0100 Subject: [PATCH 056/127] expanded testset for minio functions --- tests/test_minio.py | 520 ++++++++++++++++++++++++++++++++------------ 1 file changed, 383 insertions(+), 137 deletions(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 3c9f836..2958e28 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -1,10 +1,9 @@ -import os import json import pytest from io import BytesIO from minio import Minio from minio.error import S3Error -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from unittest import mock @@ -14,8 +13,14 @@ def mock_minio_response(): response.data.decode.return_value = json.dumps({"status": "valid"}) return response -# Testing function: get_minio_client_and_bucket +class DummyObject: + def __init__(self, name, is_dir=False): + self.object_name = name + self.is_dir = is_dir + + +# Testing function: get_minio_client_and_bucket def test_get_minio_client_success(monkeypatch): # Set required env vars @@ -44,6 +49,257 @@ def test_get_minio_client_missing_bucket_name(monkeypatch): get_minio_client_and_bucket() +# Testing function: get_minio_object_list + +def test_get_minio_object_list_success(): + # Setup mock response + mock_response = MagicMock() + mock_objects = [DummyObject("file1.txt"), DummyObject("file2.txt")] + mock_response.__iter__.return_value = iter(mock_objects) + + # Patch minio_client + mock_minio_client = MagicMock() + mock_minio_client.list_objects.return_value = mock_response + + # Call function + from app.utils.minio_utils import get_minio_object_list + result = get_minio_object_list("path/", mock_minio_client, "my-bucket", recursive=True) + + # Assert + assert result == mock_objects + mock_minio_client.list_objects.assert_called_once_with("my-bucket", "path/", recursive=True) + mock_response.close.assert_called_once() + mock_response.release_conn.assert_called_once() + + +def test_get_minio_object_list_s3error(): + mock_minio_client = MagicMock() + mock_minio_client.list_objects.side_effect = S3Error( + code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None + ) + + from app.utils.minio_utils import get_minio_object_list, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + get_minio_object_list("path/", mock_minio_client, "my-bucket") + + assert exc.value.status_code == 500 + assert "MinIO S3 Error" in str(exc.value.message) + + +def test_get_minio_object_list_value_error(): + mock_minio_client = MagicMock() + mock_minio_client.list_objects.side_effect = ValueError("Missing config") + + from app.utils.minio_utils import get_minio_object_list, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + get_minio_object_list("path/", mock_minio_client, "my-bucket") + + assert exc.value.status_code == 500 + assert "Configuration Error" in str(exc.value.message) + + +def test_get_minio_object_list_unexpected_error(): + mock_minio_client = MagicMock() + mock_minio_client.list_objects.side_effect = RuntimeError("Something went wrong") + + from app.utils.minio_utils import get_minio_object_list, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + get_minio_object_list("path/", mock_minio_client, "my-bucket") + + assert exc.value.status_code == 500 + assert "Unknown Error" in str(exc.value.message) + + +# Testing function: find_rocrate_object_on_minio + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_rocrate_found_as_directory(mock_get_list): + # Simulate a directory object match + obj = DummyObject("my/path/rocrate123", is_dir=True) + mock_get_list.return_value = [obj] + minio_client = MagicMock() + + from app.utils.minio_utils import find_rocrate_object_on_minio + result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", storage_path="my/path") + assert result == obj + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_rocrate_found_as_zip(mock_get_list): + # Simulate a zip object match + obj = DummyObject("rocrate123.zip") + mock_get_list.return_value = [obj] + minio_client = MagicMock() + + from app.utils.minio_utils import find_rocrate_object_on_minio + result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket") + assert result == obj + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_rocrate_not_found_raises(mock_get_list): + # Simulate no matching object + mock_get_list.return_value = [ + DummyObject("something_else"), + DummyObject("another_dir", is_dir=True) + ] + minio_client = MagicMock() + + from app.utils.minio_utils import find_rocrate_object_on_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + find_rocrate_object_on_minio("rocrate123", minio_client, "bucket") + + assert exc.value.status_code == 400 + assert "No RO-Crate with prefix" in str(exc.value.message) + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_storage_path_none(mock_get_list): + # Ensures correct rocrate_path is used when storage_path is None + obj = DummyObject("rocrate456.zip") + mock_get_list.return_value = [obj] + minio_client = MagicMock() + + from app.utils.minio_utils import find_rocrate_object_on_minio + result = find_rocrate_object_on_minio("rocrate456", minio_client, "bucket") + assert result == obj + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_storage_path_provided(mock_get_list): + # Ensures correct rocrate_path is used when storage_path is provided + obj = DummyObject("data/rocrate789", is_dir=True) + mock_get_list.return_value = [obj] + minio_client = MagicMock() + + from app.utils.minio_utils import find_rocrate_object_on_minio + result = find_rocrate_object_on_minio("rocrate789", minio_client, "bucket", storage_path="data") + assert result == obj + + +# Testing function: find_validation_object_on_minio + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_validation_object_found_with_storage_path(mock_get_list): + # Setup + expected_path = "my/storage/rocrate123_validation/validation_status.txt" + obj = DummyObject(expected_path) + mock_get_list.return_value = [obj] + + from app.utils.minio_utils import find_validation_object_on_minio + # Execute + result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket", storage_path="my/storage") + + # Assert + assert result == obj + mock_get_list.assert_called_once_with(expected_path, mock.ANY, "bucket") + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_validation_object_found_without_storage_path(mock_get_list): + # Setup + expected_path = "rocrate123_validation/validation_status.txt" + obj = DummyObject(expected_path) + mock_get_list.return_value = [obj] + + from app.utils.minio_utils import find_validation_object_on_minio + # Execute + result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket") + + # Assert + assert result == obj + mock_get_list.assert_called_once_with(expected_path, mock.ANY, "bucket") + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_validation_object_not_found(mock_get_list): + # Setup: object name does not match exactly + mock_get_list.return_value = [DummyObject("some/other/object.txt")] + + from app.utils.minio_utils import find_validation_object_on_minio, InvalidAPIUsage + # Execute + Assert + with pytest.raises(InvalidAPIUsage) as exc: + find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") + + assert exc.value.status_code == 400 + assert "No validation result yet for RO-Crate: rocrate999" in str(exc.value.message) + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_validation_object_empty_list(mock_get_list): + # Setup: no objects returned + mock_get_list.return_value = [] + + from app.utils.minio_utils import find_validation_object_on_minio, InvalidAPIUsage + # Execute + Assert + with pytest.raises(InvalidAPIUsage) as exc: + find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") + + assert exc.value.status_code == 400 + assert "No validation result yet for RO-Crate: rocrate999" in str(exc.value.message) + + +# Testing function: download_file_from_minio + +@patch("app.utils.minio_utils.logging") +def test_download_success(mock_logging): + mock_minio = MagicMock() + + from app.utils.minio_utils import download_file_from_minio + # No exceptions raised + download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") + + mock_minio.fget_object.assert_called_once_with("bucket", "remote/path.txt", "local/path.txt") + mock_logging.error.assert_not_called() + + +@patch("app.utils.minio_utils.logging") +def test_download_s3error(mock_logging): + mock_minio = MagicMock() + mock_minio.fget_object.side_effect = S3Error("S3 error", "", "", "", "", "") + + from app.utils.minio_utils import download_file_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") + + assert exc.value.status_code == 500 + assert "MinIO S3 Error" in str(exc.value.message) + mock_logging.error.assert_called_once() + + +@patch("app.utils.minio_utils.logging") +def test_download_value_error(mock_logging): + mock_minio = MagicMock() + mock_minio.fget_object.side_effect = ValueError("Missing config") + + from app.utils.minio_utils import download_file_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") + + assert exc.value.status_code == 500 + assert "Configuration Error" in str(exc.value.message) + mock_logging.error.assert_called_once() + + +@patch("app.utils.minio_utils.logging") +def test_download_unexpected_error(mock_logging): + mock_minio = MagicMock() + mock_minio.fget_object.side_effect = RuntimeError("Something bad happened") + + from app.utils.minio_utils import download_file_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") + + assert exc.value.status_code == 500 + assert "Unknown Error" in str(exc.value.message) + mock_logging.error.assert_called_once() + + # Testing function: get_validation_status_from_minio def test_successful_retrieval(mocker, mock_minio_response): @@ -73,18 +329,24 @@ def test_s3_error_raised(mocker): ) mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) - from app.utils.minio_utils import get_validation_status_from_minio - with pytest.raises(S3Error): + from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: get_validation_status_from_minio("crate123") + assert exc.value.status_code == 500 + assert "S3 Error" in str(exc.value.message) + def test_value_error_raised(mocker): mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing env var")) - from app.utils.minio_utils import get_validation_status_from_minio - with pytest.raises(ValueError): + from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: get_validation_status_from_minio("crate123") + assert exc.value.status_code == 500 + assert "Configuration Error" in str(exc.value.message) + def test_generic_exception_raised(mocker): mock_client = MagicMock() @@ -92,87 +354,18 @@ def test_generic_exception_raised(mocker): mock_client.get_object.side_effect = Exception("Unexpected failure") mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) - from app.utils.minio_utils import get_validation_status_from_minio - with pytest.raises(Exception): + from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: get_validation_status_from_minio("crate123") - -# Testing function: fetch_ro_crate_from_minio - -@mock.patch("app.utils.minio_utils.load_dotenv") -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -@mock.patch("app.utils.minio_utils.tempfile.mkdtemp") -@mock.patch("app.utils.minio_utils.os.path.join", side_effect=os.path.join) -def test_fetch_ro_crate_success(mock_join, mock_mkdtemp, mock_get_client, mock_load_dotenv): - # Setup mocks - mock_minio_client = mock.Mock() - mock_get_client.return_value = (mock_minio_client, "test-bucket") - mock_mkdtemp.return_value = "/tmp/testdir" - - crate_id = "test_crate" - expected_path = os.path.join("/tmp/testdir", f"{crate_id}.zip") - - from app.utils.minio_utils import fetch_ro_crate_from_minio - - result = fetch_ro_crate_from_minio(crate_id) - - mock_load_dotenv.assert_called_once() - mock_get_client.assert_called_once() - mock_minio_client.fget_object.assert_called_once_with( - "test-bucket", f"{crate_id}.zip", expected_path - ) - - assert result == expected_path - - -@mock.patch("app.utils.minio_utils.load_dotenv") -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -@mock.patch("app.utils.minio_utils.tempfile.mkdtemp", return_value="/tmp/testdir") -def test_fetch_ro_crate_s3_error(mock_mkdtemp, mock_get_client, mock_load_dotenv): - mock_minio_client = mock.Mock() - mock_get_client.return_value = (mock_minio_client, "test-bucket") - - crate_id = "bad_crate" - mock_minio_client.fget_object.side_effect = S3Error( - code="S3 error", - message=None, - resource=None, - request_id=None, - host_id=None, - response=None - ) - - from app.utils.minio_utils import fetch_ro_crate_from_minio - with pytest.raises(S3Error): - fetch_ro_crate_from_minio(crate_id) - - -@mock.patch("app.utils.minio_utils.load_dotenv") -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing config")) -def test_fetch_ro_crate_value_error(mock_get_client, mock_load_dotenv): - from app.utils.minio_utils import fetch_ro_crate_from_minio - with pytest.raises(ValueError): - fetch_ro_crate_from_minio("any_crate") - - -@mock.patch("app.utils.minio_utils.load_dotenv") -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -@mock.patch("app.utils.minio_utils.tempfile.mkdtemp", return_value="/tmp/testdir") -def test_fetch_ro_crate_unexpected_error(mock_mkdtemp, mock_get_client, mock_load_dotenv): - mock_minio_client = mock.Mock() - mock_get_client.return_value = (mock_minio_client, "test-bucket") - mock_minio_client.fget_object.side_effect = RuntimeError("Unexpected failure") - - from app.utils.minio_utils import fetch_ro_crate_from_minio - with pytest.raises(RuntimeError): - fetch_ro_crate_from_minio("any_crate") + assert exc.value.status_code == 500 + assert "Unknown Error" in str(exc.value.message) # Testing function: update_validation_status_in_minio -@mock.patch("app.utils.minio_utils.load_dotenv") @mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -def test_update_validation_status_success(mock_get_client, mock_load_dotenv): +def test_update_validation_status_success(mock_get_client): mock_minio_client = mock.Mock() mock_get_client.return_value = (mock_minio_client, "test-bucket") @@ -182,7 +375,7 @@ def test_update_validation_status_success(mock_get_client, mock_load_dotenv): from app.utils.minio_utils import update_validation_status_in_minio update_validation_status_in_minio(crate_id, validation_status) - expected_object_name = f"{crate_id}/validation_status.txt" + expected_object_name = f"{crate_id}_validation/validation_status.txt" expected_data = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") mock_minio_client.put_object.assert_called_once() @@ -205,9 +398,8 @@ def test_update_validation_status_success(mock_get_client, mock_load_dotenv): assert kwargs["content_type"] == "application/json" -@mock.patch("app.utils.minio_utils.load_dotenv") @mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -def test_update_validation_status_s3_error(mock_get_client, mock_load_dotenv): +def test_update_validation_status_s3_error(mock_get_client): mock_minio_client = mock.Mock() mock_get_client.return_value = (mock_minio_client, "test-bucket") mock_minio_client.put_object.side_effect = S3Error( @@ -219,80 +411,134 @@ def test_update_validation_status_s3_error(mock_get_client, mock_load_dotenv): response=None ) - from app.utils.minio_utils import update_validation_status_in_minio - with pytest.raises(S3Error): + from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + assert exc.value.status_code == 500 + assert "S3 Error" in str(exc.value.message) + -@mock.patch("app.utils.minio_utils.load_dotenv") @mock.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing env vars")) -def test_update_validation_status_value_error(mock_get_client, mock_load_dotenv): - from app.utils.minio_utils import update_validation_status_in_minio - with pytest.raises(ValueError): +def test_update_validation_status_value_error(mock_get_client): + from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + assert exc.value.status_code == 500 + assert "Configuration Error" in str(exc.value.message) + -@mock.patch("app.utils.minio_utils.load_dotenv") @mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -def test_update_validation_status_unexpected_error(mock_get_client, mock_load_dotenv): +def test_update_validation_status_unexpected_error(mock_get_client): mock_minio_client = mock.Mock() mock_get_client.return_value = (mock_minio_client, "test-bucket") mock_minio_client.put_object.side_effect = RuntimeError("Unexpected failure") - from app.utils.minio_utils import update_validation_status_in_minio - with pytest.raises(RuntimeError): + from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + assert exc.value.status_code == 500 + assert "Unknown Error" in str(exc.value.message) -# Testing function: get_validation_status_from_minio - -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -def test_get_validation_status_success(mock_get_client, mock_minio_response): - mock_minio_client = mock.Mock() - mock_minio_client.get_object.return_value = mock_minio_response - mock_get_client.return_value = (mock_minio_client, "test-bucket") - - crate_id = "crate123" - from app.utils.minio_utils import get_validation_status_from_minio - result = get_validation_status_from_minio(crate_id) - assert result == {"status": "valid"} - mock_minio_client.get_object.assert_called_once_with("test-bucket", f"{crate_id}/validation_status.txt") - mock_minio_response.close.assert_called_once() - mock_minio_response.release_conn.assert_called_once() +# Testing function: fetch_ro_crate_from_minio +@patch("app.utils.minio_utils.download_file_from_minio") +@patch("app.utils.minio_utils.get_minio_object_list") +@patch("app.utils.minio_utils.find_rocrate_object_on_minio") +@patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_fetch_rocrate_zip( + mock_get_client_and_bucket, + mock_find_object, + mock_get_list, + mock_download, + tmp_path, +): + # Setup mocks + mock_get_client_and_bucket.return_value = ("minio_client", "bucket") + rocrate_obj = DummyObject("some/path/rocrate123.zip", is_dir=False) + mock_find_object.return_value = rocrate_obj -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -def test_get_validation_status_s3_error(mock_get_client): - mock_minio_client = mock.Mock() - mock_minio_client.get_object.side_effect = S3Error( - code="S3 error", - message=None, - resource=None, - request_id=None, - host_id=None, - response=None - ) - mock_get_client.return_value = (mock_minio_client, "test-bucket") + from app.utils.minio_utils import fetch_ro_crate_from_minio - from app.utils.minio_utils import get_validation_status_from_minio - with pytest.raises(S3Error): - get_validation_status_from_minio("crate123") + with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): + # Execute + result = fetch_ro_crate_from_minio("rocrate123") + + # Assert + expected_path = tmp_path / "rocrate123.zip" + assert result == str(expected_path) + mock_download.assert_called_once_with( + "minio_client", "bucket", + "some/path/rocrate123.zip", str(expected_path)) + + +@patch("app.utils.minio_utils.download_file_from_minio") +@patch("app.utils.minio_utils.get_minio_object_list") +@patch("app.utils.minio_utils.find_rocrate_object_on_minio") +@patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_fetch_rocrate_directory( + mock_get_client_and_bucket, + mock_find_object, + mock_get_list, + mock_download, + tmp_path, +): + # Setup mocks + mock_get_client_and_bucket.return_value = ("minio_client", "bucket") + rocrate_obj = DummyObject("rocrates/rocrate124", is_dir=True) + mock_find_object.return_value = rocrate_obj + from app.utils.minio_utils import fetch_ro_crate_from_minio -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing config")) -def test_get_validation_status_value_error(mock_get_client): - from app.utils.minio_utils import get_validation_status_from_minio - with pytest.raises(ValueError): - get_validation_status_from_minio("crate123") + with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): + # Objects inside the RO-Crate + mock_get_list.return_value = [ + DummyObject("rocrates/rocrate124/metadata.json"), + DummyObject("rocrates/rocrate124/data/file1.txt"), + ] + + # Execute + result = fetch_ro_crate_from_minio("rocrate124") + + # Assert + expected_root = tmp_path / "rocrate124" + assert result == str(expected_root) + mock_download.assert_any_call( + "minio_client", "bucket", + "rocrates/rocrate124/metadata.json", + str(expected_root / "metadata.json") + ) + mock_download.assert_any_call( + "minio_client", "bucket", + "rocrates/rocrate124/data/file1.txt", + str(expected_root / "data/file1.txt") + ) + + +@patch("app.utils.minio_utils.download_file_from_minio") +@patch("app.utils.minio_utils.get_minio_object_list") +@patch("app.utils.minio_utils.find_rocrate_object_on_minio") +@patch("app.utils.minio_utils.get_minio_client_and_bucket") +def test_fetch_rocrate_handles_empty_dir( + mock_get_client_and_bucket, + mock_find_object, + mock_get_list, + mock_download, + tmp_path, +): + mock_get_client_and_bucket.return_value = ("minio_client", "bucket") + rocrate_obj = DummyObject("rocrate456", is_dir=True) + mock_find_object.return_value = rocrate_obj + mock_get_list.return_value = [] + from app.utils.minio_utils import fetch_ro_crate_from_minio -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") -def test_get_validation_status_generic_exception(mock_get_client): - mock_minio_client = mock.Mock() - mock_minio_client.get_object.side_effect = RuntimeError("Unexpected failure") - mock_get_client.return_value = (mock_minio_client, "test-bucket") + with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): + result = fetch_ro_crate_from_minio("rocrate456") - from app.utils.minio_utils import get_validation_status_from_minio - with pytest.raises(RuntimeError): - get_validation_status_from_minio("crate123") + expected_root = tmp_path / "rocrate456" + assert result == str(expected_root) + mock_download.assert_not_called() From 1b9210609c603d8df9c01294581cb46315f4bc74 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:24:17 +0100 Subject: [PATCH 057/127] cleaned service and task tests to reflect addition of InvalidAPIUsage function --- tests/test_services.py | 13 ++++++++---- tests/test_validation_tasks.py | 39 ++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/tests/test_services.py b/tests/test_services.py index 978f9c8..0489ef9 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -8,6 +8,8 @@ get_ro_crate_validation_task ) +from app.utils.minio_utils import InvalidAPIUsage + @pytest.fixture def flask_app(): @@ -127,8 +129,11 @@ def test_get_validation_missing_id(flask_app): def test_get_validation_exception(flask_app): - with patch("app.services.validation_service.return_ro_crate_validation", side_effect=Exception("DB error")): - response, status = get_ro_crate_validation_task("crate123") + with patch("app.services.validation_service.return_ro_crate_validation", + side_effect=InvalidAPIUsage("MinIO S3 Error: empty", 500)): - assert status == 500 - assert response.json == {"service error": "DB error"} + with pytest.raises(InvalidAPIUsage) as exc_info: + get_ro_crate_validation_task("crate789") + + assert exc_info.value.status_code == 500 + assert "MinIO S3 Error" in str(exc_info.value.message) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 78b786c..e27de42 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -1,4 +1,5 @@ from unittest import mock +import pytest from app.tasks.validation_tasks import ( process_validation_task_by_id, @@ -7,6 +8,7 @@ process_validation_task_by_metadata ) +from app.utils.minio_utils import InvalidAPIUsage # Test function: process_validation_task_by_id @@ -285,21 +287,32 @@ def test_validation_settings_error(mock_validation_settings, mock_validate): # Test function: return_ro_crate_validation @mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") -def test_return_validation_success(mock_get_validation): - mock_get_validation.return_value = {"status": "valid"} +def test_return_validation_returns_dict(mock_get_status): + # Simulate dict result + mock_get_status.return_value = {"status": "passed", "errors": []} - crate_id = "crate-123" - result = return_ro_crate_validation(crate_id) + result = return_ro_crate_validation("crate123") + assert isinstance(result, dict) + assert result["status"] == "passed" + mock_get_status.assert_called_once_with("crate123") - assert result == {"status": "valid"} - mock_get_validation.assert_called_once_with(crate_id) +@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") +def test_return_validation_returns_string(mock_get_status): + # Simulate string result + mock_get_status.return_value = "Validation result: OK" + result = return_ro_crate_validation("crate456") + assert isinstance(result, str) + assert "OK" in result + mock_get_status.assert_called_once_with("crate456") -@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio", side_effect=Exception("Fetch failed")) -def test_return_validation_exception(mock_get_validation): - crate_id = "crate-123" - result = return_ro_crate_validation(crate_id) +@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") +def test_return_validation_raises_error(mock_get_status): + # Simulate exception + mock_get_status.side_effect = InvalidAPIUsage("MinIO S3 Error: empty", 500) - assert isinstance(result, str) - assert "Fetch failed" in result - mock_get_validation.assert_called_once_with(crate_id) + with pytest.raises(InvalidAPIUsage) as exc_info: + return_ro_crate_validation("crate789") + + assert "MinIO S3 Error" in str(exc_info.value.message) + mock_get_status.assert_called_once_with("crate789") \ No newline at end of file From ef4e2745fe5ddd3e5d10fee64f8f2a37fc2c844e Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:25:31 +0100 Subject: [PATCH 058/127] whitespace cleanup --- tests/test_validation_tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index e27de42..45b46f1 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -10,6 +10,7 @@ from app.utils.minio_utils import InvalidAPIUsage + # Test function: process_validation_task_by_id @mock.patch("app.tasks.validation_tasks.os.remove") @@ -296,6 +297,7 @@ def test_return_validation_returns_dict(mock_get_status): assert result["status"] == "passed" mock_get_status.assert_called_once_with("crate123") + @mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") def test_return_validation_returns_string(mock_get_status): # Simulate string result @@ -306,6 +308,7 @@ def test_return_validation_returns_string(mock_get_status): assert "OK" in result mock_get_status.assert_called_once_with("crate456") + @mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") def test_return_validation_raises_error(mock_get_status): # Simulate exception From 2069cebb36268e4dc04f2083ee0296138c8d0723 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:26:08 +0100 Subject: [PATCH 059/127] whitespace cleanup --- tests/test_validation_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 45b46f1..4b395ee 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -318,4 +318,4 @@ def test_return_validation_raises_error(mock_get_status): return_ro_crate_validation("crate789") assert "MinIO S3 Error" in str(exc_info.value.message) - mock_get_status.assert_called_once_with("crate789") \ No newline at end of file + mock_get_status.assert_called_once_with("crate789") From 5c1b1b373bb04d027a3ecd8f15ee7a8ab5485c6d Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 00:45:05 +0100 Subject: [PATCH 060/127] remove erroneous response.release_conn() --- app/utils/minio_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index 5620215..b8edf09 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -280,7 +280,6 @@ def get_minio_object_list(object_path: str, minio_client, bucket_name: str, recu object_list = [obj for obj in response] response.close() - response.release_conn() except S3Error as s3_error: logging.error(f"MinIO S3 Error: {s3_error}") From 712f9ebd6348aeacc35fc95d2c6e43b466a5c7db Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 25 Jul 2025 11:25:20 +0100 Subject: [PATCH 061/127] remove unneeded test for closing connection --- tests/test_minio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 2958e28..93e2af0 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -69,7 +69,6 @@ def test_get_minio_object_list_success(): assert result == mock_objects mock_minio_client.list_objects.assert_called_once_with("my-bucket", "path/", recursive=True) mock_response.close.assert_called_once() - mock_response.release_conn.assert_called_once() def test_get_minio_object_list_s3error(): From 69636292a9ed133fd707fadc36c978ec41cc0ac3 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sat, 26 Jul 2025 15:50:03 +0100 Subject: [PATCH 062/127] clean minio_util doc_strings --- app/utils/minio_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index b8edf09..98d0012 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -186,7 +186,6 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: If it does not exist then a False value is returned. If it does exist then the minio.datatypes.Object is returned. - Inputs: :param rocrate_id: string containing the name of ro-crate :param storage_path: string containing the path within which the ro-crate should be :param minio_client: minio object @@ -224,7 +223,6 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str If it does not exist then a False value is returned. If it does exist then the minio.datatypes.Object is returned. - Inputs: :param rocrate_id: string containing the name of ro-crate :param storage_path: string containing the path within which the ro-crate should be :param minio_client: minio object From a2912abd607722251cfd6965f2c8607aae38a395 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 17:34:11 +0100 Subject: [PATCH 063/127] check crate existence before queuing validation task --- app/services/validation_service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 726988e..893fd5b 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -12,7 +12,8 @@ from app.tasks.validation_tasks import ( process_validation_task_by_id, process_validation_task_by_metadata, - return_ro_crate_validation + return_ro_crate_validation, + check_ro_crate_exists ) logger = logging.getLogger(__name__) @@ -33,8 +34,8 @@ def queue_ro_crate_validation_task( logging.info(f"Processing: {crate_id}, {profile_name}, {webhook_url}") - if not crate_id: - return jsonify({"error": "Missing required parameter: crate_id"}), 400 + if check_ro_crate_exists(crate_id): + logging.info("RO-Crate exists") try: process_validation_task_by_id.delay(crate_id, profile_name, webhook_url) From aa26f6f3c0bbac7b2914c1670bb6db6a9ae0b6d6 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 18:56:39 +0100 Subject: [PATCH 064/127] find_rocrate returns False not raise error --- app/utils/minio_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index 98d0012..d4c8f8f 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -216,7 +216,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: return return_object -def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str, storage_path: str = None) -> object: +def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str, storage_path: str = None) -> object | bool: """ Checks that the requested object exists on the MinIO instance. @@ -227,7 +227,7 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str :param storage_path: string containing the path within which the ro-crate should be :param minio_client: minio object :param bucket_name: string containing bucket on minio - :return return_object: rocrate object we require + :return return_object or False: rocrate object we require, or False result :raise Exception: If RO-Crate can't be found, 400 """ @@ -249,7 +249,7 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str if not return_object: logging.error(f"No RO-Crate with prefix: {rocrate_path}") - raise InvalidAPIUsage(f"No RO-Crate with prefix: {rocrate_path}", 400) + return False else: return return_object From c033cacac3f2e4699e65801733978981bf8c2619 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 18:57:00 +0100 Subject: [PATCH 065/127] tests updated for find_rocrate --- tests/test_minio.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 93e2af0..8a368b8 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -141,7 +141,7 @@ def test_rocrate_found_as_zip(mock_get_list): @patch("app.utils.minio_utils.get_minio_object_list") -def test_rocrate_not_found_raises(mock_get_list): +def test_rocrate_not_found(mock_get_list): # Simulate no matching object mock_get_list.return_value = [ DummyObject("something_else"), @@ -149,12 +149,11 @@ def test_rocrate_not_found_raises(mock_get_list): ] minio_client = MagicMock() - from app.utils.minio_utils import find_rocrate_object_on_minio, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - find_rocrate_object_on_minio("rocrate123", minio_client, "bucket") + from app.utils.minio_utils import find_rocrate_object_on_minio + result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket") - assert exc.value.status_code == 400 - assert "No RO-Crate with prefix" in str(exc.value.message) + mock_get_list.assert_called_once() + assert not result @patch("app.utils.minio_utils.get_minio_object_list") From 204f70d243a0f6a7a3c5128268841e1d21ac629b Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 19:02:32 +0100 Subject: [PATCH 066/127] check_ro_crate_exists added to validation_tasks --- app/tasks/validation_tasks.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index 4d5d9dc..4ebcdac 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -16,7 +16,9 @@ from app.utils.minio_utils import ( fetch_ro_crate_from_minio, update_validation_status_in_minio, - get_validation_status_from_minio + get_validation_status_from_minio, + get_minio_client_and_bucket, + find_rocrate_object_on_minio ) from app.utils.webhook_utils import send_webhook_notification from app.utils.file_utils import build_metadata_only_rocrate @@ -179,6 +181,25 @@ def perform_ro_crate_validation( return str(e) +def check_ro_crate_exists( + crate_id: str, +) -> bool: + """ + Checks for the existence of an RO-Crate using the provided Crate ID. + + :param crate_id: The ID of the RO-Crate that needs validating + :return: Boolean indicating existence + """ + + logging.info(f"Checking for existence of RO-Crate {crate_id}") + + minio_client, bucket_name = get_minio_client_and_bucket() + if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, storage_path=''): + return True + else: + return False + + def return_ro_crate_validation( crate_id: str, ) -> dict | str: @@ -187,7 +208,6 @@ def return_ro_crate_validation( :param crate_id: The ID of the RO-Crate that has been validated :return: The validation result - :raises Exception: If an error occurs in the retrieving the validation result """ logging.info(f"Fetching validation result for RO-Crate {crate_id}") From 71a72b5f6095897178a7c31154724dc1510d859a Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 19:03:01 +0100 Subject: [PATCH 067/127] tests for check_ro_crate_exists --- tests/test_validation_tasks.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 4b395ee..2b53448 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -5,7 +5,8 @@ process_validation_task_by_id, perform_ro_crate_validation, return_ro_crate_validation, - process_validation_task_by_metadata + process_validation_task_by_metadata, + check_ro_crate_exists ) from app.utils.minio_utils import InvalidAPIUsage @@ -319,3 +320,31 @@ def test_return_validation_raises_error(mock_get_status): assert "MinIO S3 Error" in str(exc_info.value.message) mock_get_status.assert_called_once_with("crate789") + + +# Test function: check_ro_crate_exists + +@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio", return_value="crate123") +def test_ro_crate_exists( + mock_find_rocrate, + mock_get_client +): + result = check_ro_crate_exists("crate123") + + mock_get_client.assert_called_once() + mock_find_rocrate.assert_called_once_with("crate123", "mock_client", "mock_bucket", storage_path='') + assert result is True + + +@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio", return_value=False) +def test_ro_crate_does_not_exist( + mock_find_rocrate, + mock_get_client +): + result = check_ro_crate_exists("crate12z") + + mock_get_client.assert_called_once() + mock_find_rocrate.assert_called_once_with("crate12z", "mock_client", "mock_bucket", storage_path='') + assert result is False From 969d5d8a5e5c254ab2cf327f9133868aa925ebc9 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 19:21:32 +0100 Subject: [PATCH 068/127] move invalidapiusage call up to service level --- app/services/validation_service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 893fd5b..7c2a8dd 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -16,6 +16,9 @@ check_ro_crate_exists ) +from app.utils.config import InvalidAPIUsage + + logger = logging.getLogger(__name__) @@ -36,6 +39,9 @@ def queue_ro_crate_validation_task( if check_ro_crate_exists(crate_id): logging.info("RO-Crate exists") + else: + logging.info("RO-Crate does not exist") + raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) try: process_validation_task_by_id.delay(crate_id, profile_name, webhook_url) From 97c25c2786d4db996c041e4ca49aec7d22fe9908 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 19:22:24 +0100 Subject: [PATCH 069/127] rework tests for queue_ro_crate_validation_task --- tests/test_services.py | 64 +++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/tests/test_services.py b/tests/test_services.py index 0489ef9..b8cf05f 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -20,28 +20,48 @@ def flask_app(): # Test function: queue_ro_crate_validation_task -def test_queue_task_success(flask_app): - with patch("app.services.validation_service.process_validation_task_by_id.delay") as mock_delay: - response, status_code = queue_ro_crate_validation_task("crate123", "profileA", "http://webhook.com") - - mock_delay.assert_called_once_with("crate123", "profileA", "http://webhook.com") - assert status_code == 202 - assert response.json == {"message": "Validation in progress"} - - -def test_queue_task_missing_crate_id(flask_app): - response, status_code = queue_ro_crate_validation_task(None) - - assert status_code == 400 - assert response.json == {"error": "Missing required parameter: crate_id"} - - -def test_queue_task_exception(flask_app): - with patch("app.services.validation_service.process_validation_task_by_id.delay", side_effect=Exception("Celery down")): - response, status_code = queue_ro_crate_validation_task("crate123") - - assert status_code == 500 - assert response.json == {"error": "Celery down"} +@patch("app.services.validation_service.process_validation_task_by_id.delay") +@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) +def test_queue_task_success( + mock_exists, + mock_delay, + flask_app +): + response, status_code = queue_ro_crate_validation_task("crate123", "profileA", "http://webhook.com") + + mock_exists.assert_called_once_with("crate123") + mock_delay.assert_called_once_with("crate123", "profileA", "http://webhook.com") + assert status_code == 202 + assert response.json == {"message": "Validation in progress"} + + +@patch("app.services.validation_service.process_validation_task_by_id.delay") +@patch("app.services.validation_service.check_ro_crate_exists", return_value=False) +def test_queue_ro_crate_missing_exception( + mock_exists, + mock_delay, + flask_app +): + with pytest.raises(InvalidAPIUsage) as exc_info: + queue_ro_crate_validation_task("crate12z", "profileA", "http://webhook.com") + + assert "No RO-Crate with prefix: crate12z" in str(exc_info.value.message) + mock_exists.assert_called_once_with("crate12z") + mock_delay.assert_not_called() + + +@patch("app.services.validation_service.process_validation_task_by_id.delay", side_effect=Exception("Celery down")) +@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) +def test_queue_task_exception( + mock_exists, + mock_delay, + flask_app +): + response, status_code = queue_ro_crate_validation_task("crate123") + + mock_exists.assert_called_once_with("crate123") + assert status_code == 500 + assert response.json == {"error": "Celery down"} # Test function: queue_ro_crate_metadata_validation_task From abf1f8cde76c48f46ee270a4bcd9aba45d3657c3 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 22:11:01 +0100 Subject: [PATCH 070/127] remove unused invalidapiusage import --- app/ro_crates/routes/post_routes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 1e15c3d..bf1e1a2 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -12,8 +12,6 @@ queue_ro_crate_validation_task, queue_ro_crate_metadata_validation_task ) -from app.utils.config import InvalidAPIUsage - post_routes_bp = APIBlueprint("post_routes", __name__) From 8082bcbaf1b82aa53a1626d9169b575972004634 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 22:14:28 +0100 Subject: [PATCH 071/127] Validation result retrieval is more robust. Better error catching, with checks for both RO-Crate and validation result existence. Directories are treated properly for RO-Crates and temp directory deletion. --- app/services/validation_service.py | 18 ++++++++++++++---- app/tasks/validation_tasks.py | 27 +++++++++++++++++++++++++-- app/utils/minio_utils.py | 4 ++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 7c2a8dd..b3a2689 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -13,7 +13,8 @@ process_validation_task_by_id, process_validation_task_by_metadata, return_ro_crate_validation, - check_ro_crate_exists + check_ro_crate_exists, + check_validation_exists ) from app.utils.config import InvalidAPIUsage @@ -100,11 +101,20 @@ def get_ro_crate_validation_task( :param crate_id: The ID of the RO-Crate to validate. :return: A tuple containing a JSON response and an HTTP status code. - :raises Exception: If an error occurs whilst retrieving RO-Crate + :raises Exception: If an error occurs whilst retreiving validation result """ logging.info(f"Retrieving validation for: {crate_id}") - if not crate_id: - return jsonify({"error": "Missing required parameter: crate_id"}), 400 + if check_ro_crate_exists(crate_id): + logging.info("RO-Crate exists") + else: + logging.info("RO-Crate does not exist") + raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) + + if check_validation_exists(crate_id): + logging.info("Validation result exists") + else: + logging.info("Validation does not exist") + raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {crate_id}", 400) return return_ro_crate_validation(crate_id), 200 diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index 4ebcdac..e836959 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -18,7 +18,8 @@ update_validation_status_in_minio, get_validation_status_from_minio, get_minio_client_and_bucket, - find_rocrate_object_on_minio + find_rocrate_object_on_minio, + find_validation_object_on_minio ) from app.utils.webhook_utils import send_webhook_notification from app.utils.file_utils import build_metadata_only_rocrate @@ -80,7 +81,10 @@ def process_validation_task_by_id( finally: # Clean up the temporary file if it was created: if file_path and os.path.exists(file_path): - os.remove(file_path) + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) @celery.task @@ -200,6 +204,25 @@ def check_ro_crate_exists( return False +def check_validation_exists( + crate_id: str, +) -> bool: + """ + Checks for the existence of a validation result using the provided Crate ID. + + :param crate_id: The ID of the RO-Crate that needs validating + :return: Boolean indicating existence + """ + + logging.info(f"Checking for existence of RO-Crate {crate_id}") + + minio_client, bucket_name = get_minio_client_and_bucket() + if find_validation_object_on_minio(crate_id, minio_client, bucket_name, storage_path=''): + return True + else: + return False + + def return_ro_crate_validation( crate_id: str, ) -> dict | str: diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index d4c8f8f..efb0c6f 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -211,7 +211,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: if not return_object: logging.error(f"No validation result yet for RO-Crate: {rocrate_id}") - raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {rocrate_id}", 400) + return False else: return return_object @@ -243,7 +243,7 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str return_object = False for obj in rocrate_list: # TODO: We should be checking here for the existence of the ro-crate metadata file within this object too - if (obj.object_name == rocrate_path and obj.is_dir) or obj.object_name == f"{rocrate_path}.zip": + if (obj.object_name == f"{rocrate_path}/" and obj.is_dir) or obj.object_name == f"{rocrate_path}.zip": return_object = obj break From 14808946cb6a3d85bdcac70366a2f35439ba783c Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 22:19:22 +0100 Subject: [PATCH 072/127] integration tests extended, with example files for MinIO store --- tests/data/ro_crates/project_a/ro_crate_4.zip | Bin 0 -> 2070 bytes .../ro_crate_5/ro-crate-metadata.json | 112 +++++++++ .../ro_crate_5/sort-and-change-case.ga | 118 ++++++++++ tests/data/ro_crates/ro_crate_1.zip | Bin 0 -> 2070 bytes .../ro_crate_2/ro-crate-metadata.json | 112 +++++++++ .../ro_crate_2/sort-and-change-case.ga | 118 ++++++++++ tests/data/ro_crates/ro_crate_3.zip | Bin 0 -> 2070 bytes .../validation_status.txt | 1 + .../data/ro_crates/ro_crate_not_validated.zip | Bin 0 -> 2070 bytes tests/test_integration.py | 220 ++++++++++++++++++ 10 files changed, 681 insertions(+) create mode 100644 tests/data/ro_crates/project_a/ro_crate_4.zip create mode 100644 tests/data/ro_crates/project_a/ro_crate_5/ro-crate-metadata.json create mode 100644 tests/data/ro_crates/project_a/ro_crate_5/sort-and-change-case.ga create mode 100644 tests/data/ro_crates/ro_crate_1.zip create mode 100644 tests/data/ro_crates/ro_crate_2/ro-crate-metadata.json create mode 100644 tests/data/ro_crates/ro_crate_2/sort-and-change-case.ga create mode 100644 tests/data/ro_crates/ro_crate_3.zip create mode 100644 tests/data/ro_crates/ro_crate_3_validation/validation_status.txt create mode 100644 tests/data/ro_crates/ro_crate_not_validated.zip diff --git a/tests/data/ro_crates/project_a/ro_crate_4.zip b/tests/data/ro_crates/project_a/ro_crate_4.zip new file mode 100644 index 0000000000000000000000000000000000000000..c574f2029598e72c485405b8b8631460ee2f1734 GIT binary patch literal 2070 zcma)-c{Cf?9>-%XZIKB>wU{)NSYj!)jZkf)B=)5qgEWm)jU`AJrM8(k=|Y#OwXvnd zT1#~-wM^9(Vym5~NE<3Gni*;B#-p9{&NJtD=iPhGUCz1Z^Sk$t@BKKUgoFO z`vQ56Oi4lu0RXa6003k^gn$nt!Z5+UFzhue)Ed{Mg0S^W1fBx2{x{@6FXJSwz`c2}GpX1WN4+ zW8*ht!&AZ*49mz1Myjd;s0S*c86Y)31znuYv3e94ii1skCeeLvEGpLtL#Ksmh>n77 z)#F#7-1{6v@8~6)M-8x)lGP?Rx~G9B+vEve&_dr8QK~l9+e(l0m;Vd$uXmU`TL}B znQ~F&Nd}>F`}sG=rK9BAtg1L6YB}G{KRrSUm9oAsY7JgC33dK>MdpeU-?xYEZ?);e z)f>&<7)o(>;LUkGYN)(r`0I-yRl->AO}==%uOUrh@%~U?R;cE1{SD|-bl0L*Pau>M z&E4^h(3uOLTq?6!w1*&1F)D_};Rc_g`#vQNc%Q(QT4EG#Y+uzPw`S1jtnX+|q}tKW z$*R)I+y>;g(OZfP$<57Qo;FRWSd_=j8Jf1u~<>R$2;kkXwZAhi`|kctT? zO=V*5YF3-yZlnm=#I z9}~C6Dy|f5d3)(BG!STw3`@^!% z4$!JhN_BCf*9-ED4xStgx-`q&m>{vTMwWTYHk8m{vhCP}nrNhTEp`+8xdD~0d%v<$?Xh%Tt@Dc!0B7~ zk`#Yqof5AK8{SIPY^MfQ^{}Y;*q@g_+KG%8=BSVmMo})cQahoo8!`fk>jXmOX^I&p z?_(i7y8A9o38TteC>RSs_EG;OHRx(}lc`fA++oS_b{(FTd8j^kW*`Kyeh-+e2v_;~`USwpl`F0+3*z@*{j+Pe$1N~Zv?L$|&1ZH=E@ z>JJ{J!d+Lc=QLVo>K7H2#Bx3s^=vgnroB!byVkW9Vn&f`GVRnXt@|UGq{XJ1Yq)QS zh%5L6O6$Z*I0b{!C~WVyH8=R1+FzWaxdz5)-j x-B;kQ7Xbwci2c9=-S_|1*&ogSt^*L>ckzRQdQce=(Y==o?RN0)z1d%O{|5DWeX{@n literal 0 HcmV?d00001 diff --git a/tests/data/ro_crates/project_a/ro_crate_5/ro-crate-metadata.json b/tests/data/ro_crates/project_a/ro_crate_5/ro-crate-metadata.json new file mode 100644 index 0000000..7f490d6 --- /dev/null +++ b/tests/data/ro_crates/project_a/ro_crate_5/ro-crate-metadata.json @@ -0,0 +1,112 @@ +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "datePublished": "2024-04-17T13:39:44+00:00", + "hasPart": [ + { + "@id": "sort-and-change-case.ga" + }, + { + "@id": "sort-and-change-case.cwl" + }, + { + "@id": "blank.png" + }, + { + "@id": "README.md" + } + ], + "mainEntity": { + "@id": "sort-and-change-case.ga" + } + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "./" + }, + "conformsTo": [ + { + "@id": "https://w3id.org/ro/crate/1.1" + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" + } + ] + }, + { + "@id": "sort-and-change-case.ga", + "@type": [ + "File", + "SoftwareSourceCode", + "ComputationalWorkflow" + ], + "description": "sort lines and change text to upper case", + "image": { + "@id": "blank.png" + }, + "license": "https://spdx.org/licenses/MIT.html", + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy" + }, + "subjectOf": { + "@id": "sort-and-change-case.cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy", + "@type": "ComputerLanguage", + "identifier": { + "@id": "https://galaxyproject.org/" + }, + "name": "Galaxy", + "url": { + "@id": "https://galaxyproject.org/" + } + }, + { + "@id": "sort-and-change-case.cwl", + "@type": [ + "File", + "SoftwareSourceCode", + "HowTo" + ], + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl", + "@type": "ComputerLanguage", + "alternateName": "CWL", + "identifier": { + "@id": "https://w3id.org/cwl/" + }, + "name": "Common Workflow Language", + "url": { + "@id": "https://www.commonwl.org/" + } + }, + { + "@id": "blank.png", + "@type": [ + "File", + "ImageObject" + ] + }, + { + "@id": "README.md", + "@type": "File", + "about": { + "@id": "./" + }, + "encodingFormat": "text/markdown" + } + ] +} \ No newline at end of file diff --git a/tests/data/ro_crates/project_a/ro_crate_5/sort-and-change-case.ga b/tests/data/ro_crates/project_a/ro_crate_5/sort-and-change-case.ga new file mode 100644 index 0000000..5a19996 --- /dev/null +++ b/tests/data/ro_crates/project_a/ro_crate_5/sort-and-change-case.ga @@ -0,0 +1,118 @@ +{ + "uuid": "e2a8566c-c025-4181-9e90-7ed29d4e4df1", + "tags": [], + "format-version": "0.1", + "name": "sort-and-change-case", + "version": 0, + "steps": { + "0": { + "tool_id": null, + "tool_version": null, + "outputs": [], + "workflow_outputs": [], + "input_connections": {}, + "tool_state": "{}", + "id": 0, + "uuid": "5a36fad2-66c7-4b9e-8759-0fbcae9b8541", + "errors": null, + "name": "Input dataset", + "label": "bed_input", + "inputs": [], + "position": { + "top": 200, + "left": 200 + }, + "annotation": "", + "content_id": null, + "type": "data_input" + }, + "1": { + "tool_id": "sort1", + "tool_version": "1.1.0", + "outputs": [ + { + "type": "input", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "8237f71a-bc2a-494e-a63c-09c1e65ef7c8", + "label": "sorted_bed" + } + ], + "input_connections": { + "input": { + "output_name": "output", + "id": 0 + } + }, + "tool_state": "{\"__page__\": null, \"style\": \"\\\"alpha\\\"\", \"column\": \"\\\"1\\\"\", \"__rerun_remap_job_id__\": null, \"column_set\": \"[]\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\", \"header_lines\": \"\\\"0\\\"\", \"order\": \"\\\"ASC\\\"\"}", + "id": 1, + "uuid": "0b6b3cda-c75f-452b-85b1-8ae4f3302ba4", + "errors": null, + "name": "Sort", + "post_job_actions": {}, + "label": "sort", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Sort" + } + ], + "position": { + "top": 200, + "left": 420 + }, + "annotation": "", + "content_id": "sort1", + "type": "tool" + }, + "2": { + "tool_id": "ChangeCase", + "tool_version": "1.0.0", + "outputs": [ + { + "type": "tabular", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "c31cd733-dab6-4d50-9fec-b644d162397b", + "label": "uppercase_bed" + } + ], + "input_connections": { + "input": { + "output_name": "out_file1", + "id": 1 + } + }, + "tool_state": "{\"__page__\": null, \"casing\": \"\\\"up\\\"\", \"__rerun_remap_job_id__\": null, \"cols\": \"\\\"c1\\\"\", \"delimiter\": \"\\\"TAB\\\"\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\"}", + "id": 2, + "uuid": "9698bcde-0729-48fe-b88d-ccfb6f6153b4", + "errors": null, + "name": "Change Case", + "post_job_actions": {}, + "label": "change_case", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Change Case" + } + ], + "position": { + "top": 200, + "left": 640 + }, + "annotation": "", + "content_id": "ChangeCase", + "type": "tool" + } + }, + "annotation": "", + "a_galaxy_workflow": "true" +} diff --git a/tests/data/ro_crates/ro_crate_1.zip b/tests/data/ro_crates/ro_crate_1.zip new file mode 100644 index 0000000000000000000000000000000000000000..c574f2029598e72c485405b8b8631460ee2f1734 GIT binary patch literal 2070 zcma)-c{Cf?9>-%XZIKB>wU{)NSYj!)jZkf)B=)5qgEWm)jU`AJrM8(k=|Y#OwXvnd zT1#~-wM^9(Vym5~NE<3Gni*;B#-p9{&NJtD=iPhGUCz1Z^Sk$t@BKKUgoFO z`vQ56Oi4lu0RXa6003k^gn$nt!Z5+UFzhue)Ed{Mg0S^W1fBx2{x{@6FXJSwz`c2}GpX1WN4+ zW8*ht!&AZ*49mz1Myjd;s0S*c86Y)31znuYv3e94ii1skCeeLvEGpLtL#Ksmh>n77 z)#F#7-1{6v@8~6)M-8x)lGP?Rx~G9B+vEve&_dr8QK~l9+e(l0m;Vd$uXmU`TL}B znQ~F&Nd}>F`}sG=rK9BAtg1L6YB}G{KRrSUm9oAsY7JgC33dK>MdpeU-?xYEZ?);e z)f>&<7)o(>;LUkGYN)(r`0I-yRl->AO}==%uOUrh@%~U?R;cE1{SD|-bl0L*Pau>M z&E4^h(3uOLTq?6!w1*&1F)D_};Rc_g`#vQNc%Q(QT4EG#Y+uzPw`S1jtnX+|q}tKW z$*R)I+y>;g(OZfP$<57Qo;FRWSd_=j8Jf1u~<>R$2;kkXwZAhi`|kctT? zO=V*5YF3-yZlnm=#I z9}~C6Dy|f5d3)(BG!STw3`@^!% z4$!JhN_BCf*9-ED4xStgx-`q&m>{vTMwWTYHk8m{vhCP}nrNhTEp`+8xdD~0d%v<$?Xh%Tt@Dc!0B7~ zk`#Yqof5AK8{SIPY^MfQ^{}Y;*q@g_+KG%8=BSVmMo})cQahoo8!`fk>jXmOX^I&p z?_(i7y8A9o38TteC>RSs_EG;OHRx(}lc`fA++oS_b{(FTd8j^kW*`Kyeh-+e2v_;~`USwpl`F0+3*z@*{j+Pe$1N~Zv?L$|&1ZH=E@ z>JJ{J!d+Lc=QLVo>K7H2#Bx3s^=vgnroB!byVkW9Vn&f`GVRnXt@|UGq{XJ1Yq)QS zh%5L6O6$Z*I0b{!C~WVyH8=R1+FzWaxdz5)-j x-B;kQ7Xbwci2c9=-S_|1*&ogSt^*L>ckzRQdQce=(Y==o?RN0)z1d%O{|5DWeX{@n literal 0 HcmV?d00001 diff --git a/tests/data/ro_crates/ro_crate_2/ro-crate-metadata.json b/tests/data/ro_crates/ro_crate_2/ro-crate-metadata.json new file mode 100644 index 0000000..7f490d6 --- /dev/null +++ b/tests/data/ro_crates/ro_crate_2/ro-crate-metadata.json @@ -0,0 +1,112 @@ +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "datePublished": "2024-04-17T13:39:44+00:00", + "hasPart": [ + { + "@id": "sort-and-change-case.ga" + }, + { + "@id": "sort-and-change-case.cwl" + }, + { + "@id": "blank.png" + }, + { + "@id": "README.md" + } + ], + "mainEntity": { + "@id": "sort-and-change-case.ga" + } + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "./" + }, + "conformsTo": [ + { + "@id": "https://w3id.org/ro/crate/1.1" + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" + } + ] + }, + { + "@id": "sort-and-change-case.ga", + "@type": [ + "File", + "SoftwareSourceCode", + "ComputationalWorkflow" + ], + "description": "sort lines and change text to upper case", + "image": { + "@id": "blank.png" + }, + "license": "https://spdx.org/licenses/MIT.html", + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy" + }, + "subjectOf": { + "@id": "sort-and-change-case.cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy", + "@type": "ComputerLanguage", + "identifier": { + "@id": "https://galaxyproject.org/" + }, + "name": "Galaxy", + "url": { + "@id": "https://galaxyproject.org/" + } + }, + { + "@id": "sort-and-change-case.cwl", + "@type": [ + "File", + "SoftwareSourceCode", + "HowTo" + ], + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl", + "@type": "ComputerLanguage", + "alternateName": "CWL", + "identifier": { + "@id": "https://w3id.org/cwl/" + }, + "name": "Common Workflow Language", + "url": { + "@id": "https://www.commonwl.org/" + } + }, + { + "@id": "blank.png", + "@type": [ + "File", + "ImageObject" + ] + }, + { + "@id": "README.md", + "@type": "File", + "about": { + "@id": "./" + }, + "encodingFormat": "text/markdown" + } + ] +} \ No newline at end of file diff --git a/tests/data/ro_crates/ro_crate_2/sort-and-change-case.ga b/tests/data/ro_crates/ro_crate_2/sort-and-change-case.ga new file mode 100644 index 0000000..5a19996 --- /dev/null +++ b/tests/data/ro_crates/ro_crate_2/sort-and-change-case.ga @@ -0,0 +1,118 @@ +{ + "uuid": "e2a8566c-c025-4181-9e90-7ed29d4e4df1", + "tags": [], + "format-version": "0.1", + "name": "sort-and-change-case", + "version": 0, + "steps": { + "0": { + "tool_id": null, + "tool_version": null, + "outputs": [], + "workflow_outputs": [], + "input_connections": {}, + "tool_state": "{}", + "id": 0, + "uuid": "5a36fad2-66c7-4b9e-8759-0fbcae9b8541", + "errors": null, + "name": "Input dataset", + "label": "bed_input", + "inputs": [], + "position": { + "top": 200, + "left": 200 + }, + "annotation": "", + "content_id": null, + "type": "data_input" + }, + "1": { + "tool_id": "sort1", + "tool_version": "1.1.0", + "outputs": [ + { + "type": "input", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "8237f71a-bc2a-494e-a63c-09c1e65ef7c8", + "label": "sorted_bed" + } + ], + "input_connections": { + "input": { + "output_name": "output", + "id": 0 + } + }, + "tool_state": "{\"__page__\": null, \"style\": \"\\\"alpha\\\"\", \"column\": \"\\\"1\\\"\", \"__rerun_remap_job_id__\": null, \"column_set\": \"[]\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\", \"header_lines\": \"\\\"0\\\"\", \"order\": \"\\\"ASC\\\"\"}", + "id": 1, + "uuid": "0b6b3cda-c75f-452b-85b1-8ae4f3302ba4", + "errors": null, + "name": "Sort", + "post_job_actions": {}, + "label": "sort", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Sort" + } + ], + "position": { + "top": 200, + "left": 420 + }, + "annotation": "", + "content_id": "sort1", + "type": "tool" + }, + "2": { + "tool_id": "ChangeCase", + "tool_version": "1.0.0", + "outputs": [ + { + "type": "tabular", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "c31cd733-dab6-4d50-9fec-b644d162397b", + "label": "uppercase_bed" + } + ], + "input_connections": { + "input": { + "output_name": "out_file1", + "id": 1 + } + }, + "tool_state": "{\"__page__\": null, \"casing\": \"\\\"up\\\"\", \"__rerun_remap_job_id__\": null, \"cols\": \"\\\"c1\\\"\", \"delimiter\": \"\\\"TAB\\\"\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\"}", + "id": 2, + "uuid": "9698bcde-0729-48fe-b88d-ccfb6f6153b4", + "errors": null, + "name": "Change Case", + "post_job_actions": {}, + "label": "change_case", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Change Case" + } + ], + "position": { + "top": 200, + "left": 640 + }, + "annotation": "", + "content_id": "ChangeCase", + "type": "tool" + } + }, + "annotation": "", + "a_galaxy_workflow": "true" +} diff --git a/tests/data/ro_crates/ro_crate_3.zip b/tests/data/ro_crates/ro_crate_3.zip new file mode 100644 index 0000000000000000000000000000000000000000..c574f2029598e72c485405b8b8631460ee2f1734 GIT binary patch literal 2070 zcma)-c{Cf?9>-%XZIKB>wU{)NSYj!)jZkf)B=)5qgEWm)jU`AJrM8(k=|Y#OwXvnd zT1#~-wM^9(Vym5~NE<3Gni*;B#-p9{&NJtD=iPhGUCz1Z^Sk$t@BKKUgoFO z`vQ56Oi4lu0RXa6003k^gn$nt!Z5+UFzhue)Ed{Mg0S^W1fBx2{x{@6FXJSwz`c2}GpX1WN4+ zW8*ht!&AZ*49mz1Myjd;s0S*c86Y)31znuYv3e94ii1skCeeLvEGpLtL#Ksmh>n77 z)#F#7-1{6v@8~6)M-8x)lGP?Rx~G9B+vEve&_dr8QK~l9+e(l0m;Vd$uXmU`TL}B znQ~F&Nd}>F`}sG=rK9BAtg1L6YB}G{KRrSUm9oAsY7JgC33dK>MdpeU-?xYEZ?);e z)f>&<7)o(>;LUkGYN)(r`0I-yRl->AO}==%uOUrh@%~U?R;cE1{SD|-bl0L*Pau>M z&E4^h(3uOLTq?6!w1*&1F)D_};Rc_g`#vQNc%Q(QT4EG#Y+uzPw`S1jtnX+|q}tKW z$*R)I+y>;g(OZfP$<57Qo;FRWSd_=j8Jf1u~<>R$2;kkXwZAhi`|kctT? zO=V*5YF3-yZlnm=#I z9}~C6Dy|f5d3)(BG!STw3`@^!% z4$!JhN_BCf*9-ED4xStgx-`q&m>{vTMwWTYHk8m{vhCP}nrNhTEp`+8xdD~0d%v<$?Xh%Tt@Dc!0B7~ zk`#Yqof5AK8{SIPY^MfQ^{}Y;*q@g_+KG%8=BSVmMo})cQahoo8!`fk>jXmOX^I&p z?_(i7y8A9o38TteC>RSs_EG;OHRx(}lc`fA++oS_b{(FTd8j^kW*`Kyeh-+e2v_;~`USwpl`F0+3*z@*{j+Pe$1N~Zv?L$|&1ZH=E@ z>JJ{J!d+Lc=QLVo>K7H2#Bx3s^=vgnroB!byVkW9Vn&f`GVRnXt@|UGq{XJ1Yq)QS zh%5L6O6$Z*I0b{!C~WVyH8=R1+FzWaxdz5)-j x-B;kQ7Xbwci2c9=-S_|1*&ogSt^*L>ckzRQdQce=(Y==o?RN0)z1d%O{|5DWeX{@n literal 0 HcmV?d00001 diff --git a/tests/data/ro_crates/ro_crate_3_validation/validation_status.txt b/tests/data/ro_crates/ro_crate_3_validation/validation_status.txt new file mode 100644 index 0000000..40b627e --- /dev/null +++ b/tests/data/ro_crates/ro_crate_3_validation/validation_status.txt @@ -0,0 +1 @@ +{"meta": {"version": "0.2"}, "validation_settings": {"profile_identifier": "ro-crate-1.1", "enable_profile_inheritance": true, "abort_on_first": false, "requirement_severity": "REQUIRED", "rocrate_validator_version": "0.6.5"}, "passed": false, "issues": [{"severity": "REQUIRED", "message": "RO-Crate file descriptor \"ro-crate-metadata.json\" is not in the correct format", "violatingEntity": null, "violatingProperty": null, "violatingPropertyValue": null, "check": {"identifier": "ro-crate-1.1_1.1", "label": "REQUIRED 1.1", "order": 1, "name": "File Descriptor JSON format", "description": "Check if the file descriptor is in the correct format", "severity": "REQUIRED", "requirement": {"identifier": "ro-crate-1.1_1", "name": "File Descriptor JSON format", "description": "The file descriptor MUST be a valid JSON file", "order": 1, "profile": {"identifier": "ro-crate-1.1", "uri": "https://w3id.org/ro/crate/1.1", "name": "RO-Crate Metadata Specification 1.1", "description": "RO-Crate Metadata Specification."}}}}, {"severity": "REQUIRED", "message": "Unexpected error: File not found: ro-crate-metadata.json", "violatingEntity": null, "violatingProperty": null, "violatingPropertyValue": null, "check": {"identifier": "ro-crate-1.1_2.1", "label": "REQUIRED 2.1", "order": 1, "name": "Validation of the compaction format of the file descriptor", "description": "Check if the file descriptor is in the **compacted** JSON-LD format", "severity": "REQUIRED", "requirement": {"identifier": "ro-crate-1.1_2", "name": "File Descriptor JSON-LD format", "description": "The file descriptor MUST be a valid JSON-LD file", "order": 2, "profile": {"identifier": "ro-crate-1.1", "uri": "https://w3id.org/ro/crate/1.1", "name": "RO-Crate Metadata Specification 1.1", "description": "RO-Crate Metadata Specification."}}}}, {"severity": "REQUIRED", "message": "RO-Crate \"ro-crate-metadata.json\" file descriptor is empty", "violatingEntity": null, "violatingProperty": null, "violatingPropertyValue": null, "check": {"identifier": "ro-crate-1.1_3.2", "label": "REQUIRED 3.2", "order": 2, "name": "File Descriptor size check", "description": "Check if the file descriptor is not empty", "severity": "REQUIRED", "requirement": {"identifier": "ro-crate-1.1_3", "name": "File Descriptor existence", "description": "The file descriptor MUST be present in the RO-Crate and MUST not be empty.", "order": 3, "profile": {"identifier": "ro-crate-1.1", "uri": "https://w3id.org/ro/crate/1.1", "name": "RO-Crate Metadata Specification 1.1", "description": "RO-Crate Metadata Specification."}}}}]} \ No newline at end of file diff --git a/tests/data/ro_crates/ro_crate_not_validated.zip b/tests/data/ro_crates/ro_crate_not_validated.zip new file mode 100644 index 0000000000000000000000000000000000000000..c574f2029598e72c485405b8b8631460ee2f1734 GIT binary patch literal 2070 zcma)-c{Cf?9>-%XZIKB>wU{)NSYj!)jZkf)B=)5qgEWm)jU`AJrM8(k=|Y#OwXvnd zT1#~-wM^9(Vym5~NE<3Gni*;B#-p9{&NJtD=iPhGUCz1Z^Sk$t@BKKUgoFO z`vQ56Oi4lu0RXa6003k^gn$nt!Z5+UFzhue)Ed{Mg0S^W1fBx2{x{@6FXJSwz`c2}GpX1WN4+ zW8*ht!&AZ*49mz1Myjd;s0S*c86Y)31znuYv3e94ii1skCeeLvEGpLtL#Ksmh>n77 z)#F#7-1{6v@8~6)M-8x)lGP?Rx~G9B+vEve&_dr8QK~l9+e(l0m;Vd$uXmU`TL}B znQ~F&Nd}>F`}sG=rK9BAtg1L6YB}G{KRrSUm9oAsY7JgC33dK>MdpeU-?xYEZ?);e z)f>&<7)o(>;LUkGYN)(r`0I-yRl->AO}==%uOUrh@%~U?R;cE1{SD|-bl0L*Pau>M z&E4^h(3uOLTq?6!w1*&1F)D_};Rc_g`#vQNc%Q(QT4EG#Y+uzPw`S1jtnX+|q}tKW z$*R)I+y>;g(OZfP$<57Qo;FRWSd_=j8Jf1u~<>R$2;kkXwZAhi`|kctT? zO=V*5YF3-yZlnm=#I z9}~C6Dy|f5d3)(BG!STw3`@^!% z4$!JhN_BCf*9-ED4xStgx-`q&m>{vTMwWTYHk8m{vhCP}nrNhTEp`+8xdD~0d%v<$?Xh%Tt@Dc!0B7~ zk`#Yqof5AK8{SIPY^MfQ^{}Y;*q@g_+KG%8=BSVmMo})cQahoo8!`fk>jXmOX^I&p z?_(i7y8A9o38TteC>RSs_EG;OHRx(}lc`fA++oS_b{(FTd8j^kW*`Kyeh-+e2v_;~`USwpl`F0+3*z@*{j+Pe$1N~Zv?L$|&1ZH=E@ z>JJ{J!d+Lc=QLVo>K7H2#Bx3s^=vgnroB!byVkW9Vn&f`GVRnXt@|UGq{XJ1Yq)QS zh%5L6O6$Z*I0b{!C~WVyH8=R1+FzWaxdz5)-j x-B;kQ7Xbwci2c9=-S_|1*&ogSt^*L>ckzRQdQce=(Y==o?RN0)z1d%O{|5DWeX{@n literal 0 HcmV?d00001 diff --git a/tests/test_integration.py b/tests/test_integration.py index bdf1030..94f781b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -4,6 +4,7 @@ import requests import json import os +from minio import Minio @pytest.fixture(scope="session", autouse=True) @@ -16,12 +17,40 @@ def docker_compose(): ) time.sleep(10) # Wait for services to start — adjust as needed + load_test_data_into_minio() + yield # Run the tests print("Stopping Docker Compose...") subprocess.run(["docker", "compose", "down"], check=True) +def load_test_data_into_minio(): + """Connect to MinIO and upload test files.""" + minio_client = Minio( + endpoint="localhost:9000", + access_key="minioadmin", + secret_key="minioadmin", + secure=False + ) + + bucket_name = "ro-crates" + test_data_dir = "tests/data/ro_crates" + + # Ensure bucket exists + if not minio_client.bucket_exists(bucket_name): + minio_client.make_bucket(bucket_name) + + # Walk and upload files + for root, _, files in os.walk(test_data_dir): + for file_name in files: + file_path = os.path.join(root, file_name) + object_name = os.path.relpath(file_path, test_data_dir) + + print(f"Uploading {file_path} as {object_name} to bucket {bucket_name}") + minio_client.fput_object(bucket_name, object_name, file_path) + + def test_validate_metadata(): url = "http://localhost:5001/v1/ro_crates/validate_metadata" headers = { @@ -50,3 +79,194 @@ def test_validate_metadata(): # Assertions — update based on expected API behavior assert response.status_code == 200 assert response_result['passed'] is True + + +def test_no_rocrate_for_validation(): + ro_crate = "ro_crate_10" + url = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = {} + + response = requests.post(url, json=payload, headers=headers) + + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions — update based on expected API behavior + assert response.status_code == 400 + assert response_result['message'] == f"No RO-Crate with prefix: {ro_crate}" + + +def test_no_validation_result_for_missing_crate(): + ro_crate = "ro_crate_10" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # GET action and tests + response = requests.get(url_get, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 400 + assert response_result['message'] == f"No RO-Crate with prefix: {ro_crate}" + + +def test_get_existing_validation_result(): + ro_crate = "ro_crate_3" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # GET action and tests + response = requests.get(url_get, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_rocrate_not_validated_yet(): + ro_crate = "ro_crate_not_validated" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # GET action and tests + response = requests.get(url_get, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 400 + assert response_result['message'] == f"No validation result yet for RO-Crate: {ro_crate}" + + +def test_zipped_rocrate_validation(): + ro_crate = "ro_crate_1" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = {} + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_directory_rocrate_validation(): + ro_crate = "ro_crate_2" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = {} + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_ignore_rocrates_not_on_basepath(): + ro_crate = "ro_crate_4" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = {} + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 400 + assert response_result == "No RO-Crate with prefix: ro_crate_4" From d5aaac8cdb25f2a37298a631231ec56f48ba7d15 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 22:20:26 +0100 Subject: [PATCH 073/127] updated unit tests --- tests/test_minio.py | 22 ++++------- tests/test_services.py | 67 ++++++++++++++++++++++++---------- tests/test_validation_tasks.py | 42 ++++++++++++++++++++- 3 files changed, 96 insertions(+), 35 deletions(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 8a368b8..c841b9a 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -119,7 +119,7 @@ def test_get_minio_object_list_unexpected_error(): @patch("app.utils.minio_utils.get_minio_object_list") def test_rocrate_found_as_directory(mock_get_list): # Simulate a directory object match - obj = DummyObject("my/path/rocrate123", is_dir=True) + obj = DummyObject("my/path/rocrate123/", is_dir=True) mock_get_list.return_value = [obj] minio_client = MagicMock() @@ -171,7 +171,7 @@ def test_storage_path_none(mock_get_list): @patch("app.utils.minio_utils.get_minio_object_list") def test_storage_path_provided(mock_get_list): # Ensures correct rocrate_path is used when storage_path is provided - obj = DummyObject("data/rocrate789", is_dir=True) + obj = DummyObject("data/rocrate789/", is_dir=True) mock_get_list.return_value = [obj] minio_client = MagicMock() @@ -219,13 +219,10 @@ def test_validation_object_not_found(mock_get_list): # Setup: object name does not match exactly mock_get_list.return_value = [DummyObject("some/other/object.txt")] - from app.utils.minio_utils import find_validation_object_on_minio, InvalidAPIUsage - # Execute + Assert - with pytest.raises(InvalidAPIUsage) as exc: - find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") + from app.utils.minio_utils import find_validation_object_on_minio + result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") - assert exc.value.status_code == 400 - assert "No validation result yet for RO-Crate: rocrate999" in str(exc.value.message) + assert result is False @patch("app.utils.minio_utils.get_minio_object_list") @@ -233,13 +230,10 @@ def test_validation_object_empty_list(mock_get_list): # Setup: no objects returned mock_get_list.return_value = [] - from app.utils.minio_utils import find_validation_object_on_minio, InvalidAPIUsage - # Execute + Assert - with pytest.raises(InvalidAPIUsage) as exc: - find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") + from app.utils.minio_utils import find_validation_object_on_minio + result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") - assert exc.value.status_code == 400 - assert "No validation result yet for RO-Crate: rocrate999" in str(exc.value.message) + assert result is False # Testing function: download_file_from_minio diff --git a/tests/test_services.py b/tests/test_services.py index b8cf05f..e7740cf 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -130,30 +130,57 @@ def test_queue_metadata_exception(flask_app): # Test function: get_ro_crate_validation_task -def test_get_validation_success(flask_app): - with patch("app.services.validation_service.return_ro_crate_validation") as mock_return: - mock_return.return_value = {"status": "valid"} - - response, status = get_ro_crate_validation_task("crate123") - - mock_return.assert_called_once_with("crate123") - assert status == 200 - assert response == {"status": "valid"} +@patch("app.services.validation_service.return_ro_crate_validation", return_value={"status": "valid"}) +@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) +@patch("app.services.validation_service.check_validation_exists", return_value=True) +def test_get_validation_success( + mock_validation, + mock_rocrate, + mock_return, + flask_app +): + response, status = get_ro_crate_validation_task("crate123") + mock_return.assert_called_once_with("crate123") + mock_rocrate.assert_called_once_with("crate123") + mock_validation.assert_called_once_with("crate123") + assert status == 200 + assert response == {"status": "valid"} -def test_get_validation_missing_id(flask_app): - response, status = get_ro_crate_validation_task(None) - assert status == 400 - assert response.json == {"error": "Missing required parameter: crate_id"} +@patch("app.services.validation_service.return_ro_crate_validation", return_value=None) +@patch("app.services.validation_service.check_ro_crate_exists", return_value=False) +@patch("app.services.validation_service.check_validation_exists", return_value=False) +def test_get_validation_missing_ro_crate( + mock_validation, + mock_rocrate, + mock_return, + flask_app +): + with pytest.raises(InvalidAPIUsage) as exc_info: + get_ro_crate_validation_task("crate123") + assert exc_info.value.status_code == 400 + assert "No RO-Crate with prefix: crate123" in str(exc_info.value.message) + mock_rocrate.assert_called_once_with("crate123") + mock_validation.assert_not_called() + mock_return.assert_not_called() -def test_get_validation_exception(flask_app): - with patch("app.services.validation_service.return_ro_crate_validation", - side_effect=InvalidAPIUsage("MinIO S3 Error: empty", 500)): - with pytest.raises(InvalidAPIUsage) as exc_info: - get_ro_crate_validation_task("crate789") +@patch("app.services.validation_service.return_ro_crate_validation", return_value=None) +@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) +@patch("app.services.validation_service.check_validation_exists", return_value=False) +def test_get_validation_missing_validation( + mock_validation, + mock_rocrate, + mock_return, + flask_app +): + with pytest.raises(InvalidAPIUsage) as exc_info: + get_ro_crate_validation_task("crate123") - assert exc_info.value.status_code == 500 - assert "MinIO S3 Error" in str(exc_info.value.message) + assert exc_info.value.status_code == 400 + assert "No validation result yet for RO-Crate: crate123" in str(exc_info.value.message) + mock_rocrate.assert_called_once_with("crate123") + mock_validation.assert_called_once_with("crate123") + mock_return.assert_not_called() diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 2b53448..654b92c 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -16,15 +16,17 @@ @mock.patch("app.tasks.validation_tasks.os.remove") @mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=True) @mock.patch("app.tasks.validation_tasks.send_webhook_notification") @mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") @mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") @mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") -def test_process_validation_success( +def test_process_validation_zipfile_success( mock_fetch, mock_validate, mock_update, mock_webhook, + mock_isfile, mock_exists, mock_remove ): @@ -44,8 +46,43 @@ def test_process_validation_success( mock_remove.assert_called_once_with("/tmp/crate.zip") +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=False) +@mock.patch("app.tasks.validation_tasks.os.path.isdir", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") +def test_process_validation_directory_success( + mock_fetch, + mock_validate, + mock_update, + mock_webhook, + mock_isdir, + mock_isfile, + mock_exists, + mock_rmtree +): + mock_fetch.return_value = "/tmp/crate123/" + + mock_validation_result = mock.Mock() + mock_validation_result.has_issues.return_value = False + mock_validation_result.to_json.return_value = '{"status": "valid"}' + mock_validate.return_value = mock_validation_result + + process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + + mock_fetch.assert_called_once_with("crate123") + mock_validate.assert_called_once_with("/tmp/crate123/", "profileA") + mock_update.assert_called_once_with("crate123", '{"status": "valid"}') + mock_webhook.assert_called_once_with("https://example.com/hook", '{"status": "valid"}') + mock_rmtree.assert_called_once_with("/tmp/crate123/") + + @mock.patch("app.tasks.validation_tasks.os.remove") @mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=True) @mock.patch("app.tasks.validation_tasks.send_webhook_notification") @mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") @mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") @@ -55,6 +92,7 @@ def test_process_validation_fails_with_message( mock_validate, mock_update, mock_webhook, + mock_isfile, mock_exists, mock_remove ): @@ -73,6 +111,7 @@ def test_process_validation_fails_with_message( @mock.patch("app.tasks.validation_tasks.os.remove") @mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=True) @mock.patch("app.tasks.validation_tasks.send_webhook_notification") @mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") @mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation", side_effect=Exception("Unexpected error")) @@ -82,6 +121,7 @@ def test_process_validation_exception( mock_validate, mock_update, mock_webhook, + mock_isfile, mock_exists, mock_remove ): From 75fa69dd1487b47807990a279c89fef4da01f2c7 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 22:23:33 +0100 Subject: [PATCH 074/127] add minio to integration tests --- .github/workflows/test_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 9befbba..6acfa6e 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest requests + pip install pytest requests minio - name: Build Docker Compose Containers run: | From 312dfb017dd0a50dc571241913ba75e26c6a341d Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Sun, 27 Jul 2025 22:44:55 +0100 Subject: [PATCH 075/127] updated example usage in README --- README.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c4c7a2f..b1de0e9 100644 --- a/README.md +++ b/README.md @@ -70,24 +70,20 @@ For testing locally developed containers use the alternate Docker Compose file: ## Example Usage -Validation of RO-Crate with the ID of `1`. No webhook is used here: +Submission of validation of RO-Crate with the ID of `ro_crate_1`. No webhook is used here: ```bash curl -X 'POST' \ - 'http://localhost:5001/ro_crates/validate_by_id_no_webhook' \ + 'http://localhost:5001/ro_crates/v1/ro_crate_1/validation' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ - -d '{ - "crate_id": "1" -}' + -d '{}' ``` -Retrieval of validation result for RO-Crate `1`: +Retrieval of validation result for RO-Crate `ro_crate_1`: ```bash curl -X 'GET' \ - 'http://localhost:5001/ro_crates/get_validation_by_id' \ + 'http://localhost:5001/ro_crates/v1/ro_crate_1/validation' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ - -d '{ - "crate_id": "1" -}' + -d '{}' ``` From 1688aa07115f22cbadf1da8ebed90d0f6a617cd8 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:47:23 +0100 Subject: [PATCH 076/127] add path parameters to API docstrings --- app/ro_crates/routes/get_routes.py | 3 +++ app/ro_crates/routes/post_routes.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/ro_crates/routes/get_routes.py b/app/ro_crates/routes/get_routes.py index 3b43144..e31cae8 100644 --- a/app/ro_crates/routes/get_routes.py +++ b/app/ro_crates/routes/get_routes.py @@ -17,6 +17,9 @@ def get_ro_crate_validation_by_id(crate_id) -> tuple[Response, int]: """ Endpoint to obtain an RO-Crate validation result using its ID from MinIO. + Path Parameters: + - **crate_id**: The RO-Crate ID. _Required_. + Returns: - A tuple containing the validation result and an HTTP status code. diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index bf1e1a2..9dcec32 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -32,7 +32,10 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate using its ID from MinIO. - Parameters: + Path Parameters: + - **crate_id**: The RO-Crate ID. _Required_. + + Request Body Parameters: - **profile_name**: The profile name for validation. _Optional_. - **webhook_url**: The webhook URL where validation results will be sent. _Optional_. @@ -62,7 +65,7 @@ def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate JSON file uploaded to the Service. - Parameters: + Request Body Parameters: - **crate_json**: The RO-Crate JSON-LD, as a string. _Required_ - **profile_name**: The profile name for validation. _Optional_. From b320dcc877bedab459473f5271a9205f0974754d Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 28 Jul 2025 22:54:06 +0100 Subject: [PATCH 077/127] add minio_bucket and root_path to post api --- app/ro_crates/routes/post_routes.py | 13 ++++++- tests/test_post_routes.py | 55 +++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 9dcec32..075eb62 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -17,6 +17,8 @@ class ValidateCrate(Schema): + minio_bucket = String(required=True) + root_path = String(required=False) profile_name = String(required=False) webhook_url = String(required=False) @@ -36,6 +38,8 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: - **crate_id**: The RO-Crate ID. _Required_. Request Body Parameters: + - **minio_bucket**: The MinIO bucket containing the RO-Crate. _Required_ + - **root_path**: The root path containing the RO-Crate. _Optional_ - **profile_name**: The profile name for validation. _Optional_. - **webhook_url**: The webhook URL where validation results will be sent. _Optional_. @@ -46,6 +50,13 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id` or `webhook_url`) are missing. """ + minio_bucket = json_data["minio_bucket"] + + if "root_path" in json_data: + root_path = json_data["root_path"] + else: + root_path = None + if "webhook_url" in json_data: webhook_url = json_data["webhook_url"] else: @@ -56,7 +67,7 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: else: profile_name = None - return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url) + return queue_ro_crate_validation_task(minio_bucket, crate_id, root_path, profile_name, webhook_url) @post_routes_bp.post("/validate_metadata") diff --git a/tests/test_post_routes.py b/tests/test_post_routes.py index c593a70..5f5d2cc 100644 --- a/tests/test_post_routes.py +++ b/tests/test_post_routes.py @@ -15,6 +15,8 @@ def client(): def test_validate_by_id_success(client): crate_id = "crate-123" payload = { + "minio_bucket": "test_bucket", + "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" } @@ -26,12 +28,15 @@ def test_validate_by_id_success(client): assert response.status_code == 202 assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("crate-123", "default", "https://webhook.example.com") + mock_queue.assert_called_once_with("test_bucket", "crate-123", "base_path", "default", "https://webhook.example.com") -def test_validate_by_id_missing_crate_id(client): +def test_validate_by_id_fails_missing_crate_id(client): payload = { - "webhook_url": "https://webhook.example.com" + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" } response = client.post("/v1/ro_crates//validation", json=payload) @@ -39,9 +44,23 @@ def test_validate_by_id_missing_crate_id(client): assert response.status_code == 404 -def test_validate_by_id_missing_profile_name_and_webhook_url(client): +def test_validate_by_id_fails_missing_minio_bucket(client): + crate_id = "crate-123" + payload = { + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + } + + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 422 + + +def test_validate_by_id_missing_root_path_and_profile_name_and_webhook_url(client): crate_id = "crate-123" payload = { + "minio_bucket": "test_bucket", } with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: @@ -51,12 +70,14 @@ def test_validate_by_id_missing_profile_name_and_webhook_url(client): assert response.status_code == 202 assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("crate-123", None, None) + mock_queue.assert_called_once_with("test_bucket", "crate-123", None, None, None) def test_validate_by_id_missing_profile_name(client): crate_id = "crate-123" payload = { + "minio_bucket": "test_bucket", + "root_path": "base_path", "webhook_url": "https://webhook.example.com" } @@ -67,12 +88,14 @@ def test_validate_by_id_missing_profile_name(client): assert response.status_code == 202 assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("crate-123", None, "https://webhook.example.com") + mock_queue.assert_called_once_with("test_bucket", "crate-123", "base_path", None, "https://webhook.example.com") def test_validate_by_id_missing_webhook_url(client): crate_id = "crate-123" payload = { + "minio_bucket": "test_bucket", + "root_path": "base_path", "profile_name": "default" } @@ -83,7 +106,25 @@ def test_validate_by_id_missing_webhook_url(client): assert response.status_code == 202 assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("crate-123", "default", None) + mock_queue.assert_called_once_with("test_bucket", "crate-123", "base_path", "default", None) + + +def test_validate_by_id_missing_root_path(client): + crate_id = "crate-123" + payload = { + "minio_bucket": "test_bucket", + "profile_name": "default", + "webhook_url": "https://webhook.example.com" + } + + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: + mock_queue.return_value = ({"message": "Validation in progress"}, 202) + + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 202 + assert response.json == {"message": "Validation in progress"} + mock_queue.assert_called_once_with("test_bucket", None, "crate-123", "default", "https://webhook.example.com") # Test API: /v1/ro_crates/validate_metadata From 144741fe7742d3a5e961798a69fb86f60f66873e Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 28 Jul 2025 23:05:33 +0100 Subject: [PATCH 078/127] minio_bucket and root_path added to queue_ro_crate_validation --- app/services/validation_service.py | 9 ++++++--- tests/test_services.py | 14 +++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index b3a2689..2142426 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -24,12 +24,14 @@ def queue_ro_crate_validation_task( - crate_id, profile_name=None, webhook_url=None + minio_bucket, crate_id, root_path=None, profile_name=None, webhook_url=None ) -> tuple[Response, int]: """ Queues an RO-Crate for validation with Celery. + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :param profile_name: The profile to validate against. :param webhook_url: The URL to POST the validation results to. :return: A tuple containing a JSON response and an HTTP status code. @@ -37,15 +39,16 @@ def queue_ro_crate_validation_task( """ logging.info(f"Processing: {crate_id}, {profile_name}, {webhook_url}") + logging.info(f"Minio Bucket: {minio_bucket}; Root path: {root_path}") - if check_ro_crate_exists(crate_id): + if check_ro_crate_exists(minio_bucket, crate_id, root_path): logging.info("RO-Crate exists") else: logging.info("RO-Crate does not exist") raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) try: - process_validation_task_by_id.delay(crate_id, profile_name, webhook_url) + process_validation_task_by_id.delay(minio_bucket, crate_id, root_path, profile_name, webhook_url) return jsonify({"message": "Validation in progress"}), 202 except Exception as e: diff --git a/tests/test_services.py b/tests/test_services.py index e7740cf..2c00606 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -27,10 +27,10 @@ def test_queue_task_success( mock_delay, flask_app ): - response, status_code = queue_ro_crate_validation_task("crate123", "profileA", "http://webhook.com") + response, status_code = queue_ro_crate_validation_task("test_bucket", "crate123", "base_path", "profileA", "http://webhook.com") - mock_exists.assert_called_once_with("crate123") - mock_delay.assert_called_once_with("crate123", "profileA", "http://webhook.com") + mock_exists.assert_called_once_with("test_bucket", "crate123", "base_path") + mock_delay.assert_called_once_with("test_bucket", "crate123", "base_path", "profileA", "http://webhook.com") assert status_code == 202 assert response.json == {"message": "Validation in progress"} @@ -43,10 +43,10 @@ def test_queue_ro_crate_missing_exception( flask_app ): with pytest.raises(InvalidAPIUsage) as exc_info: - queue_ro_crate_validation_task("crate12z", "profileA", "http://webhook.com") + queue_ro_crate_validation_task("test_bucket", "crate12z", "base_path", "profileA", "http://webhook.com") assert "No RO-Crate with prefix: crate12z" in str(exc_info.value.message) - mock_exists.assert_called_once_with("crate12z") + mock_exists.assert_called_once_with("test_bucket", "crate12z", "base_path") mock_delay.assert_not_called() @@ -57,9 +57,9 @@ def test_queue_task_exception( mock_delay, flask_app ): - response, status_code = queue_ro_crate_validation_task("crate123") + response, status_code = queue_ro_crate_validation_task("test_bucket", "crate123", None) - mock_exists.assert_called_once_with("crate123") + mock_exists.assert_called_once_with("test_bucket", "crate123", None) assert status_code == 500 assert response.json == {"error": "Celery down"} From 0665d09aad359f0f49a2b4ee5119bacc022051c1 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 28 Jul 2025 23:17:17 +0100 Subject: [PATCH 079/127] minio_bucket and root_path for get ro-crate route --- app/ro_crates/routes/get_routes.py | 24 +++++++++++++++++++++--- app/services/validation_service.py | 10 ++++++---- tests/test_services.py | 18 +++++++++--------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/app/ro_crates/routes/get_routes.py b/app/ro_crates/routes/get_routes.py index e31cae8..4298a79 100644 --- a/app/ro_crates/routes/get_routes.py +++ b/app/ro_crates/routes/get_routes.py @@ -4,7 +4,8 @@ # License: MIT # Copyright (c) 2025 eScience Lab, The University of Manchester -from apiflask import APIBlueprint +from apiflask import APIBlueprint, Schema +from apiflask.fields import String from flask import Response from app.services.validation_service import get_ro_crate_validation_task @@ -12,14 +13,24 @@ get_routes_bp = APIBlueprint("get_routes", __name__) +class ValidateResult(Schema): + minio_bucket = String(required=True) + root_path = String(required=False) + + @get_routes_bp.get("/validation") -def get_ro_crate_validation_by_id(crate_id) -> tuple[Response, int]: +@get_routes_bp.input(ValidateResult(partial=False), location='json') +def get_ro_crate_validation_by_id(json_data, crate_id) -> tuple[Response, int]: """ Endpoint to obtain an RO-Crate validation result using its ID from MinIO. Path Parameters: - **crate_id**: The RO-Crate ID. _Required_. + Request Body Parameters: + - **minio_bucket**: The MinIO bucket containing the RO-Crate. _Required_ + - **root_path**: The root path containing the RO-Crate. _Optional_ + Returns: - A tuple containing the validation result and an HTTP status code. @@ -27,4 +38,11 @@ def get_ro_crate_validation_by_id(crate_id) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id`) are missing. """ - return get_ro_crate_validation_task(crate_id) + minio_bucket = json_data["minio_bucket"] + + if "root_path" in json_data: + root_path = json_data["root_path"] + else: + root_path = None + + return get_ro_crate_validation_task(minio_bucket, crate_id, root_path) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 2142426..f166839 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -97,7 +97,9 @@ def queue_ro_crate_metadata_validation_task( def get_ro_crate_validation_task( - crate_id + minio_bucket: str, + crate_id: str, + root_path: str, ) -> tuple[Response, int]: """ Retrieves an RO-Crate validation result. @@ -108,16 +110,16 @@ def get_ro_crate_validation_task( """ logging.info(f"Retrieving validation for: {crate_id}") - if check_ro_crate_exists(crate_id): + if check_ro_crate_exists(minio_bucket, crate_id, root_path): logging.info("RO-Crate exists") else: logging.info("RO-Crate does not exist") raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) - if check_validation_exists(crate_id): + if check_validation_exists(minio_bucket, crate_id, root_path): logging.info("Validation result exists") else: logging.info("Validation does not exist") raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {crate_id}", 400) - return return_ro_crate_validation(crate_id), 200 + return return_ro_crate_validation(minio_bucket, crate_id, root_path), 200 diff --git a/tests/test_services.py b/tests/test_services.py index 2c00606..068e487 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -139,11 +139,11 @@ def test_get_validation_success( mock_return, flask_app ): - response, status = get_ro_crate_validation_task("crate123") + response, status = get_ro_crate_validation_task("test_bucket", "crate123", "base_path") - mock_return.assert_called_once_with("crate123") - mock_rocrate.assert_called_once_with("crate123") - mock_validation.assert_called_once_with("crate123") + mock_return.assert_called_once_with("test_bucket", "crate123", "base_path") + mock_rocrate.assert_called_once_with("test_bucket", "crate123", "base_path") + mock_validation.assert_called_once_with("test_bucket", "crate123", "base_path") assert status == 200 assert response == {"status": "valid"} @@ -158,11 +158,11 @@ def test_get_validation_missing_ro_crate( flask_app ): with pytest.raises(InvalidAPIUsage) as exc_info: - get_ro_crate_validation_task("crate123") + get_ro_crate_validation_task("test_bucket", "crate123", "base_path") assert exc_info.value.status_code == 400 assert "No RO-Crate with prefix: crate123" in str(exc_info.value.message) - mock_rocrate.assert_called_once_with("crate123") + mock_rocrate.assert_called_once_with("test_bucket", "crate123", "base_path") mock_validation.assert_not_called() mock_return.assert_not_called() @@ -177,10 +177,10 @@ def test_get_validation_missing_validation( flask_app ): with pytest.raises(InvalidAPIUsage) as exc_info: - get_ro_crate_validation_task("crate123") + get_ro_crate_validation_task("test_bucket", "crate123", "base_path") assert exc_info.value.status_code == 400 assert "No validation result yet for RO-Crate: crate123" in str(exc_info.value.message) - mock_rocrate.assert_called_once_with("crate123") - mock_validation.assert_called_once_with("crate123") + mock_rocrate.assert_called_once_with("test_bucket", "crate123", "base_path") + mock_validation.assert_called_once_with("test_bucket", "crate123", "base_path") mock_return.assert_not_called() From 6797e3ceca1438dbd8bac15a9a0863598c096bcd Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 28 Jul 2025 23:33:42 +0100 Subject: [PATCH 080/127] added get_route tests --- tests/test_post_routes.py | 61 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/tests/test_post_routes.py b/tests/test_post_routes.py index 5f5d2cc..92ada57 100644 --- a/tests/test_post_routes.py +++ b/tests/test_post_routes.py @@ -10,7 +10,7 @@ def client(): return app.test_client() -# Test API: /v1/ro_crates/{crate_id}/validation +# Test POST API: /v1/ro_crates/{crate_id}/validation def test_validate_by_id_success(client): crate_id = "crate-123" @@ -127,7 +127,7 @@ def test_validate_by_id_missing_root_path(client): mock_queue.assert_called_once_with("test_bucket", None, "crate-123", "default", "https://webhook.example.com") -# Test API: /v1/ro_crates/validate_metadata +# Test POST API: /v1/ro_crates/validate_metadata def test_validate_metadata_with_all_fields(client: FlaskClient): """ @@ -235,3 +235,60 @@ def test_validate_metadata_emptydict_crate_json(client: FlaskClient): response = client.post("/v1/ro_crates/validate_metadata", json=test_data) assert response.status_code == 422 assert "Required parameter crate_json is empty" in response.get_data(as_text=True) + + +# Test GET API: /v1/ro_crates/{crate_id}/validation + +def test_get_validation_by_id_success(client): + crate_id = "crate-123" + payload = { + "minio_bucket": "test_bucket", + "root_path": "base_path" + } + + with patch("app.ro_crates.routes.get_routes.get_ro_crate_validation_task") as mock_get: + mock_get.return_value = ({"status": "valid"}, 200) + + response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 200 + assert response.json == {"status": "valid"} + mock_get.assert_called_once_with("test_bucket", "crate-123", "base_path") + + +def test_get_validation_by_id_fails_missing_crate_id(client): + payload = { + "minio_bucket": "test_bucket", + "root_path": "base_path" + } + + response = client.get("/v1/ro_crates//validation", json=payload) + + assert response.status_code == 404 + + +def test_get_validation_by_id_fails_missing_minio_bucket(client): + crate_id = "crate-123" + payload = { + "root_path": "base_path" + } + + response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 422 + + +def test_get_validation_by_id_missing_root_path(client): + crate_id = "crate-123" + payload = { + "minio_bucket": "test_bucket", + } + + with patch("app.ro_crates.routes.get_routes.get_ro_crate_validation_task") as mock_get: + mock_get.return_value = ({"message": "Validation in progress"}, 202) + + response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 202 + assert response.json == {"message": "Validation in progress"} + mock_get.assert_called_once_with("test_bucket", "crate-123", None) From f10cfcf47c2bf59f5d1a54ae3b5a3b7b0965a93c Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 28 Jul 2025 23:34:21 +0100 Subject: [PATCH 081/127] rename test_post_routes to test_api_routes --- tests/{test_post_routes.py => test_api_routes.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_post_routes.py => test_api_routes.py} (100%) diff --git a/tests/test_post_routes.py b/tests/test_api_routes.py similarity index 100% rename from tests/test_post_routes.py rename to tests/test_api_routes.py From 94f986aaa51a6f005238eda6cf41c0b3eecf7713 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 28 Jul 2025 23:54:38 +0100 Subject: [PATCH 082/127] add minio_bucket and root_path to checks for ro-crates and validation result --- app/services/validation_service.py | 2 ++ app/tasks/validation_tasks.py | 26 ++++++++++++++------ tests/test_validation_tasks.py | 39 ++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index f166839..b60ae62 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -104,7 +104,9 @@ def get_ro_crate_validation_task( """ Retrieves an RO-Crate validation result. + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :return: A tuple containing a JSON response and an HTTP status code. :raises Exception: If an error occurs whilst retreiving validation result """ diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index e836959..930aadd 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -186,45 +186,55 @@ def perform_ro_crate_validation( def check_ro_crate_exists( + bucket_name: str, crate_id: str, + root_path: str = None, ) -> bool: """ Checks for the existence of an RO-Crate using the provided Crate ID. - :param crate_id: The ID of the RO-Crate that needs validating + :param minio_bucket: The MinIO bucket containing the RO-Crate. + :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :return: Boolean indicating existence """ logging.info(f"Checking for existence of RO-Crate {crate_id}") - minio_client, bucket_name = get_minio_client_and_bucket() - if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, storage_path=''): + minio_client, _ = get_minio_client_and_bucket() + if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path): return True else: return False def check_validation_exists( + bucket_name: str, crate_id: str, + root_path: str = None, ) -> bool: """ Checks for the existence of a validation result using the provided Crate ID. - :param crate_id: The ID of the RO-Crate that needs validating + :param minio_bucket: The MinIO bucket containing the RO-Crate. + :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :return: Boolean indicating existence """ logging.info(f"Checking for existence of RO-Crate {crate_id}") - minio_client, bucket_name = get_minio_client_and_bucket() - if find_validation_object_on_minio(crate_id, minio_client, bucket_name, storage_path=''): + minio_client, _ = get_minio_client_and_bucket() + if find_validation_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path): return True else: return False def return_ro_crate_validation( - crate_id: str, + bucket_name: str, + crate_id: str, + root_path: str = None, ) -> dict | str: """ Retrieves the validation result for an RO-Crate using the provided Crate ID. @@ -235,4 +245,4 @@ def return_ro_crate_validation( logging.info(f"Fetching validation result for RO-Crate {crate_id}") - return get_validation_status_from_minio(crate_id) + return get_validation_status_from_minio(bucket_name, crate_id, root_path) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 654b92c..5696029 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -6,7 +6,8 @@ perform_ro_crate_validation, return_ro_crate_validation, process_validation_task_by_metadata, - check_ro_crate_exists + check_ro_crate_exists, + check_validation_exists ) from app.utils.minio_utils import InvalidAPIUsage @@ -370,10 +371,10 @@ def test_ro_crate_exists( mock_find_rocrate, mock_get_client ): - result = check_ro_crate_exists("crate123") + result = check_ro_crate_exists("test_bucket", "crate123", "base_path") mock_get_client.assert_called_once() - mock_find_rocrate.assert_called_once_with("crate123", "mock_client", "mock_bucket", storage_path='') + mock_find_rocrate.assert_called_once_with("crate123", "mock_client", "test_bucket", storage_path="base_path") assert result is True @@ -383,8 +384,36 @@ def test_ro_crate_does_not_exist( mock_find_rocrate, mock_get_client ): - result = check_ro_crate_exists("crate12z") + result = check_ro_crate_exists("test_bucket", "crate12z", "base_path") mock_get_client.assert_called_once() - mock_find_rocrate.assert_called_once_with("crate12z", "mock_client", "mock_bucket", storage_path='') + mock_find_rocrate.assert_called_once_with("crate12z", "mock_client", "test_bucket", storage_path="base_path") + assert result is False + + +# Test function: check_validation_exists + +@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio", return_value="crate123") +def test_validation_exists( + mock_find_validation, + mock_get_client +): + result = check_validation_exists("test_bucket", "crate123", "base_path") + + mock_get_client.assert_called_once() + mock_find_validation.assert_called_once_with("crate123", "mock_client", "test_bucket", storage_path="base_path") + assert result is True + + +@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio", return_value=False) +def test_validation_does_not_exist( + mock_find_validation, + mock_get_client +): + result = check_validation_exists("test_bucket", "crate12z", "base_path") + + mock_get_client.assert_called_once() + mock_find_validation.assert_called_once_with("crate12z", "mock_client", "test_bucket", storage_path="base_path") assert result is False From 7eb187c746afd8580dc258df655f4a8e29e6ef68 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 00:12:00 +0100 Subject: [PATCH 083/127] change to get_minio_client in validation tasks --- app/tasks/validation_tasks.py | 6 +++--- tests/test_validation_tasks.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index 930aadd..f495c60 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -17,7 +17,7 @@ fetch_ro_crate_from_minio, update_validation_status_in_minio, get_validation_status_from_minio, - get_minio_client_and_bucket, + get_minio_client, find_rocrate_object_on_minio, find_validation_object_on_minio ) @@ -201,7 +201,7 @@ def check_ro_crate_exists( logging.info(f"Checking for existence of RO-Crate {crate_id}") - minio_client, _ = get_minio_client_and_bucket() + minio_client = get_minio_client() if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path): return True else: @@ -224,7 +224,7 @@ def check_validation_exists( logging.info(f"Checking for existence of RO-Crate {crate_id}") - minio_client, _ = get_minio_client_and_bucket() + minio_client = get_minio_client() if find_validation_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path): return True else: diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 5696029..78d4401 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -365,7 +365,7 @@ def test_return_validation_raises_error(mock_get_status): # Test function: check_ro_crate_exists -@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") @mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio", return_value="crate123") def test_ro_crate_exists( mock_find_rocrate, @@ -378,7 +378,7 @@ def test_ro_crate_exists( assert result is True -@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") @mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio", return_value=False) def test_ro_crate_does_not_exist( mock_find_rocrate, @@ -393,7 +393,7 @@ def test_ro_crate_does_not_exist( # Test function: check_validation_exists -@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") @mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio", return_value="crate123") def test_validation_exists( mock_find_validation, @@ -406,7 +406,7 @@ def test_validation_exists( assert result is True -@mock.patch("app.tasks.validation_tasks.get_minio_client_and_bucket", return_value=("mock_client", "mock_bucket")) +@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") @mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio", return_value=False) def test_validation_does_not_exist( mock_find_validation, From 975060732308346566f3e3f1fd9a60dccb9cb2f0 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 00:23:30 +0100 Subject: [PATCH 084/127] pass minio_bucket down to minio_utils instead of using get_minio_client_and_bucket --- app/utils/minio_utils.py | 71 ++++++++++++++++---------------- tests/test_minio.py | 87 ++++++++++++++++------------------------ 2 files changed, 69 insertions(+), 89 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index efb0c6f..401fddd 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -18,41 +18,42 @@ logger = logging.getLogger(__name__) -def fetch_ro_crate_from_minio(crate_id: str) -> str: +def fetch_ro_crate_from_minio(minio_bucket: str, crate_id: str) -> str: """ Fetches an RO-Crate from MinIO based on the crate ID. Downloads the crate as a file and returns local file path. + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to fetch from MinIO. :return: The local file path where the RO-Crate is saved. """ - minio_client, bucket_name = get_minio_client_and_bucket() + minio_client = get_minio_client() - rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, bucket_name) + rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, minio_bucket) rocrate_path = rocrate_object.object_name - rocrate_name = rocrate_path.split('/')[-1] + rocrate_name = rocrate_path.split('/')[-1] temp_dir = tempfile.mkdtemp() root_path = os.path.join(temp_dir, rocrate_name) logging.info( - f"Fetching RO-Crate {rocrate_name} from MinIO bucket {bucket_name}. File path {root_path}" + f"Fetching RO-Crate {rocrate_name} from MinIO bucket {minio_bucket}. File path {root_path}" ) if rocrate_object.is_dir: os.makedirs(os.path.dirname(root_path), exist_ok=True) - objects_list = get_minio_object_list(rocrate_path, minio_client, bucket_name, recursive=True) + objects_list = get_minio_object_list(rocrate_path, minio_client, minio_bucket, recursive=True) for obj in objects_list: relative_path = obj.object_name[len(rocrate_path):].lstrip("/") local_file_path = os.path.join(root_path, relative_path) os.makedirs(os.path.dirname(local_file_path), exist_ok=True) - download_file_from_minio(minio_client, bucket_name, obj.object_name, local_file_path) + download_file_from_minio(minio_client, minio_bucket, obj.object_name, local_file_path) else: file_path = root_path - download_file_from_minio(minio_client, bucket_name, rocrate_path, file_path) + download_file_from_minio(minio_client, minio_bucket, rocrate_path, file_path) logging.info( f"RO-Crate {rocrate_name} fetched successfully and saved to {root_path}." @@ -61,10 +62,11 @@ def fetch_ro_crate_from_minio(crate_id: str) -> str: return root_path -def update_validation_status_in_minio(crate_id: str, validation_status: str) -> None: +def update_validation_status_in_minio(minio_bucket: str, crate_id: str, validation_status: str) -> None: """ Uploads the validation status to the MinIO bucket. + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate in MinIO :param validation_status: The validation result to upload :raises S3Error: If an error occurs during the MinIO operation @@ -79,10 +81,10 @@ def update_validation_status_in_minio(crate_id: str, validation_status: str) -> try: - minio_client, bucket_name = get_minio_client_and_bucket() + minio_client = get_minio_client() minio_client.put_object( - bucket_name, + minio_bucket, object_name, data=BytesIO(validation_string), length=len(validation_string), @@ -102,15 +104,16 @@ def update_validation_status_in_minio(crate_id: str, validation_status: str) -> raise InvalidAPIUsage(f"Unknown Error: {e}", 500) logging.info( - f"Validation status file uploaded to {bucket_name}/{object_name} successfully." + f"Validation status file uploaded to {minio_bucket}/{object_name} successfully." ) -def get_validation_status_from_minio(crate_id: str) -> dict: +def get_validation_status_from_minio(minio_bucket: str, crate_id: str) -> dict: """ Checks for the existence of a validation report for the given RO-Crate in the MinIO bucket. Returns validation message if it exists, or notification that it is missing if not. + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate in MinIO :return validation_status: Either the validation status, or note that this does not exist @@ -123,10 +126,10 @@ def get_validation_status_from_minio(crate_id: str) -> dict: try: - minio_client, bucket_name = get_minio_client_and_bucket() + minio_client = get_minio_client() response = minio_client.get_object( - bucket_name, + minio_bucket, object_name, ) @@ -150,12 +153,12 @@ def get_validation_status_from_minio(crate_id: str) -> dict: return validation_message -def download_file_from_minio(minio_client: object, bucket_name: str, object_path: str, file_path: str) -> None: +def download_file_from_minio(minio_client: object, minio_bucket: str, object_path: str, file_path: str) -> None: """ Downloads a file from MinIO :param minio_client: MinIO object - :param bucket_name: name of MinIO bucket, string + :param minio_bucket: name of MinIO bucket, string :param object_path: path to object on MinIO, string :param file_path: local path, string :raises S3Error: If an error occurs during the MinIO operation @@ -164,7 +167,7 @@ def download_file_from_minio(minio_client: object, bucket_name: str, object_path """ try: - minio_client.fget_object(bucket_name, object_path, file_path) + minio_client.fget_object(minio_bucket, object_path, file_path) except S3Error as s3_error: logging.error(f"MinIO S3 Error: {s3_error}") @@ -179,7 +182,7 @@ def download_file_from_minio(minio_client: object, bucket_name: str, object_path raise InvalidAPIUsage(f"Unknown Error: {e}", 500) -def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: str, storage_path: str = None) -> object: +def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str = None) -> object: """ Checks that the requested object exists on the MinIO instance. @@ -189,7 +192,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: :param rocrate_id: string containing the name of ro-crate :param storage_path: string containing the path within which the ro-crate should be :param minio_client: minio object - :param bucket_name: string containing bucket on minio + :param minio_bucket: string containing bucket on minio :return return_object: rocrate object we require :raise Exception: If validation result can't be found, 400 """ @@ -201,7 +204,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: else: file_path = f"{rocrate_id}_validation/validation_status.txt" - file_list = get_minio_object_list(file_path, minio_client, bucket_name) + file_list = get_minio_object_list(file_path, minio_client, minio_bucket) return_object = False for obj in file_list: @@ -216,7 +219,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, bucket_name: return return_object -def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str, storage_path: str = None) -> object | bool: +def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str = None) -> object | bool: """ Checks that the requested object exists on the MinIO instance. @@ -226,7 +229,7 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str :param rocrate_id: string containing the name of ro-crate :param storage_path: string containing the path within which the ro-crate should be :param minio_client: minio object - :param bucket_name: string containing bucket on minio + :param minio_bucket: string containing bucket on minio :return return_object or False: rocrate object we require, or False result :raise Exception: If RO-Crate can't be found, 400 """ @@ -238,7 +241,7 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str else: rocrate_path = rocrate_id - rocrate_list = get_minio_object_list(rocrate_path, minio_client, bucket_name) + rocrate_list = get_minio_object_list(rocrate_path, minio_client, minio_bucket) return_object = False for obj in rocrate_list: @@ -254,14 +257,14 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, bucket_name: str return return_object -def get_minio_object_list(object_path: str, minio_client, bucket_name: str, recursive: bool = False) -> list: +def get_minio_object_list(object_path: str, minio_client, minio_bucket: str, recursive: bool = False) -> list: """ Creates a list of objects which match the object_id and path_prefix :param object_path: The object ID, string :param path_prefix: Path prefix, string, optional :param minio_client: MinIO client object - :param bucket_name: string + :param minio_bucket: string :param recursive: boolean, default = False :return object_list: List containing objects of type minio.datatypes.Object :raises S3Error: If an error occurs during the MinIO operation, 500 @@ -271,7 +274,7 @@ def get_minio_object_list(object_path: str, minio_client, bucket_name: str, recu try: response = minio_client.list_objects( - bucket_name, + minio_bucket, object_path, recursive=recursive ) @@ -295,11 +298,11 @@ def get_minio_object_list(object_path: str, minio_client, bucket_name: str, recu return object_list -def get_minio_client_and_bucket() -> [Minio, str]: +def get_minio_client() -> Minio: """ - Initialises the MinIO client and retrieves the bucket name from environment variables. + Initialises the MinIO client from environment variables. - :return: A tuple containing the MinIO client and the bucket name. + :return: The MinIO client. :raises ValueError: If required environment variables are not set. """ load_dotenv() @@ -311,10 +314,4 @@ def get_minio_client_and_bucket() -> [Minio, str]: secure=False, ) - bucket_name = os.environ.get("MINIO_BUCKET_NAME") - if not bucket_name: - raise ValueError( - "RO Crate MINIO_BUCKET_NAME is not set in the environment variables." - ) - - return minio_client, bucket_name + return minio_client diff --git a/tests/test_minio.py b/tests/test_minio.py index c841b9a..b30a9f2 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -20,35 +20,21 @@ def __init__(self, name, is_dir=False): self.is_dir = is_dir -# Testing function: get_minio_client_and_bucket +# Testing function: get_minio_client def test_get_minio_client_success(monkeypatch): # Set required env vars monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000") monkeypatch.setenv("MINIO_ROOT_USER", "admin") monkeypatch.setenv("MINIO_ROOT_PASSWORD", "password123") - monkeypatch.setenv("MINIO_BUCKET_NAME", "test-bucket") - from app.utils.minio_utils import get_minio_client_and_bucket - client, bucket = get_minio_client_and_bucket() + from app.utils.minio_utils import get_minio_client + client = get_minio_client() assert isinstance(client, Minio) - assert bucket == "test-bucket" assert client._base_url.host == "localhost:9000" -def test_get_minio_client_missing_bucket_name(monkeypatch): - # Set all except MINIO_BUCKET_NAME - monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000") - monkeypatch.setenv("MINIO_ROOT_USER", "admin") - monkeypatch.setenv("MINIO_ROOT_PASSWORD", "password123") - monkeypatch.setenv("MINIO_BUCKET_NAME", "") - - from app.utils.minio_utils import get_minio_client_and_bucket - with pytest.raises(ValueError, match="MINIO_BUCKET_NAME is not set"): - get_minio_client_and_bucket() - - # Testing function: get_minio_object_list def test_get_minio_object_list_success(): @@ -296,12 +282,11 @@ def test_download_unexpected_error(mock_logging): def test_successful_retrieval(mocker, mock_minio_response): mock_client = MagicMock() - mock_bucket = "test-bucket" mock_client.get_object.return_value = mock_minio_response - mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) + mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) from app.utils.minio_utils import get_validation_status_from_minio - result = get_validation_status_from_minio("crate123") + result = get_validation_status_from_minio("test_bucket", "crate123") assert result == {"status": "valid"} mock_minio_response.close.assert_called_once() @@ -310,7 +295,6 @@ def test_successful_retrieval(mocker, mock_minio_response): def test_s3_error_raised(mocker): mock_client = MagicMock() - mock_bucket = "test-bucket" mock_client.get_object.side_effect = S3Error( code="S3 error", message=None, @@ -319,22 +303,22 @@ def test_s3_error_raised(mocker): host_id=None, response=None ) - mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) + mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("crate123") + get_validation_status_from_minio("test_bucket", "crate123") assert exc.value.status_code == 500 assert "S3 Error" in str(exc.value.message) def test_value_error_raised(mocker): - mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing env var")) + mocker.patch("app.utils.minio_utils.get_minio_client", side_effect=ValueError("Missing env var")) from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("crate123") + get_validation_status_from_minio("test_bucket", "crate123") assert exc.value.status_code == 500 assert "Configuration Error" in str(exc.value.message) @@ -342,13 +326,12 @@ def test_value_error_raised(mocker): def test_generic_exception_raised(mocker): mock_client = MagicMock() - mock_bucket = "test-bucket" mock_client.get_object.side_effect = Exception("Unexpected failure") - mocker.patch("app.utils.minio_utils.get_minio_client_and_bucket", return_value=(mock_client, mock_bucket)) + mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("crate123") + get_validation_status_from_minio("test_bucket", "crate123") assert exc.value.status_code == 500 assert "Unknown Error" in str(exc.value.message) @@ -356,16 +339,16 @@ def test_generic_exception_raised(mocker): # Testing function: update_validation_status_in_minio -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +@mock.patch("app.utils.minio_utils.get_minio_client") def test_update_validation_status_success(mock_get_client): mock_minio_client = mock.Mock() - mock_get_client.return_value = (mock_minio_client, "test-bucket") + mock_get_client.return_value = mock_minio_client crate_id = "crate123" validation_status = json.dumps({"status": "valid", "errors": []}) from app.utils.minio_utils import update_validation_status_in_minio - update_validation_status_in_minio(crate_id, validation_status) + update_validation_status_in_minio("test_bucket", crate_id, validation_status) expected_object_name = f"{crate_id}_validation/validation_status.txt" expected_data = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") @@ -381,7 +364,7 @@ def test_update_validation_status_success(mock_get_client): actual_data_stream = args[2] if len(args) > 2 else kwargs["data"] length = args[3] if len(args) > 3 else kwargs["length"] - assert bucket_name == "test-bucket" + assert bucket_name == "test_bucket" assert object_name == expected_object_name assert isinstance(actual_data_stream, BytesIO) actual_data_stream.seek(0) @@ -390,10 +373,10 @@ def test_update_validation_status_success(mock_get_client): assert kwargs["content_type"] == "application/json" -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +@mock.patch("app.utils.minio_utils.get_minio_client") def test_update_validation_status_s3_error(mock_get_client): mock_minio_client = mock.Mock() - mock_get_client.return_value = (mock_minio_client, "test-bucket") + mock_get_client.return_value = mock_minio_client mock_minio_client.put_object.side_effect = S3Error( code="S3 error", message=None, @@ -405,31 +388,31 @@ def test_update_validation_status_s3_error(mock_get_client): from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + update_validation_status_in_minio("test_bucket", "crate123", json.dumps({"status": "valid"})) assert exc.value.status_code == 500 assert "S3 Error" in str(exc.value.message) -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket", side_effect=ValueError("Missing env vars")) +@mock.patch("app.utils.minio_utils.get_minio_client", side_effect=ValueError("Missing env vars")) def test_update_validation_status_value_error(mock_get_client): from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + update_validation_status_in_minio("test_bucket", "crate123", json.dumps({"status": "valid"})) assert exc.value.status_code == 500 assert "Configuration Error" in str(exc.value.message) -@mock.patch("app.utils.minio_utils.get_minio_client_and_bucket") +@mock.patch("app.utils.minio_utils.get_minio_client") def test_update_validation_status_unexpected_error(mock_get_client): mock_minio_client = mock.Mock() - mock_get_client.return_value = (mock_minio_client, "test-bucket") + mock_get_client.return_value = mock_minio_client mock_minio_client.put_object.side_effect = RuntimeError("Unexpected failure") from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("crate123", json.dumps({"status": "valid"})) + update_validation_status_in_minio("test_bucket", "crate123", json.dumps({"status": "valid"})) assert exc.value.status_code == 500 assert "Unknown Error" in str(exc.value.message) @@ -440,7 +423,7 @@ def test_update_validation_status_unexpected_error(mock_get_client): @patch("app.utils.minio_utils.download_file_from_minio") @patch("app.utils.minio_utils.get_minio_object_list") @patch("app.utils.minio_utils.find_rocrate_object_on_minio") -@patch("app.utils.minio_utils.get_minio_client_and_bucket") +@patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_zip( mock_get_client_and_bucket, mock_find_object, @@ -449,7 +432,7 @@ def test_fetch_rocrate_zip( tmp_path, ): # Setup mocks - mock_get_client_and_bucket.return_value = ("minio_client", "bucket") + mock_get_client_and_bucket.return_value = "minio_client" rocrate_obj = DummyObject("some/path/rocrate123.zip", is_dir=False) mock_find_object.return_value = rocrate_obj @@ -457,20 +440,20 @@ def test_fetch_rocrate_zip( with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): # Execute - result = fetch_ro_crate_from_minio("rocrate123") + result = fetch_ro_crate_from_minio("test_bucket", "rocrate123") # Assert expected_path = tmp_path / "rocrate123.zip" assert result == str(expected_path) mock_download.assert_called_once_with( - "minio_client", "bucket", + "minio_client", "test_bucket", "some/path/rocrate123.zip", str(expected_path)) @patch("app.utils.minio_utils.download_file_from_minio") @patch("app.utils.minio_utils.get_minio_object_list") @patch("app.utils.minio_utils.find_rocrate_object_on_minio") -@patch("app.utils.minio_utils.get_minio_client_and_bucket") +@patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_directory( mock_get_client_and_bucket, mock_find_object, @@ -479,7 +462,7 @@ def test_fetch_rocrate_directory( tmp_path, ): # Setup mocks - mock_get_client_and_bucket.return_value = ("minio_client", "bucket") + mock_get_client_and_bucket.return_value = "minio_client" rocrate_obj = DummyObject("rocrates/rocrate124", is_dir=True) mock_find_object.return_value = rocrate_obj @@ -493,18 +476,18 @@ def test_fetch_rocrate_directory( ] # Execute - result = fetch_ro_crate_from_minio("rocrate124") + result = fetch_ro_crate_from_minio("test_bucket", "rocrate124") # Assert expected_root = tmp_path / "rocrate124" assert result == str(expected_root) mock_download.assert_any_call( - "minio_client", "bucket", + "minio_client", "test_bucket", "rocrates/rocrate124/metadata.json", str(expected_root / "metadata.json") ) mock_download.assert_any_call( - "minio_client", "bucket", + "minio_client", "test_bucket", "rocrates/rocrate124/data/file1.txt", str(expected_root / "data/file1.txt") ) @@ -513,7 +496,7 @@ def test_fetch_rocrate_directory( @patch("app.utils.minio_utils.download_file_from_minio") @patch("app.utils.minio_utils.get_minio_object_list") @patch("app.utils.minio_utils.find_rocrate_object_on_minio") -@patch("app.utils.minio_utils.get_minio_client_and_bucket") +@patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_handles_empty_dir( mock_get_client_and_bucket, mock_find_object, @@ -521,7 +504,7 @@ def test_fetch_rocrate_handles_empty_dir( mock_download, tmp_path, ): - mock_get_client_and_bucket.return_value = ("minio_client", "bucket") + mock_get_client_and_bucket.return_value = "minio_client" rocrate_obj = DummyObject("rocrate456", is_dir=True) mock_find_object.return_value = rocrate_obj mock_get_list.return_value = [] @@ -529,7 +512,7 @@ def test_fetch_rocrate_handles_empty_dir( from app.utils.minio_utils import fetch_ro_crate_from_minio with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): - result = fetch_ro_crate_from_minio("rocrate456") + result = fetch_ro_crate_from_minio("test_bucket", "rocrate456") expected_root = tmp_path / "rocrate456" assert result == str(expected_root) From bbb6987a015a49509b2a94000a061a2b4dd10b90 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 00:25:49 +0100 Subject: [PATCH 085/127] correct api test call order --- tests/test_api_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 92ada57..76df5a4 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -124,7 +124,7 @@ def test_validate_by_id_missing_root_path(client): assert response.status_code == 202 assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("test_bucket", None, "crate-123", "default", "https://webhook.example.com") + mock_queue.assert_called_once_with("test_bucket", "crate-123", None, "default", "https://webhook.example.com") # Test POST API: /v1/ro_crates/validate_metadata From d9a8f8774e13d8e723635581e4f851f8953f1071 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 00:28:57 +0100 Subject: [PATCH 086/127] add test_bucket and base_path to validation task tests --- tests/test_validation_tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 78d4401..e7b45a8 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -334,10 +334,10 @@ def test_return_validation_returns_dict(mock_get_status): # Simulate dict result mock_get_status.return_value = {"status": "passed", "errors": []} - result = return_ro_crate_validation("crate123") + result = return_ro_crate_validation("test_bucket", "crate123", None) assert isinstance(result, dict) assert result["status"] == "passed" - mock_get_status.assert_called_once_with("crate123") + mock_get_status.assert_called_once_with("test_bucket", "crate123", None) @mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") @@ -345,10 +345,10 @@ def test_return_validation_returns_string(mock_get_status): # Simulate string result mock_get_status.return_value = "Validation result: OK" - result = return_ro_crate_validation("crate456") + result = return_ro_crate_validation("test_bucket", "crate456", None) assert isinstance(result, str) assert "OK" in result - mock_get_status.assert_called_once_with("crate456") + mock_get_status.assert_called_once_with("test_bucket", "crate456", None) @mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") @@ -357,10 +357,10 @@ def test_return_validation_raises_error(mock_get_status): mock_get_status.side_effect = InvalidAPIUsage("MinIO S3 Error: empty", 500) with pytest.raises(InvalidAPIUsage) as exc_info: - return_ro_crate_validation("crate789") + return_ro_crate_validation("test_bucket", "crate789", None) assert "MinIO S3 Error" in str(exc_info.value.message) - mock_get_status.assert_called_once_with("crate789") + mock_get_status.assert_called_once_with("test_bucket", "crate789", None) # Test function: check_ro_crate_exists From c64d6c2b30a0ac5c08616cb22b57f6334f64802a Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 00:46:37 +0100 Subject: [PATCH 087/127] root_path mandatory for minio_util calls --- app/tasks/validation_tasks.py | 6 +++--- app/utils/minio_utils.py | 11 +++++++---- tests/test_minio.py | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index f495c60..601efa9 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -188,7 +188,7 @@ def perform_ro_crate_validation( def check_ro_crate_exists( bucket_name: str, crate_id: str, - root_path: str = None, + root_path: str, ) -> bool: """ Checks for the existence of an RO-Crate using the provided Crate ID. @@ -211,7 +211,7 @@ def check_ro_crate_exists( def check_validation_exists( bucket_name: str, crate_id: str, - root_path: str = None, + root_path: str, ) -> bool: """ Checks for the existence of a validation result using the provided Crate ID. @@ -234,7 +234,7 @@ def check_validation_exists( def return_ro_crate_validation( bucket_name: str, crate_id: str, - root_path: str = None, + root_path: str, ) -> dict | str: """ Retrieves the validation result for an RO-Crate using the provided Crate ID. diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index 401fddd..2181579 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -108,7 +108,7 @@ def update_validation_status_in_minio(minio_bucket: str, crate_id: str, validati ) -def get_validation_status_from_minio(minio_bucket: str, crate_id: str) -> dict: +def get_validation_status_from_minio(minio_bucket: str, crate_id: str, root_path: str) -> dict: """ Checks for the existence of a validation report for the given RO-Crate in the MinIO bucket. Returns validation message if it exists, or notification that it is missing if not. @@ -120,7 +120,10 @@ def get_validation_status_from_minio(minio_bucket: str, crate_id: str) -> dict: """ # The object in MinIO is _validation/validation_status.txt - object_name = f"{crate_id}_validation/validation_status.txt" + if root_path: + object_name = f"{root_path}/{crate_id}_validation/validation_status.txt" + else: + object_name = f"{crate_id}_validation/validation_status.txt" logging.info(f"Getting object {object_name}") @@ -182,7 +185,7 @@ def download_file_from_minio(minio_client: object, minio_bucket: str, object_pat raise InvalidAPIUsage(f"Unknown Error: {e}", 500) -def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str = None) -> object: +def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str) -> object: """ Checks that the requested object exists on the MinIO instance. @@ -219,7 +222,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: return return_object -def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str = None) -> object | bool: +def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str) -> object | bool: """ Checks that the requested object exists on the MinIO instance. diff --git a/tests/test_minio.py b/tests/test_minio.py index b30a9f2..13c41e3 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -122,7 +122,7 @@ def test_rocrate_found_as_zip(mock_get_list): minio_client = MagicMock() from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket") + result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", None) assert result == obj @@ -136,7 +136,7 @@ def test_rocrate_not_found(mock_get_list): minio_client = MagicMock() from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket") + result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", None) mock_get_list.assert_called_once() assert not result @@ -150,7 +150,7 @@ def test_storage_path_none(mock_get_list): minio_client = MagicMock() from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate456", minio_client, "bucket") + result = find_rocrate_object_on_minio("rocrate456", minio_client, "bucket", None) assert result == obj @@ -193,7 +193,7 @@ def test_validation_object_found_without_storage_path(mock_get_list): from app.utils.minio_utils import find_validation_object_on_minio # Execute - result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket") + result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket", None) # Assert assert result == obj @@ -206,7 +206,7 @@ def test_validation_object_not_found(mock_get_list): mock_get_list.return_value = [DummyObject("some/other/object.txt")] from app.utils.minio_utils import find_validation_object_on_minio - result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") + result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket", None) assert result is False @@ -217,7 +217,7 @@ def test_validation_object_empty_list(mock_get_list): mock_get_list.return_value = [] from app.utils.minio_utils import find_validation_object_on_minio - result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket") + result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket", None) assert result is False @@ -286,7 +286,7 @@ def test_successful_retrieval(mocker, mock_minio_response): mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) from app.utils.minio_utils import get_validation_status_from_minio - result = get_validation_status_from_minio("test_bucket", "crate123") + result = get_validation_status_from_minio("test_bucket", "crate123", None) assert result == {"status": "valid"} mock_minio_response.close.assert_called_once() @@ -307,7 +307,7 @@ def test_s3_error_raised(mocker): from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("test_bucket", "crate123") + get_validation_status_from_minio("test_bucket", "crate123", None) assert exc.value.status_code == 500 assert "S3 Error" in str(exc.value.message) @@ -318,7 +318,7 @@ def test_value_error_raised(mocker): from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("test_bucket", "crate123") + get_validation_status_from_minio("test_bucket", "crate123", None) assert exc.value.status_code == 500 assert "Configuration Error" in str(exc.value.message) @@ -331,7 +331,7 @@ def test_generic_exception_raised(mocker): from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("test_bucket", "crate123") + get_validation_status_from_minio("test_bucket", "crate123", None) assert exc.value.status_code == 500 assert "Unknown Error" in str(exc.value.message) From 8e16ba8cdda749554d4d42027812cce53393dc83 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 11:59:48 +0100 Subject: [PATCH 088/127] added rocrate_bucket and root_path to all required functions now --- app/tasks/validation_tasks.py | 16 +++++++------ app/utils/minio_utils.py | 45 ++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index 601efa9..f8833de 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -29,12 +29,14 @@ @celery.task def process_validation_task_by_id( - crate_id: str, profile_name: str | None, webhook_url: str | None + minio_bucket: str, crate_id: str, root_path: str, profile_name: str | None, webhook_url: str | None ) -> None: """ Background task to process the RO-Crate validation by ID. + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :param profile_name: The name of the validation profile to use. Defaults to None. :param webhook_url: The webhook URL to send notifications to. Defaults to None. :raises Exception: If an error occurs during the validation process. @@ -46,7 +48,7 @@ def process_validation_task_by_id( try: # Fetch the RO-Crate from MinIO using the provided ID: - file_path = fetch_ro_crate_from_minio(crate_id) + file_path = fetch_ro_crate_from_minio(minio_bucket, crate_id, root_path) logging.info(f"Processing validation task for {file_path}") @@ -59,12 +61,12 @@ def process_validation_task_by_id( raise Exception(f"Validation failed: {validation_result}") if not validation_result.has_issues(): - logging.info(f"RO Crate {file_path} is valid.") + logging.info(f"RO Crate {crate_id} is valid.") else: - logging.info(f"RO Crate {file_path} is invalid.") + logging.info(f"RO Crate {crate_id} is invalid.") # Update the validation status in MinIO: - update_validation_status_in_minio(crate_id, validation_result.to_json()) + update_validation_status_in_minio(minio_bucket, crate_id, root_path, validation_result.to_json()) # TODO: Prepare the data to send to the webhook, and send the webhook notification. @@ -202,7 +204,7 @@ def check_ro_crate_exists( logging.info(f"Checking for existence of RO-Crate {crate_id}") minio_client = get_minio_client() - if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path): + if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, root_path): return True else: return False @@ -225,7 +227,7 @@ def check_validation_exists( logging.info(f"Checking for existence of RO-Crate {crate_id}") minio_client = get_minio_client() - if find_validation_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path): + if find_validation_object_on_minio(crate_id, minio_client, bucket_name, root_path): return True else: return False diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index 2181579..5bdb0a6 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -18,48 +18,49 @@ logger = logging.getLogger(__name__) -def fetch_ro_crate_from_minio(minio_bucket: str, crate_id: str) -> str: +def fetch_ro_crate_from_minio(minio_bucket: str, crate_id: str, root_path: str) -> str: """ Fetches an RO-Crate from MinIO based on the crate ID. Downloads the crate as a file and returns local file path. :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to fetch from MinIO. + :param root_path: The root path containing the RO-Crate. :return: The local file path where the RO-Crate is saved. """ minio_client = get_minio_client() - rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, minio_bucket) + rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, minio_bucket, root_path) - rocrate_path = rocrate_object.object_name - rocrate_name = rocrate_path.split('/')[-1] + rocrate_minio_path = rocrate_object.object_name + rocrate_name = rocrate_minio_path.split('/')[-1] temp_dir = tempfile.mkdtemp() - root_path = os.path.join(temp_dir, rocrate_name) + local_root_path = os.path.join(temp_dir, rocrate_name) logging.info( - f"Fetching RO-Crate {rocrate_name} from MinIO bucket {minio_bucket}. File path {root_path}" + f"Fetching RO-Crate {rocrate_name} from MinIO bucket {minio_bucket}. File path {local_root_path}" ) if rocrate_object.is_dir: - os.makedirs(os.path.dirname(root_path), exist_ok=True) + os.makedirs(os.path.dirname(local_root_path), exist_ok=True) - objects_list = get_minio_object_list(rocrate_path, minio_client, minio_bucket, recursive=True) + objects_list = get_minio_object_list(rocrate_minio_path, minio_client, minio_bucket, recursive=True) for obj in objects_list: - relative_path = obj.object_name[len(rocrate_path):].lstrip("/") - local_file_path = os.path.join(root_path, relative_path) + relative_path = obj.object_name[len(rocrate_minio_path):].lstrip("/") + local_file_path = os.path.join(local_root_path, relative_path) os.makedirs(os.path.dirname(local_file_path), exist_ok=True) download_file_from_minio(minio_client, minio_bucket, obj.object_name, local_file_path) else: - file_path = root_path - download_file_from_minio(minio_client, minio_bucket, rocrate_path, file_path) + file_path = local_root_path + download_file_from_minio(minio_client, minio_bucket, rocrate_minio_path, file_path) logging.info( - f"RO-Crate {rocrate_name} fetched successfully and saved to {root_path}." + f"RO-Crate {rocrate_name} fetched successfully and saved to {local_root_path}." ) - return root_path + return local_root_path def update_validation_status_in_minio(minio_bucket: str, crate_id: str, validation_status: str) -> None: @@ -185,7 +186,7 @@ def download_file_from_minio(minio_client: object, minio_bucket: str, object_pat raise InvalidAPIUsage(f"Unknown Error: {e}", 500) -def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str) -> object: +def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, root_path: str) -> object: """ Checks that the requested object exists on the MinIO instance. @@ -193,7 +194,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: If it does exist then the minio.datatypes.Object is returned. :param rocrate_id: string containing the name of ro-crate - :param storage_path: string containing the path within which the ro-crate should be + :param root_path: string containing the path within which the ro-crate should be :param minio_client: minio object :param minio_bucket: string containing bucket on minio :return return_object: rocrate object we require @@ -202,8 +203,8 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: logging.info(f"Finding Validation result: {rocrate_id}_validation/validation_status.txt") - if storage_path: - file_path = f"{storage_path}/{rocrate_id}_validation/validation_status.txt" + if root_path: + file_path = f"{root_path}/{rocrate_id}_validation/validation_status.txt" else: file_path = f"{rocrate_id}_validation/validation_status.txt" @@ -222,7 +223,7 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: return return_object -def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str) -> object | bool: +def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, root_path: str) -> object | bool: """ Checks that the requested object exists on the MinIO instance. @@ -230,7 +231,7 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: st If it does exist then the minio.datatypes.Object is returned. :param rocrate_id: string containing the name of ro-crate - :param storage_path: string containing the path within which the ro-crate should be + :param root_path: string containing the path within which the ro-crate should be :param minio_client: minio object :param minio_bucket: string containing bucket on minio :return return_object or False: rocrate object we require, or False result @@ -239,8 +240,8 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: st logging.info(f"Finding RO-Crate: {rocrate_id}") - if storage_path: - rocrate_path = f"{storage_path}/{rocrate_id}" + if root_path: + rocrate_path = f"{root_path}/{rocrate_id}" else: rocrate_path = rocrate_id From de23a9699489706a58ced752b90a02c084ef3bee Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:02:56 +0100 Subject: [PATCH 089/127] minio and validation_tasks tests updated --- tests/test_minio.py | 36 +++++++++++++++++----------------- tests/test_validation_tasks.py | 18 ++++++++--------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 13c41e3..f2bc21f 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -110,7 +110,7 @@ def test_rocrate_found_as_directory(mock_get_list): minio_client = MagicMock() from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", storage_path="my/path") + result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", root_path="my/path") assert result == obj @@ -162,7 +162,7 @@ def test_storage_path_provided(mock_get_list): minio_client = MagicMock() from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate789", minio_client, "bucket", storage_path="data") + result = find_rocrate_object_on_minio("rocrate789", minio_client, "bucket", root_path="data") assert result == obj @@ -177,7 +177,7 @@ def test_validation_object_found_with_storage_path(mock_get_list): from app.utils.minio_utils import find_validation_object_on_minio # Execute - result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket", storage_path="my/storage") + result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket", root_path="my/storage") # Assert assert result == obj @@ -425,14 +425,14 @@ def test_update_validation_status_unexpected_error(mock_get_client): @patch("app.utils.minio_utils.find_rocrate_object_on_minio") @patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_zip( - mock_get_client_and_bucket, + mock_get_client, mock_find_object, mock_get_list, mock_download, tmp_path, ): # Setup mocks - mock_get_client_and_bucket.return_value = "minio_client" + mock_get_client.return_value = "minio_client" rocrate_obj = DummyObject("some/path/rocrate123.zip", is_dir=False) mock_find_object.return_value = rocrate_obj @@ -440,14 +440,14 @@ def test_fetch_rocrate_zip( with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): # Execute - result = fetch_ro_crate_from_minio("test_bucket", "rocrate123") + result = fetch_ro_crate_from_minio("test_bucket", "rocrate123", "some/path") - # Assert - expected_path = tmp_path / "rocrate123.zip" - assert result == str(expected_path) - mock_download.assert_called_once_with( - "minio_client", "test_bucket", - "some/path/rocrate123.zip", str(expected_path)) + # Assert + expected_path = tmp_path / "rocrate123.zip" + assert result == str(expected_path) + mock_download.assert_called_once_with( + "minio_client", "test_bucket", + "some/path/rocrate123.zip", str(expected_path)) @patch("app.utils.minio_utils.download_file_from_minio") @@ -455,14 +455,14 @@ def test_fetch_rocrate_zip( @patch("app.utils.minio_utils.find_rocrate_object_on_minio") @patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_directory( - mock_get_client_and_bucket, + mock_get_client, mock_find_object, mock_get_list, mock_download, tmp_path, ): # Setup mocks - mock_get_client_and_bucket.return_value = "minio_client" + mock_get_client.return_value = "minio_client" rocrate_obj = DummyObject("rocrates/rocrate124", is_dir=True) mock_find_object.return_value = rocrate_obj @@ -476,7 +476,7 @@ def test_fetch_rocrate_directory( ] # Execute - result = fetch_ro_crate_from_minio("test_bucket", "rocrate124") + result = fetch_ro_crate_from_minio("test_bucket", "rocrate124", "rocrates") # Assert expected_root = tmp_path / "rocrate124" @@ -498,13 +498,13 @@ def test_fetch_rocrate_directory( @patch("app.utils.minio_utils.find_rocrate_object_on_minio") @patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_handles_empty_dir( - mock_get_client_and_bucket, + mock_get_client, mock_find_object, mock_get_list, mock_download, tmp_path, ): - mock_get_client_and_bucket.return_value = "minio_client" + mock_get_client.return_value = "minio_client" rocrate_obj = DummyObject("rocrate456", is_dir=True) mock_find_object.return_value = rocrate_obj mock_get_list.return_value = [] @@ -512,7 +512,7 @@ def test_fetch_rocrate_handles_empty_dir( from app.utils.minio_utils import fetch_ro_crate_from_minio with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): - result = fetch_ro_crate_from_minio("test_bucket", "rocrate456") + result = fetch_ro_crate_from_minio("test_bucket", "rocrate456", "") expected_root = tmp_path / "rocrate456" assert result == str(expected_root) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index e7b45a8..a58eac3 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -38,11 +38,11 @@ def test_process_validation_zipfile_success( mock_validation_result.to_json.return_value = '{"status": "valid"}' mock_validate.return_value = mock_validation_result - process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") - mock_fetch.assert_called_once_with("crate123") + mock_fetch.assert_called_once_with("test_bucket", "crate123", "") mock_validate.assert_called_once_with("/tmp/crate.zip", "profileA") - mock_update.assert_called_once_with("crate123", '{"status": "valid"}') + mock_update.assert_called_once_with("test_bucket", "crate123", "", '{"status": "valid"}') mock_webhook.assert_called_once_with("https://example.com/hook", '{"status": "valid"}') mock_remove.assert_called_once_with("/tmp/crate.zip") @@ -72,11 +72,11 @@ def test_process_validation_directory_success( mock_validation_result.to_json.return_value = '{"status": "valid"}' mock_validate.return_value = mock_validation_result - process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") - mock_fetch.assert_called_once_with("crate123") + mock_fetch.assert_called_once_with("test_bucket", "crate123", "") mock_validate.assert_called_once_with("/tmp/crate123/", "profileA") - mock_update.assert_called_once_with("crate123", '{"status": "valid"}') + mock_update.assert_called_once_with("test_bucket", "crate123", "", '{"status": "valid"}') mock_webhook.assert_called_once_with("https://example.com/hook", '{"status": "valid"}') mock_rmtree.assert_called_once_with("/tmp/crate123/") @@ -100,7 +100,7 @@ def test_process_validation_fails_with_message( mock_fetch.return_value = "/tmp/crate.zip" mock_validate.return_value = "Validation failed" - process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") mock_update.assert_not_called() mock_webhook.assert_called_once() @@ -128,7 +128,7 @@ def test_process_validation_exception( ): mock_fetch.return_value = "/tmp/crate.zip" - process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") mock_update.assert_not_called() mock_webhook.assert_called_once() @@ -150,7 +150,7 @@ def test_process_validation_fetch_error( mock_webhook, mock_exists ): - process_validation_task_by_id("crate123", "profileA", "https://example.com/hook") + process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") mock_validate.assert_not_called() mock_update.assert_not_called() From f47d6763b3f97d51d3d9e0a6dc5c8b547888232c Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:05:06 +0100 Subject: [PATCH 090/127] integration tests will now return docker container logs on failure --- .github/workflows/test_docker.yml | 2 +- tests/test_integration.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 6acfa6e..98ff5f2 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest requests minio + pip install pytest requests minio docker - name: Build Docker Compose Containers run: | diff --git a/tests/test_integration.py b/tests/test_integration.py index 94f781b..f51fb2f 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -4,11 +4,17 @@ import requests import json import os +import docker from minio import Minio +@pytest.fixture(scope="session") +def docker_client(): + return docker.from_env() + + @pytest.fixture(scope="session", autouse=True) -def docker_compose(): +def docker_compose(docker_client): """Start Docker Compose before tests, shut down after.""" print("Starting Docker Compose...") subprocess.run( @@ -21,6 +27,13 @@ def docker_compose(): yield # Run the tests + for container in docker_client.containers.list(): + if "cratey-validator" in container.name: + logs = container.logs().decode("utf-8") + + print(f"\n======= Logs from {container.name} container =======") + print(logs) + print("Stopping Docker Compose...") subprocess.run(["docker", "compose", "down"], check=True) From b7c053e3d0384d5fa9616de21141358819f63af8 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:05:32 +0100 Subject: [PATCH 091/127] integration tests updated --- tests/test_integration.py | 41 ++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index f51fb2f..443d323 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -103,7 +103,9 @@ def test_no_rocrate_for_validation(): } # The API expects the JSON to be passed as a string - payload = {} + payload = { + "minio_bucket" : "ro-crates" + } response = requests.post(url, json=payload, headers=headers) @@ -126,8 +128,13 @@ def test_no_validation_result_for_missing_crate(): "Content-Type": "application/json" } + # The API expects the JSON to be passed as a string + payload = { + "minio_bucket" : "ro-crates" + } + # GET action and tests - response = requests.get(url_get, headers=headers) + response = requests.get(url_get, json=payload, headers=headers) response_result = response.json() # Print response for debugging @@ -147,8 +154,13 @@ def test_get_existing_validation_result(): "Content-Type": "application/json" } + # The API expects the JSON to be passed as a string + payload = { + "minio_bucket" : "ro-crates" + } + # GET action and tests - response = requests.get(url_get, headers=headers) + response = requests.get(url_get, json=payload, headers=headers) response_result = response.json() # Print response for debugging @@ -168,8 +180,13 @@ def test_rocrate_not_validated_yet(): "Content-Type": "application/json" } + # The API expects the JSON to be passed as a string + payload = { + "minio_bucket" : "ro-crates" + } + # GET action and tests - response = requests.get(url_get, headers=headers) + response = requests.get(url_get, json=payload, headers=headers) response_result = response.json() # Print response for debugging @@ -191,7 +208,9 @@ def test_zipped_rocrate_validation(): } # The API expects the JSON to be passed as a string - payload = {} + payload = { + "minio_bucket" : "ro-crates" + } # POST action and tests response = requests.post(url_post, json=payload, headers=headers) @@ -209,7 +228,7 @@ def test_zipped_rocrate_validation(): time.sleep(10) # GET action and tests - response = requests.get(url_get, headers=headers) + response = requests.get(url_get, json=payload, headers=headers) response_result = response.json() # Print response for debugging @@ -231,7 +250,9 @@ def test_directory_rocrate_validation(): } # The API expects the JSON to be passed as a string - payload = {} + payload = { + "minio_bucket" : "ro-crates" + } # POST action and tests response = requests.post(url_post, json=payload, headers=headers) @@ -249,7 +270,7 @@ def test_directory_rocrate_validation(): time.sleep(10) # GET action and tests - response = requests.get(url_get, headers=headers) + response = requests.get(url_get, json=payload, headers=headers) response_result = response.json() # Print response for debugging @@ -270,7 +291,9 @@ def test_ignore_rocrates_not_on_basepath(): } # The API expects the JSON to be passed as a string - payload = {} + payload = { + "minio_bucket" : "ro-crates" + } # POST action and tests response = requests.post(url_post, json=payload, headers=headers) From 1d60d2b90b894015d0b158a0dfe45d1a057206b7 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:06:00 +0100 Subject: [PATCH 092/127] add formatting to pytest output --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..96735eb --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S From 773f3fa08e86ca5c469307e35269c85fa53121a8 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:49:00 +0100 Subject: [PATCH 093/127] remove last storage_path references in tests --- tests/test_validation_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index a58eac3..51b8063 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -374,7 +374,7 @@ def test_ro_crate_exists( result = check_ro_crate_exists("test_bucket", "crate123", "base_path") mock_get_client.assert_called_once() - mock_find_rocrate.assert_called_once_with("crate123", "mock_client", "test_bucket", storage_path="base_path") + mock_find_rocrate.assert_called_once_with("crate123", "mock_client", "test_bucket", "base_path") assert result is True @@ -387,7 +387,7 @@ def test_ro_crate_does_not_exist( result = check_ro_crate_exists("test_bucket", "crate12z", "base_path") mock_get_client.assert_called_once() - mock_find_rocrate.assert_called_once_with("crate12z", "mock_client", "test_bucket", storage_path="base_path") + mock_find_rocrate.assert_called_once_with("crate12z", "mock_client", "test_bucket", "base_path") assert result is False @@ -402,7 +402,7 @@ def test_validation_exists( result = check_validation_exists("test_bucket", "crate123", "base_path") mock_get_client.assert_called_once() - mock_find_validation.assert_called_once_with("crate123", "mock_client", "test_bucket", storage_path="base_path") + mock_find_validation.assert_called_once_with("crate123", "mock_client", "test_bucket", "base_path") assert result is True @@ -415,5 +415,5 @@ def test_validation_does_not_exist( result = check_validation_exists("test_bucket", "crate12z", "base_path") mock_get_client.assert_called_once() - mock_find_validation.assert_called_once_with("crate12z", "mock_client", "test_bucket", storage_path="base_path") + mock_find_validation.assert_called_once_with("crate12z", "mock_client", "test_bucket", "base_path") assert result is False From 30717a69a95752415d5a3dcb048b349c0909a649 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:59:21 +0100 Subject: [PATCH 094/127] added extra wait for integration tests, for GH actions --- tests/test_integration.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index 443d323..a770bef 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -235,6 +235,21 @@ def test_zipped_rocrate_validation(): print("Status Code:", response.status_code) print("Response JSON:", response_result) + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + # Assertions assert response.status_code == 200 assert response_result["passed"] is False @@ -277,6 +292,21 @@ def test_directory_rocrate_validation(): print("Status Code:", response.status_code) print("Response JSON:", response_result) + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + # Assertions assert response.status_code == 200 assert response_result["passed"] is False From 66f96b5a3f631766a5d911c50e28e5dfae3433a0 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:19:58 +0100 Subject: [PATCH 095/127] output more logs from integration tests --- .github/workflows/test_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 98ff5f2..9af7b23 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -28,7 +28,7 @@ jobs: docker compose -f docker-compose-develop.yml build - name: Spin Up Docker Compose and Run Tests - run: pytest tests/test_integration.py -v + run: pytest -s -v tests/test_integration.py - name: Ensure that Docker Compose is Shutdown if: always() From f2290079a7bd812b3be509f71e31869d31652300 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:32:54 +0100 Subject: [PATCH 096/127] add root_path to update_validation_status_in_minio --- app/utils/minio_utils.py | 8 ++++++-- tests/test_minio.py | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index 5bdb0a6..925af05 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -63,7 +63,7 @@ def fetch_ro_crate_from_minio(minio_bucket: str, crate_id: str, root_path: str) return local_root_path -def update_validation_status_in_minio(minio_bucket: str, crate_id: str, validation_status: str) -> None: +def update_validation_status_in_minio(minio_bucket: str, crate_id: str, root_path: str, validation_status: str) -> None: """ Uploads the validation status to the MinIO bucket. @@ -75,7 +75,11 @@ def update_validation_status_in_minio(minio_bucket: str, crate_id: str, validati :raises Exception: If an unexpected error occurs """ - object_name = f"{crate_id}_validation/validation_status.txt" + # The object in MinIO is _validation/validation_status.txt + if root_path: + object_name = f"{root_path}/{crate_id}_validation/validation_status.txt" + else: + object_name = f"{crate_id}_validation/validation_status.txt" # convert pretty string to dictionary, then back to plain utf-8 encoded string validation_string = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") diff --git a/tests/test_minio.py b/tests/test_minio.py index f2bc21f..02cde5a 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -348,7 +348,7 @@ def test_update_validation_status_success(mock_get_client): validation_status = json.dumps({"status": "valid", "errors": []}) from app.utils.minio_utils import update_validation_status_in_minio - update_validation_status_in_minio("test_bucket", crate_id, validation_status) + update_validation_status_in_minio("test_bucket", crate_id, "", validation_status) expected_object_name = f"{crate_id}_validation/validation_status.txt" expected_data = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") @@ -388,7 +388,7 @@ def test_update_validation_status_s3_error(mock_get_client): from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("test_bucket", "crate123", json.dumps({"status": "valid"})) + update_validation_status_in_minio("test_bucket", "crate123", "", json.dumps({"status": "valid"})) assert exc.value.status_code == 500 assert "S3 Error" in str(exc.value.message) @@ -398,7 +398,7 @@ def test_update_validation_status_s3_error(mock_get_client): def test_update_validation_status_value_error(mock_get_client): from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("test_bucket", "crate123", json.dumps({"status": "valid"})) + update_validation_status_in_minio("test_bucket", "crate123", "", json.dumps({"status": "valid"})) assert exc.value.status_code == 500 assert "Configuration Error" in str(exc.value.message) @@ -412,7 +412,7 @@ def test_update_validation_status_unexpected_error(mock_get_client): from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("test_bucket", "crate123", json.dumps({"status": "valid"})) + update_validation_status_in_minio("test_bucket", "crate123", "", json.dumps({"status": "valid"})) assert exc.value.status_code == 500 assert "Unknown Error" in str(exc.value.message) From dd782ef642f9584bfadee2dec30c78c4e06a57d0 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:44:06 +0100 Subject: [PATCH 097/127] integration tests for ro-crates not in base directory --- tests/test_integration.py | 118 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index a770bef..4d7e5ec 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -336,3 +336,121 @@ def test_ignore_rocrates_not_on_basepath(): # Assertions assert response.status_code == 400 assert response_result == "No RO-Crate with prefix: ro_crate_4" + + +def test_zipped_rocrate_in_subdirectory_validation(): + ro_crate = "ro_crate_4" + subdir_path = "project_a" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_bucket" : "ro-crates", + "root_path" : subdir_path + } + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_directory_rocrate_in_subdirectory_validation(): + ro_crate = "ro_crate_5" + subdir_path = "project_a" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_bucket" : "ro-crates", + "root_path" : subdir_path + } + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False From 16a444211900f7b36f216bd7c8976e23680d58ba Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 31 Jul 2025 21:20:01 +0100 Subject: [PATCH 098/127] merge api missing element tests --- tests/test_api_routes.py | 49 +++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 76df5a4..50252eb 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -31,30 +31,33 @@ def test_validate_by_id_success(client): mock_queue.assert_called_once_with("test_bucket", "crate-123", "base_path", "default", "https://webhook.example.com") -def test_validate_by_id_fails_missing_crate_id(client): - payload = { - "minio_bucket": "test_bucket", - "root_path": "base_path", - "webhook_url": "https://webhook.example.com", - "profile_name": "default" - } - - response = client.post("/v1/ro_crates//validation", json=payload) - - assert response.status_code == 404 - - -def test_validate_by_id_fails_missing_minio_bucket(client): - crate_id = "crate-123" - payload = { - "root_path": "base_path", - "webhook_url": "https://webhook.example.com", - "profile_name": "default" - } - +@pytest.mark.parametrize( + "crate_id, payload, status_code", + [ + ( + "", { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 404 + ), + ( + "crate-123", { + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 422 + ), + ], + ids=[ + "missing_crate_id_returns_404", + "missing_minio_bucket_returns_422" + ] +) +def test_validate_fails_missing_elements(client, crate_id, payload, status_code): response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) - - assert response.status_code == 422 + assert response.status_code == status_code def test_validate_by_id_missing_root_path_and_profile_name_and_webhook_url(client): From 03bb03b0a18d1429878f7dc34858e70ef66c0c99 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 31 Jul 2025 21:41:02 +0100 Subject: [PATCH 099/127] combine the validate_by_id success tests --- tests/test_api_routes.py | 55 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 50252eb..ecc9aa7 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -12,23 +12,38 @@ def client(): # Test POST API: /v1/ro_crates/{crate_id}/validation -def test_validate_by_id_success(client): - crate_id = "crate-123" - payload = { - "minio_bucket": "test_bucket", - "root_path": "base_path", - "webhook_url": "https://webhook.example.com", - "profile_name": "default" - } - +@pytest.mark.parametrize( + "crate_id, payload, status_code, response_json", + [ + ( + "crate-123", { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), + ( + "crate-123", { + "minio_bucket": "test_bucket" + }, 202, {"message": "Validation in progress"} + ), + ], + ids=["validate_by_id", "validate_with_missing_root_path_and_profile_name_and_webhook_url"] +) +def test_validate_by_id_success(client, crate_id, payload, status_code, response_json): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: - mock_queue.return_value = ({"message": "Validation in progress"}, 202) + mock_queue.return_value = (response_json, status_code) response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) - assert response.status_code == 202 - assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("test_bucket", "crate-123", "base_path", "default", "https://webhook.example.com") + minio_bucket = payload["minio_bucket"] if "minio_bucket" in payload else None + root_path = payload["root_path"] if "root_path" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None + webhook_url = payload["webhook_url"] if "webhook_url" in payload else None + assert response.status_code == status_code + assert response.json == response_json + mock_queue.assert_called_once_with(minio_bucket, crate_id, root_path, profile_name, webhook_url) @pytest.mark.parametrize( @@ -60,20 +75,6 @@ def test_validate_fails_missing_elements(client, crate_id, payload, status_code) assert response.status_code == status_code -def test_validate_by_id_missing_root_path_and_profile_name_and_webhook_url(client): - crate_id = "crate-123" - payload = { - "minio_bucket": "test_bucket", - } - - with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: - mock_queue.return_value = ({"message": "Validation in progress"}, 202) - - response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) - - assert response.status_code == 202 - assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("test_bucket", "crate-123", None, None, None) def test_validate_by_id_missing_profile_name(client): From ac4187eb2e742035c0a9a9c8f0cc4174ab53363c Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 31 Jul 2025 22:13:10 +0100 Subject: [PATCH 100/127] validate_metadata api tests parameterised --- tests/test_api_routes.py | 241 +++++++++++++-------------------------- 1 file changed, 82 insertions(+), 159 deletions(-) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index ecc9aa7..661e7c1 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -23,15 +23,38 @@ def client(): "profile_name": "default" }, 202, {"message": "Validation in progress"} ), + ( + "crate-123", { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + }, 202, {"message": "Validation in progress"} + ), + ( + "crate-123", { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), + ( + "crate-123", { + "minio_bucket": "test_bucket", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), ( "crate-123", { "minio_bucket": "test_bucket" }, 202, {"message": "Validation in progress"} ), ], - ids=["validate_by_id", "validate_with_missing_root_path_and_profile_name_and_webhook_url"] + ids=["validate_by_id", "validate_with_missing_profile_name", + "validate_with_missing_webhook_url", "validate_with_missing_root_path", + "validate_with_missing_root_path_and_profile_name_and_webhook_url"] ) -def test_validate_by_id_success(client, crate_id, payload, status_code, response_json): +def test_validate_by_id_success(client: FlaskClient, crate_id: str, payload: dict, status_code: int, response_json: dict): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: mock_queue.return_value = (response_json, status_code) @@ -70,175 +93,75 @@ def test_validate_by_id_success(client, crate_id, payload, status_code, response "missing_minio_bucket_returns_422" ] ) -def test_validate_fails_missing_elements(client, crate_id, payload, status_code): +def test_validate_fails_missing_elements(client: FlaskClient, crate_id: str, payload: dict, status_code: int): response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) assert response.status_code == status_code - - -def test_validate_by_id_missing_profile_name(client): - crate_id = "crate-123" - payload = { - "minio_bucket": "test_bucket", - "root_path": "base_path", - "webhook_url": "https://webhook.example.com" - } - - with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: - mock_queue.return_value = ({"message": "Validation in progress"}, 202) - - response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) - - assert response.status_code == 202 - assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("test_bucket", "crate-123", "base_path", None, "https://webhook.example.com") - - -def test_validate_by_id_missing_webhook_url(client): - crate_id = "crate-123" - payload = { - "minio_bucket": "test_bucket", - "root_path": "base_path", - "profile_name": "default" - } - - with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: - mock_queue.return_value = ({"message": "Validation in progress"}, 202) - - response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) - - assert response.status_code == 202 - assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("test_bucket", "crate-123", "base_path", "default", None) - - -def test_validate_by_id_missing_root_path(client): - crate_id = "crate-123" - payload = { - "minio_bucket": "test_bucket", - "profile_name": "default", - "webhook_url": "https://webhook.example.com" - } - - with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: - mock_queue.return_value = ({"message": "Validation in progress"}, 202) - - response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) - - assert response.status_code == 202 - assert response.json == {"message": "Validation in progress"} - mock_queue.assert_called_once_with("test_bucket", "crate-123", None, "default", "https://webhook.example.com") - - # Test POST API: /v1/ro_crates/validate_metadata -def test_validate_metadata_with_all_fields(client: FlaskClient): - """ - If both profile name and crate json are provided then processing - of the RO-Crate metadata should continue, and return: - - 200 status code - - JSON data object which includes "status" which is "success" - """ - test_data = { - "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', - "profile_name": "default" - } - - with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: - mock_queue.return_value = ({"status": "success"}, 200) - - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - print(response.json) - mock_queue.assert_called_once_with( - test_data["crate_json"], - test_data["profile_name"] - ) - assert response.status_code == 200 - assert response.json == {"status": "success"} - - -def test_validate_metadata_without_profile_name(client: FlaskClient): - """ - If the profile name is not specified then processing - of the RO-Crate metadata should continue, and return: - - 200 status code - - JSON data object which includes "status" which is "success" - """ - test_data = { - "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}' - } - +@pytest.mark.parametrize( + "payload, status_code, response_json", + [ + ( + { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "profile_name": "default" + }, 200, {"status": "success"} + ), + ( + { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + }, 200, {"status": "success"} + ), + ], + ids=["success_with_all_fields", "success_without_profile_name"] +) +def test_validate_metadata_success(client: FlaskClient, payload: dict, status_code: int, response_json: dict): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: - mock_queue.return_value = ({"status": "success"}, 200) - - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - mock_queue.assert_called_once_with(test_data["crate_json"], None) - assert response.status_code == 200 - assert response.json == {"status": "success"} - + mock_queue.return_value = (response_json, status_code) -def test_validate_metadata_missing_crate_json(client: FlaskClient): - """ - If the RO-Crate is missing APIFlask should return: - - 422 status code - - Error message which includes 'Missing data for required field' - """ - test_data = { - "profile_name": "default" - } + response = client.post("/v1/ro_crates/validate_metadata", json=payload) - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - assert response.status_code == 422 - assert "Missing data for required field" in response.get_data(as_text=True) - - -def test_validate_metadata_emptystring_crate_json(client: FlaskClient): - """ - If the RO-Crate is missing APIFlask should return: - - 422 status code - - Error message which includes 'Missing required parameter' - """ - test_data = { - "crate_json": "", - "profile_name": "default" - } + crate_json = payload["crate_json"] if "crate_json" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - assert response.status_code == 422 - assert "Missing required parameter" in response.get_data(as_text=True) - - -def test_validate_metadata_malformed_crate_json(client: FlaskClient): - """ - If the RO-Crate is missing APIFlask should return: - - 422 status code - - Error message which includes 'not valid JSON' - """ - test_data = { - "crate_json": "{", - "profile_name": "default" - } + mock_queue.assert_called_once_with(crate_json, profile_name) + assert response.status_code == status_code + assert response.json == response_json - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - assert response.status_code == 422 - assert "not valid JSON" in response.get_data(as_text=True) - - -def test_validate_metadata_emptydict_crate_json(client: FlaskClient): - """ - If the RO-Crate is missing APIFlask should return: - - 422 status code - - Error message which includes 'Required parameter crate_json is empty' - """ - test_data = { - "crate_json": "{}", - "profile_name": "default" - } - response = client.post("/v1/ro_crates/validate_metadata", json=test_data) - assert response.status_code == 422 - assert "Required parameter crate_json is empty" in response.get_data(as_text=True) +@pytest.mark.parametrize( + "payload, status_code, response_text", + [ + ( + { + "profile_name": "default" + }, 422, "Missing data for required field" + ), + ( + { + "crate_json": '', + }, 422, "Missing required parameter" + ), + ( + { + "crate_json": '{', + }, 422, "not valid JSON" + ), + ( + { + "crate_json": '{}', + }, 422, "Required parameter crate_json is empty" + ), + ], + ids=["failure_missing_crate", "failure_empty_crate", + "failure_malformed_crate", "failure_empty_crate"] +) +def test_validate_metadata_failure(client: FlaskClient, payload: dict, status_code: int, response_text: str): + response = client.post("/v1/ro_crates/validate_metadata", json=payload) + assert response.status_code == status_code + assert response_text in response.get_data(as_text=True) # Test GET API: /v1/ro_crates/{crate_id}/validation From 4f6156764597e0ff8ea06a330bc49750f601977b Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 31 Jul 2025 22:23:38 +0100 Subject: [PATCH 101/127] parameterise get_validation_by_id failures --- tests/test_api_routes.py | 44 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 661e7c1..e44fe7e 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -166,6 +166,28 @@ def test_validate_metadata_failure(client: FlaskClient, payload: dict, status_co # Test GET API: /v1/ro_crates/{crate_id}/validation +@pytest.mark.parametrize( + "crate_id, payload, status_code", + [ + ( + "",{ + "minio_bucket": "test_bucket", + "root_path": "base_path" + }, 404 + ), + ( + "crate-123",{ + "root_path": "base_path" + }, 422 + ), + ], + ids=["failure_missing_crate_id", "failure_missing_minio_bucket"] +) +def test_get_validation_by_id_failures(client: FlaskClient, crate_id: str, payload: dict, status_code: int): + response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) + assert response.status_code == status_code + + def test_get_validation_by_id_success(client): crate_id = "crate-123" payload = { @@ -183,28 +205,6 @@ def test_get_validation_by_id_success(client): mock_get.assert_called_once_with("test_bucket", "crate-123", "base_path") -def test_get_validation_by_id_fails_missing_crate_id(client): - payload = { - "minio_bucket": "test_bucket", - "root_path": "base_path" - } - - response = client.get("/v1/ro_crates//validation", json=payload) - - assert response.status_code == 404 - - -def test_get_validation_by_id_fails_missing_minio_bucket(client): - crate_id = "crate-123" - payload = { - "root_path": "base_path" - } - - response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) - - assert response.status_code == 422 - - def test_get_validation_by_id_missing_root_path(client): crate_id = "crate-123" payload = { From 980279003a93522c128ec87ee36e68a7b868d713 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 31 Jul 2025 23:46:31 +0100 Subject: [PATCH 102/127] parameterized service tests --- tests/test_services.py | 322 ++++++++++++++++++++++------------------- 1 file changed, 173 insertions(+), 149 deletions(-) diff --git a/tests/test_services.py b/tests/test_services.py index 068e487..f5838e5 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1,6 +1,7 @@ import pytest from unittest.mock import patch, MagicMock from flask import Flask +from flask.testing import FlaskClient from app.services.validation_service import ( queue_ro_crate_validation_task, @@ -20,167 +21,190 @@ def flask_app(): # Test function: queue_ro_crate_validation_task -@patch("app.services.validation_service.process_validation_task_by_id.delay") -@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) -def test_queue_task_success( - mock_exists, - mock_delay, - flask_app +@pytest.mark.parametrize( + "crate_id, rocrate_exists, delay_side_effects, payload, status_code, response_dict", + [ + ( + "crate123", True, None, + { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), + ( + "crate123", True, Exception("Celery down"), + { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 500, {"error": "Celery down"} + ), + ], + ids=["successful_queue", "celery_server_down"] +) +def test_queue_ro_crate_validation_task( + flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, + delay_side_effects: Exception, payload: dict, status_code: int, response_dict: dict ): - response, status_code = queue_ro_crate_validation_task("test_bucket", "crate123", "base_path", "profileA", "http://webhook.com") - - mock_exists.assert_called_once_with("test_bucket", "crate123", "base_path") - mock_delay.assert_called_once_with("test_bucket", "crate123", "base_path", "profileA", "http://webhook.com") - assert status_code == 202 - assert response.json == {"message": "Validation in progress"} - - -@patch("app.services.validation_service.process_validation_task_by_id.delay") -@patch("app.services.validation_service.check_ro_crate_exists", return_value=False) -def test_queue_ro_crate_missing_exception( - mock_exists, - mock_delay, - flask_app + minio_bucket = payload["minio_bucket"] if "minio_bucket" in payload else None + root_path = payload["root_path"] if "root_path" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None + webhook_url = payload["webhook_url"] if "webhook_url" in payload else None + + with patch("app.services.validation_service.process_validation_task_by_id.delay", side_effect=delay_side_effects) as mock_delay: + with patch("app.services.validation_service.check_ro_crate_exists", return_value=rocrate_exists) as mock_exists: + + response, status_code = queue_ro_crate_validation_task(minio_bucket, crate_id, root_path, profile_name, webhook_url) + + mock_exists.assert_called_once_with(minio_bucket, crate_id, root_path) + mock_delay.assert_called_once_with(minio_bucket, crate_id, root_path, profile_name, webhook_url) + assert status_code == status_code + assert response.json == response_dict + + +@pytest.mark.parametrize( + "crate_id, rocrate_exists, payload, iau_message", + [ + ( + "crate12z", False, + { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, "No RO-Crate with prefix: crate12z" + ), + ], + ids=["no_rocrate_exists"] +) +def test_queue_ro_crate_validation_task_failure( + flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, + payload: dict, iau_message: str ): - with pytest.raises(InvalidAPIUsage) as exc_info: - queue_ro_crate_validation_task("test_bucket", "crate12z", "base_path", "profileA", "http://webhook.com") + minio_bucket = payload["minio_bucket"] if "minio_bucket" in payload else None + root_path = payload["root_path"] if "root_path" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None + webhook_url = payload["webhook_url"] if "webhook_url" in payload else None - assert "No RO-Crate with prefix: crate12z" in str(exc_info.value.message) - mock_exists.assert_called_once_with("test_bucket", "crate12z", "base_path") - mock_delay.assert_not_called() + with patch("app.services.validation_service.process_validation_task_by_id.delay") as mock_delay: + with patch("app.services.validation_service.check_ro_crate_exists", return_value=rocrate_exists) as mock_exists: + with pytest.raises(InvalidAPIUsage) as exc_info: + queue_ro_crate_validation_task(minio_bucket, crate_id, root_path, profile_name, webhook_url) -@patch("app.services.validation_service.process_validation_task_by_id.delay", side_effect=Exception("Celery down")) -@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) -def test_queue_task_exception( - mock_exists, - mock_delay, - flask_app -): - response, status_code = queue_ro_crate_validation_task("test_bucket", "crate123", None) - - mock_exists.assert_called_once_with("test_bucket", "crate123", None) - assert status_code == 500 - assert response.json == {"error": "Celery down"} + assert iau_message in str(exc_info.value.message) + mock_exists.assert_called_once_with(minio_bucket, crate_id, root_path) + mock_delay.assert_not_called() # Test function: queue_ro_crate_metadata_validation_task -def test_queue_metadata_with_webhook(flask_app): - with patch("app.services.validation_service.process_validation_task_by_metadata.delay") as mock_delay: - mock_result = MagicMock() - mock_delay.return_value = mock_result - - crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context"}' - response, status = queue_ro_crate_metadata_validation_task(crate_json, "profile", "http://webhook") - - mock_delay.assert_called_once_with(crate_json, "profile", "http://webhook") - assert status == 202 - assert response.json == {"message": "Validation in progress"} - - -def test_queue_metadata_without_webhook(flask_app): - with patch("app.services.validation_service.process_validation_task_by_metadata.delay") as mock_delay: +@pytest.mark.parametrize( + "crate_json, profile, webhook, status_code, return_value, response_json, delay_side_effect", + [ + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "default", "http://webhook", + 202, None, {"message": "Validation in progress"}, + None + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "default", None, + 200, {"status": "ok"}, {"result": {"status": "ok"}}, + None + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "default", "http://webhook", + 500, None, {"error": "Celery error"}, + Exception("Celery error") + ), + ], + ids=["success_with_webhook", "success_without_webhook", "failure_celery_error"] +) +def test_queue_metadata(flask_app, crate_json: dict, profile: str, webhook: str, + status_code: int, return_value: dict, response_json: dict, + delay_side_effect: Exception): + with patch("app.services.validation_service.process_validation_task_by_metadata.delay", + side_effect=delay_side_effect) as mock_delay: mock_result = MagicMock() - mock_result.get.return_value = {"status": "ok"} - mock_delay.return_value = mock_result - - crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context"}' - response, status = queue_ro_crate_metadata_validation_task(crate_json, "profile", None) - - mock_delay.assert_called_once_with(crate_json, "profile", None) - assert status == 200 - assert response.json == {"result": {"status": "ok"}} - - -def test_queue_metadata_missing_json(flask_app): - response, status = queue_ro_crate_metadata_validation_task(None) - - assert status == 422 - assert response.json == {"error": "Missing required parameter: crate_json"} - - -def test_queue_metadata_invalid_json(flask_app): - - crate_json = '{' - response, status = queue_ro_crate_metadata_validation_task(crate_json) - - assert status == 422 - assert 'not valid JSON' in response.json['error'] - - -def test_queue_metadata_empty_json(flask_app): - - crate_json = '{}' + if return_value is not None: + mock_result.get.return_value = return_value + if delay_side_effect is None: + mock_delay.return_value = mock_result + + response, status = queue_ro_crate_metadata_validation_task(crate_json, profile, webhook) + + mock_delay.assert_called_once_with(crate_json, profile, webhook) + assert status == status_code + assert response.json == response_json + + +@pytest.mark.parametrize( + "crate_json, status_code, response_error", + [ + ( + None, + 422, "Missing required parameter: crate_json" + ), + ( + "{", + 422, "not valid JSON" + ), + ( + "{}", + 422, "Required parameter crate_json is empty" + ), + ] +) +def test_queue_metadata_json_errors(flask_app, crate_json: str, status_code: int, response_error: str): response, status = queue_ro_crate_metadata_validation_task(crate_json) - - assert status == 422 - assert response.json == {"error": "Required parameter crate_json is empty"} - - -def test_queue_metadata_exception(flask_app): - with patch("app.services.validation_service.process_validation_task_by_metadata.delay", - side_effect=Exception("Celery error")): - crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context"}' - response, status = queue_ro_crate_metadata_validation_task(crate_json) - - assert status == 500 - assert response.json == {"error": "Celery error"} + assert status == status_code + assert response_error in response.json["error"] # Test function: get_ro_crate_validation_task -@patch("app.services.validation_service.return_ro_crate_validation", return_value={"status": "valid"}) -@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) -@patch("app.services.validation_service.check_validation_exists", return_value=True) -def test_get_validation_success( - mock_validation, - mock_rocrate, - mock_return, - flask_app -): - response, status = get_ro_crate_validation_task("test_bucket", "crate123", "base_path") - - mock_return.assert_called_once_with("test_bucket", "crate123", "base_path") - mock_rocrate.assert_called_once_with("test_bucket", "crate123", "base_path") - mock_validation.assert_called_once_with("test_bucket", "crate123", "base_path") - assert status == 200 - assert response == {"status": "valid"} - - -@patch("app.services.validation_service.return_ro_crate_validation", return_value=None) -@patch("app.services.validation_service.check_ro_crate_exists", return_value=False) -@patch("app.services.validation_service.check_validation_exists", return_value=False) -def test_get_validation_missing_ro_crate( - mock_validation, - mock_rocrate, - mock_return, - flask_app -): - with pytest.raises(InvalidAPIUsage) as exc_info: - get_ro_crate_validation_task("test_bucket", "crate123", "base_path") - - assert exc_info.value.status_code == 400 - assert "No RO-Crate with prefix: crate123" in str(exc_info.value.message) - mock_rocrate.assert_called_once_with("test_bucket", "crate123", "base_path") - mock_validation.assert_not_called() - mock_return.assert_not_called() - - -@patch("app.services.validation_service.return_ro_crate_validation", return_value=None) -@patch("app.services.validation_service.check_ro_crate_exists", return_value=True) -@patch("app.services.validation_service.check_validation_exists", return_value=False) -def test_get_validation_missing_validation( - mock_validation, - mock_rocrate, - mock_return, - flask_app -): - with pytest.raises(InvalidAPIUsage) as exc_info: - get_ro_crate_validation_task("test_bucket", "crate123", "base_path") - - assert exc_info.value.status_code == 400 - assert "No validation result yet for RO-Crate: crate123" in str(exc_info.value.message) - mock_rocrate.assert_called_once_with("test_bucket", "crate123", "base_path") - mock_validation.assert_called_once_with("test_bucket", "crate123", "base_path") - mock_return.assert_not_called() +@pytest.mark.parametrize( + "crate_id, crate_exists, validation_exists, validation_value, status_code, error_message", + [ + ("crate123", True, True, {"status": "valid"}, 200, None), + ("crate123", False, False, None, 400, "No RO-Crate with prefix: crate123"), + ("crate123", True, False, None, 400, "No validation result yet for RO-Crate: crate123"), + ], + ids=["validation_exists", "rocrate_missing", "validation_missing"] +) +def test_get_validation(flask_app, crate_id: str, crate_exists: bool, + validation_exists: bool, validation_value: dict, + status_code: int, error_message: str): + with patch("app.services.validation_service.check_ro_crate_exists", return_value=crate_exists) as mock_rocrate: + with patch("app.services.validation_service.check_validation_exists", return_value=validation_exists) as mock_validation: + with patch("app.services.validation_service.return_ro_crate_validation", return_value=validation_value) as mock_return: + + if crate_exists and validation_exists: + response, status = get_ro_crate_validation_task("test_bucket", crate_id, "base_path") + + mock_return.assert_called_once_with("test_bucket", crate_id, "base_path") + mock_rocrate.assert_called_once_with("test_bucket", crate_id, "base_path") + mock_validation.assert_called_once_with("test_bucket", crate_id, "base_path") + + assert status == status_code + assert response == validation_value + + else: + with pytest.raises(InvalidAPIUsage) as exc_info: + get_ro_crate_validation_task("test_bucket", crate_id, "base_path") + + assert exc_info.value.status_code == status_code + assert error_message in str(exc_info.value.message) + + mock_rocrate.assert_called_once_with("test_bucket", crate_id, "base_path") + if crate_exists: + mock_validation.assert_called_once_with("test_bucket", crate_id, "base_path") + else: + mock_validation.assert_not_called() + mock_return.assert_not_called() From 87d59cf818863d875f555d8bf683fea8b6be53b8 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 1 Aug 2025 21:53:47 +0100 Subject: [PATCH 103/127] add todo notes to validation task --- app/tasks/validation_tasks.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index f8833de..a842400 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -41,9 +41,10 @@ def process_validation_task_by_id( :param webhook_url: The webhook URL to send notifications to. Defaults to None. :raises Exception: If an error occurs during the validation process. - :todo: Replace the Crate ID with a more comprehensive system, and replace profile name with URI. """ + # TODO: Split try statements: (1) fetch and validate; (2) write to minio; (3) webhook + file_path = None try: @@ -76,9 +77,12 @@ def process_validation_task_by_id( except Exception as e: logging.error(f"Error processing validation task: {e}") + # TODO: Should we write error messages to the minio instance too? + # Send failure notification via webhook - error_data = {"profile_name": profile_name, "error": str(e)} - send_webhook_notification(webhook_url, error_data) + if webhook_url: + error_data = {"profile_name": profile_name, "error": str(e)} + send_webhook_notification(webhook_url, error_data) finally: # Clean up the temporary file if it was created: From 9a0d2697e34898acc4ee09f35a038a715903499f Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 1 Aug 2025 22:01:31 +0100 Subject: [PATCH 104/127] parametrize rocrate validation task tests --- tests/test_validation_tasks.py | 332 ++++++++++++++++----------------- 1 file changed, 159 insertions(+), 173 deletions(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 51b8063..aa8c5e0 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -15,47 +15,28 @@ # Test function: process_validation_task_by_id -@mock.patch("app.tasks.validation_tasks.os.remove") -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) -@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=True) -@mock.patch("app.tasks.validation_tasks.send_webhook_notification") -@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") -@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") -@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") -def test_process_validation_zipfile_success( - mock_fetch, - mock_validate, - mock_update, - mock_webhook, - mock_isfile, - mock_exists, - mock_remove -): - mock_fetch.return_value = "/tmp/crate.zip" - - mock_validation_result = mock.Mock() - mock_validation_result.has_issues.return_value = False - mock_validation_result.to_json.return_value = '{"status": "valid"}' - mock_validate.return_value = mock_validation_result - - process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") - - mock_fetch.assert_called_once_with("test_bucket", "crate123", "") - mock_validate.assert_called_once_with("/tmp/crate.zip", "profileA") - mock_update.assert_called_once_with("test_bucket", "crate123", "", '{"status": "valid"}') - mock_webhook.assert_called_once_with("https://example.com/hook", '{"status": "valid"}') - mock_remove.assert_called_once_with("/tmp/crate.zip") - - +@pytest.mark.parametrize( + "crate_id, os_path_exists, os_path_isfile, os_path_isdir, return_value, webhook, profile, val_success, val_result", + [ + ("crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", True, '{"status": "valid"}'), + ("crate123", True, False, True, "/tmp/crate123", + "https://example.com/hook", "profileA", True, '{"status": "valid"}'), + ("crate123", True, False, True, "/tmp/crate123", + None, "profileA", True, '{"status": "valid"}'), + ], + ids=["successful_validation_zip", "successful_validation_dir", "successful_validation_nowebhook"] +) @mock.patch("app.tasks.validation_tasks.shutil.rmtree") -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) -@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=False) -@mock.patch("app.tasks.validation_tasks.os.path.isdir", return_value=True) +@mock.patch("app.tasks.validation_tasks.os.remove") +@mock.patch("app.tasks.validation_tasks.os.path.exists") +@mock.patch("app.tasks.validation_tasks.os.path.isfile") +@mock.patch("app.tasks.validation_tasks.os.path.isdir") @mock.patch("app.tasks.validation_tasks.send_webhook_notification") @mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") @mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") @mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") -def test_process_validation_directory_success( +def test_process_validation( mock_fetch, mock_validate, mock_update, @@ -63,202 +44,207 @@ def test_process_validation_directory_success( mock_isdir, mock_isfile, mock_exists, - mock_rmtree + mock_remove, + mock_rmtree, + crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, + return_value: str, webhook: str, profile: str, val_success: bool, val_result: str ): - mock_fetch.return_value = "/tmp/crate123/" + mock_exists.return_value = os_path_exists + mock_isfile.return_value = os_path_isfile + mock_isdir.return_value = os_path_isdir + mock_fetch.return_value = return_value mock_validation_result = mock.Mock() - mock_validation_result.has_issues.return_value = False - mock_validation_result.to_json.return_value = '{"status": "valid"}' + mock_validation_result.has_issues.return_value = val_success + mock_validation_result.to_json.return_value = val_result mock_validate.return_value = mock_validation_result - process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") - - mock_fetch.assert_called_once_with("test_bucket", "crate123", "") - mock_validate.assert_called_once_with("/tmp/crate123/", "profileA") - mock_update.assert_called_once_with("test_bucket", "crate123", "", '{"status": "valid"}') - mock_webhook.assert_called_once_with("https://example.com/hook", '{"status": "valid"}') - mock_rmtree.assert_called_once_with("/tmp/crate123/") - - + process_validation_task_by_id("test_bucket", crate_id, "", profile, webhook) + + mock_fetch.assert_called_once_with("test_bucket", crate_id, "") + mock_validate.assert_called_once_with(return_value, profile) + mock_update.assert_called_once_with("test_bucket", crate_id, "", val_result) + if webhook is not None: + mock_webhook.assert_called_once_with(webhook, val_result) + else: + mock_webhook.assert_not_called() + if os_path_exists and os_path_isfile: + mock_remove.assert_called_once_with(return_value) + mock_rmtree.assert_not_called() + elif os_path_exists and os_path_isdir: + mock_rmtree.assert_called_once_with(return_value) + mock_remove.assert_not_called() + + +@pytest.mark.parametrize( + "crate_id, os_path_exists, os_path_isfile, os_path_isdir, return_fetch, " + + "webhook, profile, return_validate, validate_side_effect, fetch_side_effect", + [ + ("crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", "Validation failed", None, None), + ("crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", None, Exception("Unexpected error"), None), + ("crate123", False, False, False, None, + "https://example.com/hook", "profileA", None, None, Exception("MinIO fetch failed")), + ], + ids=["validation_fails_with_message", "validation_fails_with_validation_exception", + "validation_fails_with_fetch_exception"] +) +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") @mock.patch("app.tasks.validation_tasks.os.remove") -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) -@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=True) +@mock.patch("app.tasks.validation_tasks.os.path.exists") +@mock.patch("app.tasks.validation_tasks.os.path.isfile") +@mock.patch("app.tasks.validation_tasks.os.path.isdir") @mock.patch("app.tasks.validation_tasks.send_webhook_notification") @mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") @mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") @mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") -def test_process_validation_fails_with_message( - mock_fetch, - mock_validate, - mock_update, - mock_webhook, - mock_isfile, - mock_exists, - mock_remove -): - mock_fetch.return_value = "/tmp/crate.zip" - mock_validate.return_value = "Validation failed" - - process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") - - mock_update.assert_not_called() - mock_webhook.assert_called_once() - args, kwargs = mock_webhook.call_args - assert args[0] == "https://example.com/hook" - assert "Validation failed" in args[1]["error"] - mock_remove.assert_called_once_with("/tmp/crate.zip") - - -@mock.patch("app.tasks.validation_tasks.os.remove") -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) -@mock.patch("app.tasks.validation_tasks.os.path.isfile", return_value=True) -@mock.patch("app.tasks.validation_tasks.send_webhook_notification") -@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") -@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation", side_effect=Exception("Unexpected error")) -@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") -def test_process_validation_exception( +def test_process_validation_failure( mock_fetch, mock_validate, mock_update, mock_webhook, + mock_isdir, mock_isfile, mock_exists, - mock_remove + mock_remove, + mock_rmtree, + crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, + return_fetch: str, webhook: str, profile: str, return_validate: str, + validate_side_effect: Exception, fetch_side_effect: Exception ): - mock_fetch.return_value = "/tmp/crate.zip" + mock_exists.return_value = os_path_exists + mock_isfile.return_value = os_path_isfile + mock_isdir.return_value = os_path_isdir - process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") + if fetch_side_effect is None: + mock_fetch.return_value = return_fetch + else: + mock_fetch.side_effect = fetch_side_effect - mock_update.assert_not_called() - mock_webhook.assert_called_once() - args, kwargs = mock_webhook.call_args - assert args[0] == "https://example.com/hook" - assert "Unexpected error" in args[1]["error"] - mock_remove.assert_called_once_with("/tmp/crate.zip") + if validate_side_effect is None: + mock_validate.return_value = return_validate + else: + mock_validate.side_effect = validate_side_effect + process_validation_task_by_id("test_bucket", crate_id, "", profile, webhook) -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=False) -@mock.patch("app.tasks.validation_tasks.send_webhook_notification") -@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") -@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") -@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio", side_effect=Exception("MinIO fetch failed")) -def test_process_validation_fetch_error( - mock_fetch, - mock_validate, - mock_update, - mock_webhook, - mock_exists -): - process_validation_task_by_id("test_bucket", "crate123", "", "profileA", "https://example.com/hook") + if fetch_side_effect is None: + mock_validate.assert_called_once_with(return_fetch, profile) + else: + mock_validate.assert_not_called() - mock_validate.assert_not_called() mock_update.assert_not_called() mock_webhook.assert_called_once() args, kwargs = mock_webhook.call_args - assert args[0] == "https://example.com/hook" - assert "MinIO fetch failed" in args[1]["error"] + assert args[0] == webhook + if fetch_side_effect is not None: + assert fetch_side_effect.args[0] in args[1]["error"] + elif validate_side_effect is not None: + assert validate_side_effect.args[0] in args[1]["error"] + else: + assert return_validate in args[1]["error"] + + if not os_path_exists: + mock_remove.assert_not_called() + mock_rmtree.assert_not_called() + elif os_path_exists and os_path_isfile: + mock_remove.assert_called_once_with(return_fetch) + mock_rmtree.assert_not_called() + elif os_path_exists and os_path_isdir: + mock_rmtree.assert_called_once_with(return_fetch) + mock_remove.assert_not_called() # Test function: process_validation_task_by_metadata +@pytest.mark.parametrize( + "crate_json, profile_name, webhook_url, mock_path, validation_json, validation_value, os_path_exists", + [ + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", "https://example.com/webhook", "/tmp/crate", + '{"status": "valid"}', False, True + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", "https://example.com/webhook", "/tmp/crate", + '{"status": "invalid"}', True, True + ) + ], + ids=["success_no_issues", "success_with_issues"] +) @mock.patch("app.tasks.validation_tasks.shutil.rmtree") -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.os.path.exists") @mock.patch("app.tasks.validation_tasks.send_webhook_notification") @mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") @mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") -def test_validation_success_no_issues( - mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree +def test_metadata_validation( + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree, + crate_json: str, profile_name: str, webhook_url: str, mock_path: str, + validation_json: str, validation_value: bool, os_path_exists: bool ): - crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' - profile_name = "test-profile" - webhook_url = "https://example.com/webhook" - - mock_path = "/tmp/crate" + mock_exists.return_value = os_path_exists mock_build.return_value = mock_path mock_result = mock.Mock() - mock_result.has_issues.return_value = False - mock_result.to_json.return_value = '{"status": "valid"}' + mock_result.has_issues.return_value = validation_value + mock_result.to_json.return_value = validation_json mock_validate.return_value = mock_result result = process_validation_task_by_metadata(crate_json, profile_name, webhook_url) - assert result == '{"status": "valid"}' + assert result == validation_json mock_build.assert_called_once_with(crate_json) mock_validate.assert_called_once() - mock_webhook.assert_called_once_with(webhook_url, '{"status": "valid"}') + mock_webhook.assert_called_once_with(webhook_url, validation_json) mock_rmtree.assert_called_once_with(mock_path) +@pytest.mark.parametrize( + "crate_json, profile_name, webhook_url, mock_path, validation_message, os_path_exists", + [ + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", "https://example.com/webhook", "/tmp/crate", + "Validation error", True + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", None, "/tmp/crate", + "Validation error", True + ) + ], + ids=["validation_fails", "validation_fails_no_webhook"] +) @mock.patch("app.tasks.validation_tasks.shutil.rmtree") @mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) @mock.patch("app.tasks.validation_tasks.send_webhook_notification") @mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") @mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") -def test_validation_success_with_issues( - mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree -): - crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' - mock_path = "/tmp/crate" - mock_build.return_value = mock_path - - mock_result = mock.Mock() - mock_result.has_issues.return_value = True - mock_result.to_json.return_value = '{"status": "invalid"}' - mock_validate.return_value = mock_result - - result = process_validation_task_by_metadata(crate_json, None, None) - - assert result == '{"status": "invalid"}' - mock_webhook.assert_not_called() - mock_rmtree.assert_called_once_with(mock_path) - - -@mock.patch("app.tasks.validation_tasks.shutil.rmtree") -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) -@mock.patch("app.tasks.validation_tasks.send_webhook_notification") -@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation", return_value="Validation error") -@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") def test_validation_fails_and_sends_error_notification_to_webhook( - mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree, + crate_json: str, profile_name: str, webhook_url: str, mock_path: str, + validation_message: str, os_path_exists: bool ): - crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' - mock_path = "/tmp/crate" mock_build.return_value = mock_path - result = process_validation_task_by_metadata(crate_json, "profileX", "https://webhook.test") - - assert isinstance(result, str) - assert "Validation error" in result - - # Error webhook should be sent - mock_webhook.assert_called_once() - args, kwargs = mock_webhook.call_args - assert kwargs is None or "error" in args[1] - - mock_rmtree.assert_called_once_with(mock_path) - - -@mock.patch("app.tasks.validation_tasks.shutil.rmtree") -@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) -@mock.patch("app.tasks.validation_tasks.send_webhook_notification") -@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation", return_value="Validation error") -@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") -def test_validation_fails_with_no_webhook( - mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree -): - crate_json = '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}' - mock_path = "/tmp/crate" - mock_build.return_value = mock_path + mock_validate.return_value = validation_message - result = process_validation_task_by_metadata(crate_json, "profileX", None) + result = process_validation_task_by_metadata(crate_json, profile_name, webhook_url) assert isinstance(result, str) - assert "Validation error" in result + assert validation_message in result + + if webhook_url is not None: + # Error webhook should be sent + mock_webhook.assert_called_once() + args, kwargs = mock_webhook.call_args + assert kwargs is None or "error" in args[1] + else: + # Make sure webhook not sent + mock_webhook.assert_not_called() - # Make sure webhook not sent - mock_webhook.assert_not_called() mock_rmtree.assert_called_once_with(mock_path) From 60ea1931e4c7ac2d52a751fae010bfaa7bad5a64 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Fri, 1 Aug 2025 22:27:11 +0100 Subject: [PATCH 105/127] parameterise existence task tests --- tests/test_validation_tasks.py | 115 ++++++++++++++++----------------- 1 file changed, 56 insertions(+), 59 deletions(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index aa8c5e0..2ee1a0a 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -250,16 +250,23 @@ def test_validation_fails_and_sends_error_notification_to_webhook( # Test function: perform_ro_crate_validation +@pytest.mark.parametrize( + "file_path, profile_name, skip_checks", + [ + ("crates/test_crate", "ro_profile", ["check1", "check2"]), + ("crates/test_crate", None, None) + ], + ids=["success_with_all_args", "success_with_only_crate"] +) @mock.patch("app.tasks.validation_tasks.services.validate") @mock.patch("app.tasks.validation_tasks.services.ValidationSettings") -def test_validation_success_with_all_args(mock_validation_settings, mock_validate): +def test_validation_success_with_all_args( + mock_validation_settings, mock_validate, + file_path: str, profile_name: str, skip_checks: list +): mock_result = mock.Mock() mock_validate.return_value = mock_result - file_path = "crates/test_crate" - profile_name = "ro_profile" - skip_checks = ["check1", "check2"] - result = perform_ro_crate_validation(file_path, profile_name, skip_checks) # Assert that result was returned @@ -269,28 +276,18 @@ def test_validation_success_with_all_args(mock_validation_settings, mock_validat mock_validation_settings.assert_called_once() args, kwargs = mock_validation_settings.call_args assert kwargs["rocrate_uri"].endswith(file_path) - assert kwargs["profile_identifier"] == profile_name - assert kwargs["skip_checks"] == skip_checks + if profile_name is not None: + assert kwargs["profile_identifier"] == profile_name + else: + assert "profile_identifier" not in kwargs + if skip_checks is not None: + assert kwargs["skip_checks"] == skip_checks + else: + assert "skip_checks" not in kwargs mock_validate.assert_called_once_with(mock_validation_settings.return_value) -@mock.patch("app.tasks.validation_tasks.services.validate") -@mock.patch("app.tasks.validation_tasks.services.ValidationSettings") -def test_validation_success_with_minimal_args(mock_validation_settings, mock_validate): - mock_result = mock.Mock() - mock_validate.return_value = mock_result - - file_path = "crates/test_crate" - result = perform_ro_crate_validation(file_path, None) - - assert result == mock_result - - args, kwargs = mock_validation_settings.call_args - assert "profile_identifier" not in kwargs - assert "skip_checks" not in kwargs - - @mock.patch("app.tasks.validation_tasks.services.validate", side_effect=RuntimeError("Validation error")) @mock.patch("app.tasks.validation_tasks.services.ValidationSettings") def test_validation_raises_exception_and_returns_string(mock_validation_settings, mock_validate): @@ -351,55 +348,55 @@ def test_return_validation_raises_error(mock_get_status): # Test function: check_ro_crate_exists -@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") -@mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio", return_value="crate123") +@pytest.mark.parametrize( + "bucket, crate_id, base_path, client_return, ro_object_return, rocrate_exists", + [ + ("test_bucket", "crate123", "base_path", "mock_client", "crate123", True), + ("test_bucket", "crate12z", "base_path", "mock_client", False, False) + ], + ids=["rocrate_exists", "rocrate_does_not_exist"] +) +@mock.patch("app.tasks.validation_tasks.get_minio_client") +@mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio") def test_ro_crate_exists( mock_find_rocrate, - mock_get_client + mock_get_client, + bucket: str, crate_id: str, base_path: str, client_return: str, + ro_object_return: str, rocrate_exists: bool ): - result = check_ro_crate_exists("test_bucket", "crate123", "base_path") - - mock_get_client.assert_called_once() - mock_find_rocrate.assert_called_once_with("crate123", "mock_client", "test_bucket", "base_path") - assert result is True + mock_get_client.return_value = client_return + mock_find_rocrate.return_value = ro_object_return - -@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") -@mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio", return_value=False) -def test_ro_crate_does_not_exist( - mock_find_rocrate, - mock_get_client -): - result = check_ro_crate_exists("test_bucket", "crate12z", "base_path") + result = check_ro_crate_exists(bucket, crate_id, base_path) mock_get_client.assert_called_once() - mock_find_rocrate.assert_called_once_with("crate12z", "mock_client", "test_bucket", "base_path") - assert result is False + mock_find_rocrate.assert_called_once_with(crate_id, client_return, bucket, base_path) + assert result is rocrate_exists # Test function: check_validation_exists -@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") -@mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio", return_value="crate123") +@pytest.mark.parametrize( + "bucket, crate_id, base_path, client_return, val_object_return, validate_exists", + [ + ("test_bucket", "crate123", "base_path", "mock_client", "crate123", True), + ("test_bucket", "crate12z", "base_path", "mock_client", False, False) + ], + ids=["validation_exists", "validation_does_not_exist"] +) +@mock.patch("app.tasks.validation_tasks.get_minio_client") +@mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio") def test_validation_exists( mock_find_validation, - mock_get_client + mock_get_client, + bucket: str, crate_id: str, base_path: str, client_return: str, + val_object_return: str, validate_exists: bool ): - result = check_validation_exists("test_bucket", "crate123", "base_path") + mock_get_client.return_value = client_return + mock_find_validation.return_value = val_object_return - mock_get_client.assert_called_once() - mock_find_validation.assert_called_once_with("crate123", "mock_client", "test_bucket", "base_path") - assert result is True - - -@mock.patch("app.tasks.validation_tasks.get_minio_client", return_value="mock_client") -@mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio", return_value=False) -def test_validation_does_not_exist( - mock_find_validation, - mock_get_client -): - result = check_validation_exists("test_bucket", "crate12z", "base_path") + result = check_validation_exists(bucket, crate_id, base_path) mock_get_client.assert_called_once() - mock_find_validation.assert_called_once_with("crate12z", "mock_client", "test_bucket", "base_path") - assert result is False + mock_find_validation.assert_called_once_with(crate_id, client_return, bucket, base_path) + assert result is validate_exists From 2edf60be52cb4181df19e22a695c25831ed4f069 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 21:57:05 +0100 Subject: [PATCH 106/127] clean up minio tests --- tests/test_minio.py | 423 +++++++++++++++++++++----------------------- 1 file changed, 204 insertions(+), 219 deletions(-) diff --git a/tests/test_minio.py b/tests/test_minio.py index 02cde5a..57c8c59 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -57,73 +57,76 @@ def test_get_minio_object_list_success(): mock_response.close.assert_called_once() -def test_get_minio_object_list_s3error(): +@pytest.mark.parametrize( + "bucket, path, status_code, list_side_effect, error_check", + [ + ( + "my-bucket", "path/rocrate.zip", 500, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error" + ), + ( + "my-bucket", "path/rocrate.zip", 500, + ValueError("Missing config"), + "Configuration Error" + ), + ( + "my-bucket", "path/rocrate.zip", 500, + RuntimeError("Something went wrong"), + "Unknown Error" + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) +def test_get_minio_object_list_errors(bucket: str, path: str, status_code: int, list_side_effect, error_check: str): mock_minio_client = MagicMock() - mock_minio_client.list_objects.side_effect = S3Error( - code="S3 error", - message=None, - resource=None, - request_id=None, - host_id=None, - response=None - ) + mock_minio_client.list_objects.side_effect = list_side_effect from app.utils.minio_utils import get_minio_object_list, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_minio_object_list("path/", mock_minio_client, "my-bucket") + get_minio_object_list(path, mock_minio_client, bucket) - assert exc.value.status_code == 500 - assert "MinIO S3 Error" in str(exc.value.message) - - -def test_get_minio_object_list_value_error(): - mock_minio_client = MagicMock() - mock_minio_client.list_objects.side_effect = ValueError("Missing config") - - from app.utils.minio_utils import get_minio_object_list, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - get_minio_object_list("path/", mock_minio_client, "my-bucket") - - assert exc.value.status_code == 500 - assert "Configuration Error" in str(exc.value.message) - - -def test_get_minio_object_list_unexpected_error(): - mock_minio_client = MagicMock() - mock_minio_client.list_objects.side_effect = RuntimeError("Something went wrong") - - from app.utils.minio_utils import get_minio_object_list, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - get_minio_object_list("path/", mock_minio_client, "my-bucket") - - assert exc.value.status_code == 500 - assert "Unknown Error" in str(exc.value.message) + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) # Testing function: find_rocrate_object_on_minio -@patch("app.utils.minio_utils.get_minio_object_list") -def test_rocrate_found_as_directory(mock_get_list): - # Simulate a directory object match - obj = DummyObject("my/path/rocrate123/", is_dir=True) - mock_get_list.return_value = [obj] - minio_client = MagicMock() - - from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", root_path="my/path") - assert result == obj - +@pytest.mark.parametrize( + "rocrate_object, crateid, bucket, root_path", + [ + ( + DummyObject("my/path/rocrate123/", is_dir=True), + "rocrate123", "bucket", "my/path" + ), + ( + DummyObject("my/path/rocrate123.zip"), + "rocrate123", "bucket", "my/path" + ), + ( + DummyObject("rocrate123.zip"), + "rocrate123", "bucket", None + ), + ], + ids=["rocrate_directory", "rocrate_zip", "rootpath_none"] +) @patch("app.utils.minio_utils.get_minio_object_list") -def test_rocrate_found_as_zip(mock_get_list): - # Simulate a zip object match - obj = DummyObject("rocrate123.zip") - mock_get_list.return_value = [obj] +def test_finding_rocrate_on_minio( + mock_get_list, + rocrate_object: DummyObject, crateid: str, bucket: str, root_path: str): + # Simulate a directory object match + mock_get_list.return_value = [rocrate_object] minio_client = MagicMock() from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", None) - assert result == obj + result = find_rocrate_object_on_minio(crateid, minio_client, bucket, root_path) + assert result == rocrate_object @patch("app.utils.minio_utils.get_minio_object_list") @@ -142,82 +145,62 @@ def test_rocrate_not_found(mock_get_list): assert not result -@patch("app.utils.minio_utils.get_minio_object_list") -def test_storage_path_none(mock_get_list): - # Ensures correct rocrate_path is used when storage_path is None - obj = DummyObject("rocrate456.zip") - mock_get_list.return_value = [obj] - minio_client = MagicMock() - - from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate456", minio_client, "bucket", None) - assert result == obj - - -@patch("app.utils.minio_utils.get_minio_object_list") -def test_storage_path_provided(mock_get_list): - # Ensures correct rocrate_path is used when storage_path is provided - obj = DummyObject("data/rocrate789/", is_dir=True) - mock_get_list.return_value = [obj] - minio_client = MagicMock() - - from app.utils.minio_utils import find_rocrate_object_on_minio - result = find_rocrate_object_on_minio("rocrate789", minio_client, "bucket", root_path="data") - assert result == obj - - # Testing function: find_validation_object_on_minio +@pytest.mark.parametrize( + "object_path, crateid, bucket, root_path", + [ + ( + "my/storage/rocrate123_validation/validation_status.txt", + "rocrate123", "bucket", "my/storage" + ), + ( + "rocrate123_validation/validation_status.txt", + "rocrate123", "bucket", None + ), + ], + ids=["with_storage_path", "without_storage_path"] +) @patch("app.utils.minio_utils.get_minio_object_list") -def test_validation_object_found_with_storage_path(mock_get_list): +def test_validation_object_found_with_storage_path( + mock_get_list, + object_path: str, crateid: str, bucket: str, root_path: str): # Setup - expected_path = "my/storage/rocrate123_validation/validation_status.txt" - obj = DummyObject(expected_path) + obj = DummyObject(object_path) mock_get_list.return_value = [obj] from app.utils.minio_utils import find_validation_object_on_minio # Execute - result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket", root_path="my/storage") + result = find_validation_object_on_minio(crateid, MagicMock(), bucket, root_path) # Assert assert result == obj - mock_get_list.assert_called_once_with(expected_path, mock.ANY, "bucket") - - -@patch("app.utils.minio_utils.get_minio_object_list") -def test_validation_object_found_without_storage_path(mock_get_list): - # Setup - expected_path = "rocrate123_validation/validation_status.txt" - obj = DummyObject(expected_path) - mock_get_list.return_value = [obj] - - from app.utils.minio_utils import find_validation_object_on_minio - # Execute - result = find_validation_object_on_minio("rocrate123", MagicMock(), "bucket", None) - - # Assert - assert result == obj - mock_get_list.assert_called_once_with(expected_path, mock.ANY, "bucket") - - -@patch("app.utils.minio_utils.get_minio_object_list") -def test_validation_object_not_found(mock_get_list): - # Setup: object name does not match exactly - mock_get_list.return_value = [DummyObject("some/other/object.txt")] - - from app.utils.minio_utils import find_validation_object_on_minio - result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket", None) - - assert result is False - - + mock_get_list.assert_called_once_with(object_path, mock.ANY, bucket) + + +@pytest.mark.parametrize( + "object_list, crateid, bucket, root_path", + [ + ( + [DummyObject("some/other/object.txt")], + "rocrate999", "bucket", None + ), + ( + [], + "rocrate999", "bucket", None + ), + ], + ids=["other_objects", "empty_list"] +) @patch("app.utils.minio_utils.get_minio_object_list") -def test_validation_object_empty_list(mock_get_list): +def test_validation_object_not_found( + mock_get_list, + object_list: list, crateid: str, bucket: str, root_path: str): # Setup: no objects returned - mock_get_list.return_value = [] + mock_get_list.return_value = object_list from app.utils.minio_utils import find_validation_object_on_minio - result = find_validation_object_on_minio("rocrate999", MagicMock(), "bucket", None) + result = find_validation_object_on_minio(crateid, MagicMock(), bucket, root_path) assert result is False @@ -236,45 +219,47 @@ def test_download_success(mock_logging): mock_logging.error.assert_not_called() +@pytest.mark.parametrize( + "bucket, remotepath, localpath, status_code, get_side_effect, error_check", + [ + ( + "my-bucket", "remote/path.txt", "local/path.txt", 500, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error" + ), + ( + "my-bucket", "remote/path.txt", "local/path.txt", 500, + ValueError("Missing config"), + "Configuration Error" + ), + ( + "my-bucket", "remote/path.txt", "local/path.txt", 500, + RuntimeError("Something went wrong"), + "Unknown Error" + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) @patch("app.utils.minio_utils.logging") -def test_download_s3error(mock_logging): - mock_minio = MagicMock() - mock_minio.fget_object.side_effect = S3Error("S3 error", "", "", "", "", "") - - from app.utils.minio_utils import download_file_from_minio, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") - - assert exc.value.status_code == 500 - assert "MinIO S3 Error" in str(exc.value.message) - mock_logging.error.assert_called_once() - - -@patch("app.utils.minio_utils.logging") -def test_download_value_error(mock_logging): - mock_minio = MagicMock() - mock_minio.fget_object.side_effect = ValueError("Missing config") - - from app.utils.minio_utils import download_file_from_minio, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") - - assert exc.value.status_code == 500 - assert "Configuration Error" in str(exc.value.message) - mock_logging.error.assert_called_once() - - -@patch("app.utils.minio_utils.logging") -def test_download_unexpected_error(mock_logging): +def test_download_s3error( + mock_logging, + bucket: str, remotepath: str, localpath: str, status_code: int, + get_side_effect, error_check: str +): mock_minio = MagicMock() - mock_minio.fget_object.side_effect = RuntimeError("Something bad happened") + mock_minio.fget_object.side_effect = get_side_effect from app.utils.minio_utils import download_file_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") + download_file_from_minio(mock_minio, bucket, remotepath, localpath) - assert exc.value.status_code == 500 - assert "Unknown Error" in str(exc.value.message) + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) mock_logging.error.assert_called_once() @@ -293,48 +278,46 @@ def test_successful_retrieval(mocker, mock_minio_response): mock_minio_response.release_conn.assert_called_once() -def test_s3_error_raised(mocker): - mock_client = MagicMock() - mock_client.get_object.side_effect = S3Error( - code="S3 error", - message=None, - resource=None, - request_id=None, - host_id=None, - response=None - ) - mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) - - from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("test_bucket", "crate123", None) - - assert exc.value.status_code == 500 - assert "S3 Error" in str(exc.value.message) - - -def test_value_error_raised(mocker): - mocker.patch("app.utils.minio_utils.get_minio_client", side_effect=ValueError("Missing env var")) - - from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("test_bucket", "crate123", None) - - assert exc.value.status_code == 500 - assert "Configuration Error" in str(exc.value.message) - - -def test_generic_exception_raised(mocker): +@pytest.mark.parametrize( + "bucket, crateid, root_path, status_code, get_side_effect, error_check", + [ + ( + "my-bucket", "crate123", None, 500, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error" + ), + ( + "my-bucket", "crate123", None, 500, + ValueError("Missing env var"), + "Configuration Error" + ), + ( + "my-bucket", "crate123", None, 500, + RuntimeError("Unexpected failure"), + "Unknown Error" + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) +def test_get_validation_error_raised( + mocker, + bucket: str, crateid: str, root_path: str, status_code: int, get_side_effect, error_check: str +): mock_client = MagicMock() - mock_client.get_object.side_effect = Exception("Unexpected failure") + mock_client.get_object.side_effect = get_side_effect mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio("test_bucket", "crate123", None) + get_validation_status_from_minio(bucket, crateid, root_path) - assert exc.value.status_code == 500 - assert "Unknown Error" in str(exc.value.message) + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) # Testing function: update_validation_status_in_minio @@ -373,49 +356,51 @@ def test_update_validation_status_success(mock_get_client): assert kwargs["content_type"] == "application/json" +@pytest.mark.parametrize( + "bucket, crateid, root_path, validation_result, put_side_effect, error_check, status_code", + [ + ( + "my-bucket", "crate123", None, + {"status": "valid"}, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error", 500 + ), + ( + "my-bucket", "crate123", None, + {"status": "valid"}, + ValueError("Missing env vars"), + "Configuration Error", 500 + ), + ( + "my-bucket", "crate123", None, + {"status": "valid"}, + RuntimeError("Unexpected failure"), + "Unknown Error", 500 + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) @mock.patch("app.utils.minio_utils.get_minio_client") -def test_update_validation_status_s3_error(mock_get_client): - mock_minio_client = mock.Mock() - mock_get_client.return_value = mock_minio_client - mock_minio_client.put_object.side_effect = S3Error( - code="S3 error", - message=None, - resource=None, - request_id=None, - host_id=None, - response=None - ) - - from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("test_bucket", "crate123", "", json.dumps({"status": "valid"})) - - assert exc.value.status_code == 500 - assert "S3 Error" in str(exc.value.message) - - -@mock.patch("app.utils.minio_utils.get_minio_client", side_effect=ValueError("Missing env vars")) -def test_update_validation_status_value_error(mock_get_client): - from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage - with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("test_bucket", "crate123", "", json.dumps({"status": "valid"})) - - assert exc.value.status_code == 500 - assert "Configuration Error" in str(exc.value.message) - - -@mock.patch("app.utils.minio_utils.get_minio_client") -def test_update_validation_status_unexpected_error(mock_get_client): +def test_update_validation_status_erro( + mock_get_client, + bucket: str, crateid: str, root_path: str, validation_result: dict, + put_side_effect, error_check: str, status_code: int +): mock_minio_client = mock.Mock() mock_get_client.return_value = mock_minio_client - mock_minio_client.put_object.side_effect = RuntimeError("Unexpected failure") + mock_minio_client.put_object.side_effect = put_side_effect from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio("test_bucket", "crate123", "", json.dumps({"status": "valid"})) + update_validation_status_in_minio(bucket, crateid, root_path, json.dumps(validation_result)) - assert exc.value.status_code == 500 - assert "Unknown Error" in str(exc.value.message) + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) # Testing function: fetch_ro_crate_from_minio From 4efc8b37a483e6898e33893965268d842b5ae85b Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:07:55 +0100 Subject: [PATCH 107/127] get_minio_client now takes config dictionary --- app/utils/minio_utils.py | 19 +++++++++++-------- tests/test_minio.py | 27 +++++++++++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index 925af05..a6397ba 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -9,7 +9,6 @@ import os import tempfile -from dotenv import load_dotenv from io import BytesIO from minio import Minio, S3Error from app.utils.config import InvalidAPIUsage @@ -306,20 +305,24 @@ def get_minio_object_list(object_path: str, minio_client, minio_bucket: str, rec return object_list -def get_minio_client() -> Minio: +def get_minio_client(minio_config: dict) -> Minio: """ - Initialises the MinIO client from environment variables. + Initialises the MinIO client from provided settings. + :param minio_config: A dictionary containing the below parameters + :param endpoint: A string containing host and port. E.g. 'localhost:9000' + :param access_key: A string containing the access key / username + :param secret_key: A string containing the secret key / password + :param use_ssl: Boolean defining if SSL connection should be used or not :return: The MinIO client. :raises ValueError: If required environment variables are not set. """ - load_dotenv() minio_client = Minio( - endpoint=os.environ.get("MINIO_ENDPOINT"), - access_key=os.environ.get("MINIO_ROOT_USER"), - secret_key=os.environ.get("MINIO_ROOT_PASSWORD"), - secure=False, + endpoint=minio_config["endpoint"], + access_key=minio_config["accesskey"], + secret_key=minio_config["secret"], + secure=minio_config["ssl"], ) return minio_client diff --git a/tests/test_minio.py b/tests/test_minio.py index 57c8c59..1c15688 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -22,14 +22,29 @@ def __init__(self, name, is_dir=False): # Testing function: get_minio_client -def test_get_minio_client_success(monkeypatch): - # Set required env vars - monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000") - monkeypatch.setenv("MINIO_ROOT_USER", "admin") - monkeypatch.setenv("MINIO_ROOT_PASSWORD", "password123") +@pytest.mark.parametrize( + "minio_config", + [ + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False + }, + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "ignore_this" + } + ], + ids=["base_case", "ignore_extra_items"] +) +def test_get_minio_client_success(minio_config: dict): from app.utils.minio_utils import get_minio_client - client = get_minio_client() + client = get_minio_client(minio_config) assert isinstance(client, Minio) assert client._base_url.host == "localhost:9000" From 623f488d6b6ac1ab6f73ecb43a27343b900ca495 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:10:34 +0100 Subject: [PATCH 108/127] add minio config dictionary to validate rocrate post route --- app/ro_crates/routes/post_routes.py | 24 ++++++++++++--- tests/test_api_routes.py | 48 +++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 075eb62..c1ebcdb 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -5,7 +5,8 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester from apiflask import APIBlueprint, Schema -from apiflask.fields import String +from apiflask.fields import String, Boolean +from marshmallow.fields import Nested from flask import Response from app.services.validation_service import ( @@ -16,8 +17,16 @@ post_routes_bp = APIBlueprint("post_routes", __name__) +class MinioConfig(Schema): + endpoint = String(required=True) + accesskey = String(required=True) + secret = String(required=True) + ssl = Boolean(required=True) + bucket = String(required=True) + + class ValidateCrate(Schema): - minio_bucket = String(required=True) + minio_config = Nested(MinioConfig, required=True) root_path = String(required=False) profile_name = String(required=False) webhook_url = String(required=False) @@ -38,7 +47,12 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: - **crate_id**: The RO-Crate ID. _Required_. Request Body Parameters: - - **minio_bucket**: The MinIO bucket containing the RO-Crate. _Required_ + - **minio_config**: The MinIO bucket containing the RO-Crate. _Required_ + - **endpoint**: Endpoint, e.g. 'localhost:9000' + - **accesskey**: Access key / username + - **secret**: Secret / password + - **ssl**: Use SSL encryption? True/False + - **bucket**: The MinIO bucket to access - **root_path**: The root path containing the RO-Crate. _Optional_ - **profile_name**: The profile name for validation. _Optional_. - **webhook_url**: The webhook URL where validation results will be sent. _Optional_. @@ -50,7 +64,7 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id` or `webhook_url`) are missing. """ - minio_bucket = json_data["minio_bucket"] + minio_config = json_data["minio_config"] if "root_path" in json_data: root_path = json_data["root_path"] @@ -67,7 +81,7 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: else: profile_name = None - return queue_ro_crate_validation_task(minio_bucket, crate_id, root_path, profile_name, webhook_url) + return queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) @post_routes_bp.post("/validate_metadata") diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index e44fe7e..38c7b80 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -17,7 +17,13 @@ def client(): [ ( "crate-123", { - "minio_bucket": "test_bucket", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" @@ -25,28 +31,52 @@ def client(): ), ( "crate-123", { - "minio_bucket": "test_bucket", - "root_path": "base_path", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", "webhook_url": "https://webhook.example.com", }, 202, {"message": "Validation in progress"} ), ( "crate-123", { - "minio_bucket": "test_bucket", - "root_path": "base_path", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", "profile_name": "default" }, 202, {"message": "Validation in progress"} ), ( "crate-123", { - "minio_bucket": "test_bucket", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, "webhook_url": "https://webhook.example.com", "profile_name": "default" }, 202, {"message": "Validation in progress"} ), ( "crate-123", { - "minio_bucket": "test_bucket" + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, }, 202, {"message": "Validation in progress"} ), ], @@ -60,13 +90,13 @@ def test_validate_by_id_success(client: FlaskClient, crate_id: str, payload: dic response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) - minio_bucket = payload["minio_bucket"] if "minio_bucket" in payload else None + minio_config = payload["minio_config"] if "minio_config" in payload else None root_path = payload["root_path"] if "root_path" in payload else None profile_name = payload["profile_name"] if "profile_name" in payload else None webhook_url = payload["webhook_url"] if "webhook_url" in payload else None assert response.status_code == status_code assert response.json == response_json - mock_queue.assert_called_once_with(minio_bucket, crate_id, root_path, profile_name, webhook_url) + mock_queue.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url) @pytest.mark.parametrize( From 16665855ac6321e42c5c3cf688f4d19159f721a0 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:31:50 +0100 Subject: [PATCH 109/127] pass minio config to validation service, and load client there --- app/services/validation_service.py | 13 +++-- tests/test_services.py | 89 ++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index b60ae62..223e310 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -18,18 +18,19 @@ ) from app.utils.config import InvalidAPIUsage +from app.utils.minio_utils import get_minio_client logger = logging.getLogger(__name__) def queue_ro_crate_validation_task( - minio_bucket, crate_id, root_path=None, profile_name=None, webhook_url=None + minio_config, crate_id, root_path=None, profile_name=None, webhook_url=None ) -> tuple[Response, int]: """ Queues an RO-Crate for validation with Celery. - :param minio_bucket: The MinIO bucket containing the RO-Crate. + :param minio_config: Access settings for Minio instance containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. :param root_path: The root path containing the RO-Crate. :param profile_name: The profile to validate against. @@ -39,16 +40,18 @@ def queue_ro_crate_validation_task( """ logging.info(f"Processing: {crate_id}, {profile_name}, {webhook_url}") - logging.info(f"Minio Bucket: {minio_bucket}; Root path: {root_path}") + logging.info(f"Minio Bucket: {minio_config['bucket']}; Root path: {root_path}") - if check_ro_crate_exists(minio_bucket, crate_id, root_path): + minio_client = get_minio_client(minio_config) + + if check_ro_crate_exists(minio_client, minio_config["bucket"], crate_id, root_path): logging.info("RO-Crate exists") else: logging.info("RO-Crate does not exist") raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) try: - process_validation_task_by_id.delay(minio_bucket, crate_id, root_path, profile_name, webhook_url) + process_validation_task_by_id.delay(minio_config, crate_id, root_path, profile_name, webhook_url) return jsonify({"message": "Validation in progress"}), 202 except Exception as e: diff --git a/tests/test_services.py b/tests/test_services.py index f5838e5..12b7f4d 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -22,21 +22,33 @@ def flask_app(): # Test function: queue_ro_crate_validation_task @pytest.mark.parametrize( - "crate_id, rocrate_exists, delay_side_effects, payload, status_code, response_dict", + "crate_id, rocrate_exists, minio_client, delay_side_effects, payload, status_code, response_dict", [ ( - "crate123", True, None, + "crate123", True, "minio_client", None, { - "minio_bucket": "test_bucket", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" }, 202, {"message": "Validation in progress"} ), ( - "crate123", True, Exception("Celery down"), + "crate123", True, "minio_client", Exception("Celery down"), { - "minio_bucket": "test_bucket", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" @@ -45,33 +57,47 @@ def flask_app(): ], ids=["successful_queue", "celery_server_down"] ) +@patch("app.services.validation_service.process_validation_task_by_id.delay") +@patch("app.services.validation_service.check_ro_crate_exists") +@patch("app.services.validation_service.get_minio_client") def test_queue_ro_crate_validation_task( - flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, + mock_client, + mock_exists, + mock_delay, + flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, minio_client: str, delay_side_effects: Exception, payload: dict, status_code: int, response_dict: dict ): - minio_bucket = payload["minio_bucket"] if "minio_bucket" in payload else None + mock_delay.side_effect = delay_side_effects + mock_exists.return_value = rocrate_exists + mock_client.return_value = minio_client + + minio_config = payload["minio_config"] if "minio_config" in payload else None root_path = payload["root_path"] if "root_path" in payload else None profile_name = payload["profile_name"] if "profile_name" in payload else None webhook_url = payload["webhook_url"] if "webhook_url" in payload else None - with patch("app.services.validation_service.process_validation_task_by_id.delay", side_effect=delay_side_effects) as mock_delay: - with patch("app.services.validation_service.check_ro_crate_exists", return_value=rocrate_exists) as mock_exists: - - response, status_code = queue_ro_crate_validation_task(minio_bucket, crate_id, root_path, profile_name, webhook_url) + response, status_code = queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) - mock_exists.assert_called_once_with(minio_bucket, crate_id, root_path) - mock_delay.assert_called_once_with(minio_bucket, crate_id, root_path, profile_name, webhook_url) - assert status_code == status_code - assert response.json == response_dict + mock_client.assert_called_once_with(minio_config) + mock_exists.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, root_path) + mock_delay.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url) + assert status_code == status_code + assert response.json == response_dict @pytest.mark.parametrize( - "crate_id, rocrate_exists, payload, iau_message", + "crate_id, rocrate_exists, minio_client, payload, iau_message", [ ( - "crate12z", False, + "crate12z", False, "minio_client", { - "minio_bucket": "test_bucket", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" @@ -80,24 +106,31 @@ def test_queue_ro_crate_validation_task( ], ids=["no_rocrate_exists"] ) +@patch("app.services.validation_service.process_validation_task_by_id.delay") +@patch("app.services.validation_service.check_ro_crate_exists") +@patch("app.services.validation_service.get_minio_client") def test_queue_ro_crate_validation_task_failure( + mock_client, + mock_exists, + mock_delay, flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, - payload: dict, iau_message: str + minio_client: str, payload: dict, iau_message: str ): - minio_bucket = payload["minio_bucket"] if "minio_bucket" in payload else None + mock_exists.return_value = rocrate_exists + mock_client.return_value = minio_client + + minio_config = payload["minio_config"] if "minio_config" in payload else None root_path = payload["root_path"] if "root_path" in payload else None profile_name = payload["profile_name"] if "profile_name" in payload else None webhook_url = payload["webhook_url"] if "webhook_url" in payload else None - with patch("app.services.validation_service.process_validation_task_by_id.delay") as mock_delay: - with patch("app.services.validation_service.check_ro_crate_exists", return_value=rocrate_exists) as mock_exists: - - with pytest.raises(InvalidAPIUsage) as exc_info: - queue_ro_crate_validation_task(minio_bucket, crate_id, root_path, profile_name, webhook_url) + with pytest.raises(InvalidAPIUsage) as exc_info: + queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) - assert iau_message in str(exc_info.value.message) - mock_exists.assert_called_once_with(minio_bucket, crate_id, root_path) - mock_delay.assert_not_called() + assert iau_message in str(exc_info.value.message) + mock_client.assert_called_once_with(minio_config) + mock_exists.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, root_path) + mock_delay.assert_not_called() # Test function: queue_ro_crate_metadata_validation_task From aa9cb9007b62d916e5dca122be7f9d91ea189167 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:37:56 +0100 Subject: [PATCH 110/127] pass minio client into test_ro_crate_exists --- app/tasks/validation_tasks.py | 5 +++-- tests/test_validation_tasks.py | 16 ++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index a842400..3dc2a88 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -192,6 +192,7 @@ def perform_ro_crate_validation( def check_ro_crate_exists( + minio_client: object, bucket_name: str, crate_id: str, root_path: str, @@ -199,7 +200,8 @@ def check_ro_crate_exists( """ Checks for the existence of an RO-Crate using the provided Crate ID. - :param minio_bucket: The MinIO bucket containing the RO-Crate. + :param minio_client: The MinIO client + :param bucket_name: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. :param root_path: The root path containing the RO-Crate. :return: Boolean indicating existence @@ -207,7 +209,6 @@ def check_ro_crate_exists( logging.info(f"Checking for existence of RO-Crate {crate_id}") - minio_client = get_minio_client() if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, root_path): return True else: diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 2ee1a0a..f6f4420 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -349,28 +349,24 @@ def test_return_validation_raises_error(mock_get_status): # Test function: check_ro_crate_exists @pytest.mark.parametrize( - "bucket, crate_id, base_path, client_return, ro_object_return, rocrate_exists", + "minio_client, bucket, crate_id, base_path, client_return, ro_object_return, rocrate_exists", [ - ("test_bucket", "crate123", "base_path", "mock_client", "crate123", True), - ("test_bucket", "crate12z", "base_path", "mock_client", False, False) + ("minio_client", "test_bucket", "crate123", "base_path", "mock_client", "crate123", True), + ("minio_client", "test_bucket", "crate12z", "base_path", "mock_client", False, False) ], ids=["rocrate_exists", "rocrate_does_not_exist"] ) -@mock.patch("app.tasks.validation_tasks.get_minio_client") @mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio") def test_ro_crate_exists( mock_find_rocrate, - mock_get_client, - bucket: str, crate_id: str, base_path: str, client_return: str, + minio_client: str, bucket: str, crate_id: str, base_path: str, client_return: str, ro_object_return: str, rocrate_exists: bool ): - mock_get_client.return_value = client_return mock_find_rocrate.return_value = ro_object_return - result = check_ro_crate_exists(bucket, crate_id, base_path) + result = check_ro_crate_exists(minio_client, bucket, crate_id, base_path) - mock_get_client.assert_called_once() - mock_find_rocrate.assert_called_once_with(crate_id, client_return, bucket, base_path) + mock_find_rocrate.assert_called_once_with(crate_id, minio_client, bucket, base_path) assert result is rocrate_exists From 5bc2621b08617f82e516d035435af9ef7f5b21b1 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:42:06 +0100 Subject: [PATCH 111/127] pass minio client into get validation function --- app/tasks/validation_tasks.py | 3 ++- tests/test_validation_tasks.py | 24 ++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index 3dc2a88..c48c174 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -216,6 +216,7 @@ def check_ro_crate_exists( def check_validation_exists( + minio_client: object, bucket_name: str, crate_id: str, root_path: str, @@ -223,6 +224,7 @@ def check_validation_exists( """ Checks for the existence of a validation result using the provided Crate ID. + :param minio_client: The MinIO client :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. :param root_path: The root path containing the RO-Crate. @@ -231,7 +233,6 @@ def check_validation_exists( logging.info(f"Checking for existence of RO-Crate {crate_id}") - minio_client = get_minio_client() if find_validation_object_on_minio(crate_id, minio_client, bucket_name, root_path): return True else: diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index f6f4420..080810d 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -349,17 +349,17 @@ def test_return_validation_raises_error(mock_get_status): # Test function: check_ro_crate_exists @pytest.mark.parametrize( - "minio_client, bucket, crate_id, base_path, client_return, ro_object_return, rocrate_exists", + "minio_client, bucket, crate_id, base_path, ro_object_return, rocrate_exists", [ - ("minio_client", "test_bucket", "crate123", "base_path", "mock_client", "crate123", True), - ("minio_client", "test_bucket", "crate12z", "base_path", "mock_client", False, False) + ("minio_client", "test_bucket", "crate123", "base_path", "crate123", True), + ("minio_client", "test_bucket", "crate12z", "base_path", False, False) ], ids=["rocrate_exists", "rocrate_does_not_exist"] ) @mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio") def test_ro_crate_exists( mock_find_rocrate, - minio_client: str, bucket: str, crate_id: str, base_path: str, client_return: str, + minio_client: str, bucket: str, crate_id: str, base_path: str, ro_object_return: str, rocrate_exists: bool ): mock_find_rocrate.return_value = ro_object_return @@ -373,26 +373,22 @@ def test_ro_crate_exists( # Test function: check_validation_exists @pytest.mark.parametrize( - "bucket, crate_id, base_path, client_return, val_object_return, validate_exists", + "minio_client, bucket, crate_id, base_path, val_object_return, validate_exists", [ - ("test_bucket", "crate123", "base_path", "mock_client", "crate123", True), - ("test_bucket", "crate12z", "base_path", "mock_client", False, False) + ("minio_client", "test_bucket", "crate123", "base_path", "crate123", True), + ("minio_client", "test_bucket", "crate12z", "base_path", False, False) ], ids=["validation_exists", "validation_does_not_exist"] ) -@mock.patch("app.tasks.validation_tasks.get_minio_client") @mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio") def test_validation_exists( mock_find_validation, - mock_get_client, - bucket: str, crate_id: str, base_path: str, client_return: str, + minio_client: str, bucket: str, crate_id: str, base_path: str, val_object_return: str, validate_exists: bool ): - mock_get_client.return_value = client_return mock_find_validation.return_value = val_object_return - result = check_validation_exists(bucket, crate_id, base_path) + result = check_validation_exists(minio_client, bucket, crate_id, base_path) - mock_get_client.assert_called_once() - mock_find_validation.assert_called_once_with(crate_id, client_return, bucket, base_path) + mock_find_validation.assert_called_once_with(crate_id, minio_client, bucket, base_path) assert result is validate_exists From 26f3a144762147515e79d4c4c3cad3f454201942 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:54:27 +0100 Subject: [PATCH 112/127] pass minio config to validation celery task --- app/tasks/validation_tasks.py | 10 +++--- tests/test_validation_tasks.py | 59 +++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index c48c174..b1d93df 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -29,12 +29,12 @@ @celery.task def process_validation_task_by_id( - minio_bucket: str, crate_id: str, root_path: str, profile_name: str | None, webhook_url: str | None + minio_config: dict, crate_id: str, root_path: str, profile_name: str | None, webhook_url: str | None ) -> None: """ Background task to process the RO-Crate validation by ID. - :param minio_bucket: The MinIO bucket containing the RO-Crate. + :param minio_config: The MinIO configuration. :param crate_id: The ID of the RO-Crate to validate. :param root_path: The root path containing the RO-Crate. :param profile_name: The name of the validation profile to use. Defaults to None. @@ -45,11 +45,13 @@ def process_validation_task_by_id( # TODO: Split try statements: (1) fetch and validate; (2) write to minio; (3) webhook + minio_client = get_minio_client(minio_config) + file_path = None try: # Fetch the RO-Crate from MinIO using the provided ID: - file_path = fetch_ro_crate_from_minio(minio_bucket, crate_id, root_path) + file_path = fetch_ro_crate_from_minio(minio_client, minio_config["bucket"], crate_id, root_path) logging.info(f"Processing validation task for {file_path}") @@ -67,7 +69,7 @@ def process_validation_task_by_id( logging.info(f"RO Crate {crate_id} is invalid.") # Update the validation status in MinIO: - update_validation_status_in_minio(minio_bucket, crate_id, root_path, validation_result.to_json()) + update_validation_status_in_minio(minio_client, minio_config["bucket"], crate_id, root_path, validation_result.to_json()) # TODO: Prepare the data to send to the webhook, and send the webhook notification. diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 080810d..0715204 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -16,17 +16,49 @@ # Test function: process_validation_task_by_id @pytest.mark.parametrize( - "crate_id, os_path_exists, os_path_isfile, os_path_isdir, return_value, webhook, profile, val_success, val_result", + "minio_config, crate_id, os_path_exists, os_path_isfile, os_path_isdir, " + + "return_value, webhook, profile, val_success, val_result, minio_client", [ - ("crate123", True, True, False, "/tmp/crate.zip", - "https://example.com/hook", "profileA", True, '{"status": "valid"}'), - ("crate123", True, False, True, "/tmp/crate123", - "https://example.com/hook", "profileA", True, '{"status": "valid"}'), - ("crate123", True, False, True, "/tmp/crate123", - None, "profileA", True, '{"status": "valid"}'), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", True, '{"status": "valid"}', + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, False, True, "/tmp/crate123", + "https://example.com/hook", "profileA", True, '{"status": "valid"}', + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, False, True, "/tmp/crate123", + None, "profileA", True, '{"status": "valid"}', + "minio_client" + ), ], ids=["successful_validation_zip", "successful_validation_dir", "successful_validation_nowebhook"] ) +@mock.patch("app.tasks.validation_tasks.get_minio_client") @mock.patch("app.tasks.validation_tasks.shutil.rmtree") @mock.patch("app.tasks.validation_tasks.os.remove") @mock.patch("app.tasks.validation_tasks.os.path.exists") @@ -46,24 +78,27 @@ def test_process_validation( mock_exists, mock_remove, mock_rmtree, - crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, - return_value: str, webhook: str, profile: str, val_success: bool, val_result: str + mock_client, + minio_config: dict, crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, + return_value: str, webhook: str, profile: str, val_success: bool, val_result: str, minio_client: str ): mock_exists.return_value = os_path_exists mock_isfile.return_value = os_path_isfile mock_isdir.return_value = os_path_isdir mock_fetch.return_value = return_value + mock_client.return_value = minio_client mock_validation_result = mock.Mock() mock_validation_result.has_issues.return_value = val_success mock_validation_result.to_json.return_value = val_result mock_validate.return_value = mock_validation_result - process_validation_task_by_id("test_bucket", crate_id, "", profile, webhook) + process_validation_task_by_id(minio_config, crate_id, "", profile, webhook) - mock_fetch.assert_called_once_with("test_bucket", crate_id, "") + mock_client.assert_called_once_with(minio_config) + mock_fetch.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "") mock_validate.assert_called_once_with(return_value, profile) - mock_update.assert_called_once_with("test_bucket", crate_id, "", val_result) + mock_update.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "", val_result) if webhook is not None: mock_webhook.assert_called_once_with(webhook, val_result) else: From 6139b60cb96652c7b87fe1072e6e32d1a64ae085 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:59:07 +0100 Subject: [PATCH 113/127] pass minio client into fetch ro-crate function --- app/utils/minio_utils.py | 5 ++--- tests/test_minio.py | 18 ++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index a6397ba..61b97da 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -17,18 +17,17 @@ logger = logging.getLogger(__name__) -def fetch_ro_crate_from_minio(minio_bucket: str, crate_id: str, root_path: str) -> str: +def fetch_ro_crate_from_minio(minio_client: object, minio_bucket: str, crate_id: str, root_path: str) -> str: """ Fetches an RO-Crate from MinIO based on the crate ID. Downloads the crate as a file and returns local file path. + :param minio_client: The MinIO client :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to fetch from MinIO. :param root_path: The root path containing the RO-Crate. :return: The local file path where the RO-Crate is saved. """ - minio_client = get_minio_client() - rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, minio_bucket, root_path) rocrate_minio_path = rocrate_object.object_name diff --git a/tests/test_minio.py b/tests/test_minio.py index 1c15688..3f9e0f3 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -423,16 +423,14 @@ def test_update_validation_status_erro( @patch("app.utils.minio_utils.download_file_from_minio") @patch("app.utils.minio_utils.get_minio_object_list") @patch("app.utils.minio_utils.find_rocrate_object_on_minio") -@patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_zip( - mock_get_client, mock_find_object, mock_get_list, mock_download, tmp_path, ): # Setup mocks - mock_get_client.return_value = "minio_client" + minio_client = "minio_client" rocrate_obj = DummyObject("some/path/rocrate123.zip", is_dir=False) mock_find_object.return_value = rocrate_obj @@ -440,7 +438,7 @@ def test_fetch_rocrate_zip( with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): # Execute - result = fetch_ro_crate_from_minio("test_bucket", "rocrate123", "some/path") + result = fetch_ro_crate_from_minio(minio_client, "test_bucket", "rocrate123", "some/path") # Assert expected_path = tmp_path / "rocrate123.zip" @@ -453,16 +451,14 @@ def test_fetch_rocrate_zip( @patch("app.utils.minio_utils.download_file_from_minio") @patch("app.utils.minio_utils.get_minio_object_list") @patch("app.utils.minio_utils.find_rocrate_object_on_minio") -@patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_directory( - mock_get_client, mock_find_object, mock_get_list, mock_download, tmp_path, ): # Setup mocks - mock_get_client.return_value = "minio_client" + minio_client = "minio_client" rocrate_obj = DummyObject("rocrates/rocrate124", is_dir=True) mock_find_object.return_value = rocrate_obj @@ -476,7 +472,7 @@ def test_fetch_rocrate_directory( ] # Execute - result = fetch_ro_crate_from_minio("test_bucket", "rocrate124", "rocrates") + result = fetch_ro_crate_from_minio(minio_client, "test_bucket", "rocrate124", "rocrates") # Assert expected_root = tmp_path / "rocrate124" @@ -496,15 +492,13 @@ def test_fetch_rocrate_directory( @patch("app.utils.minio_utils.download_file_from_minio") @patch("app.utils.minio_utils.get_minio_object_list") @patch("app.utils.minio_utils.find_rocrate_object_on_minio") -@patch("app.utils.minio_utils.get_minio_client") def test_fetch_rocrate_handles_empty_dir( - mock_get_client, mock_find_object, mock_get_list, mock_download, tmp_path, ): - mock_get_client.return_value = "minio_client" + minio_client = "minio_client" rocrate_obj = DummyObject("rocrate456", is_dir=True) mock_find_object.return_value = rocrate_obj mock_get_list.return_value = [] @@ -512,7 +506,7 @@ def test_fetch_rocrate_handles_empty_dir( from app.utils.minio_utils import fetch_ro_crate_from_minio with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): - result = fetch_ro_crate_from_minio("test_bucket", "rocrate456", "") + result = fetch_ro_crate_from_minio(minio_client, "test_bucket", "rocrate456", "") expected_root = tmp_path / "rocrate456" assert result == str(expected_root) From 1629663bfb7c12ed064ff90d8cc1fd4c24d1d319 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 5 Aug 2025 00:02:37 +0100 Subject: [PATCH 114/127] pass minio client into update validation function --- app/utils/minio_utils.py | 6 ++---- tests/test_minio.py | 11 +++-------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index 61b97da..ef085fb 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -61,10 +61,11 @@ def fetch_ro_crate_from_minio(minio_client: object, minio_bucket: str, crate_id: return local_root_path -def update_validation_status_in_minio(minio_bucket: str, crate_id: str, root_path: str, validation_status: str) -> None: +def update_validation_status_in_minio(minio_client: object, minio_bucket: str, crate_id: str, root_path: str, validation_status: str) -> None: """ Uploads the validation status to the MinIO bucket. + :param minio_client: The MinIO client :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate in MinIO :param validation_status: The validation result to upload @@ -83,9 +84,6 @@ def update_validation_status_in_minio(minio_bucket: str, crate_id: str, root_pat validation_string = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") try: - - minio_client = get_minio_client() - minio_client.put_object( minio_bucket, object_name, diff --git a/tests/test_minio.py b/tests/test_minio.py index 3f9e0f3..44d7ca4 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -337,16 +337,14 @@ def test_get_validation_error_raised( # Testing function: update_validation_status_in_minio -@mock.patch("app.utils.minio_utils.get_minio_client") -def test_update_validation_status_success(mock_get_client): +def test_update_validation_status_success(): mock_minio_client = mock.Mock() - mock_get_client.return_value = mock_minio_client crate_id = "crate123" validation_status = json.dumps({"status": "valid", "errors": []}) from app.utils.minio_utils import update_validation_status_in_minio - update_validation_status_in_minio("test_bucket", crate_id, "", validation_status) + update_validation_status_in_minio(mock_minio_client, "test_bucket", crate_id, "", validation_status) expected_object_name = f"{crate_id}_validation/validation_status.txt" expected_data = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") @@ -400,19 +398,16 @@ def test_update_validation_status_success(mock_get_client): ], ids=["s3error", "value_error", "unexpected_error"] ) -@mock.patch("app.utils.minio_utils.get_minio_client") def test_update_validation_status_erro( - mock_get_client, bucket: str, crateid: str, root_path: str, validation_result: dict, put_side_effect, error_check: str, status_code: int ): mock_minio_client = mock.Mock() - mock_get_client.return_value = mock_minio_client mock_minio_client.put_object.side_effect = put_side_effect from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - update_validation_status_in_minio(bucket, crateid, root_path, json.dumps(validation_result)) + update_validation_status_in_minio(mock_minio_client, bucket, crateid, root_path, json.dumps(validation_result)) assert exc.value.status_code == status_code assert error_check in str(exc.value.message) From 0d4c307b3350c307edfad3a34e6a824a54c89827 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 5 Aug 2025 00:08:55 +0100 Subject: [PATCH 115/127] update process validation failure tests --- tests/test_validation_tasks.py | 55 +++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index 0715204..f7dad96 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -112,19 +112,50 @@ def test_process_validation( @pytest.mark.parametrize( - "crate_id, os_path_exists, os_path_isfile, os_path_isdir, return_fetch, " - + "webhook, profile, return_validate, validate_side_effect, fetch_side_effect", + "minio_config, crate_id, os_path_exists, os_path_isfile, os_path_isdir, return_fetch, " + + "webhook, profile, return_validate, validate_side_effect, fetch_side_effect, minio_client", [ - ("crate123", True, True, False, "/tmp/crate.zip", - "https://example.com/hook", "profileA", "Validation failed", None, None), - ("crate123", True, True, False, "/tmp/crate.zip", - "https://example.com/hook", "profileA", None, Exception("Unexpected error"), None), - ("crate123", False, False, False, None, - "https://example.com/hook", "profileA", None, None, Exception("MinIO fetch failed")), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", "Validation failed", None, None, + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", None, Exception("Unexpected error"), None, + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", False, False, False, None, + "https://example.com/hook", "profileA", None, None, Exception("MinIO fetch failed"), + "minio_client" + ), ], ids=["validation_fails_with_message", "validation_fails_with_validation_exception", "validation_fails_with_fetch_exception"] ) +@mock.patch("app.tasks.validation_tasks.get_minio_client") @mock.patch("app.tasks.validation_tasks.shutil.rmtree") @mock.patch("app.tasks.validation_tasks.os.remove") @mock.patch("app.tasks.validation_tasks.os.path.exists") @@ -144,13 +175,15 @@ def test_process_validation_failure( mock_exists, mock_remove, mock_rmtree, - crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, + mock_client, + minio_config: dict, crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, return_fetch: str, webhook: str, profile: str, return_validate: str, - validate_side_effect: Exception, fetch_side_effect: Exception + validate_side_effect: Exception, fetch_side_effect: Exception, minio_client: str ): mock_exists.return_value = os_path_exists mock_isfile.return_value = os_path_isfile mock_isdir.return_value = os_path_isdir + mock_client.return_value = minio_client if fetch_side_effect is None: mock_fetch.return_value = return_fetch @@ -162,7 +195,7 @@ def test_process_validation_failure( else: mock_validate.side_effect = validate_side_effect - process_validation_task_by_id("test_bucket", crate_id, "", profile, webhook) + process_validation_task_by_id(minio_config, crate_id, "", profile, webhook) if fetch_side_effect is None: mock_validate.assert_called_once_with(return_fetch, profile) From d2c730d2c29364b5b7adbe31afdda33020eb15e1 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 5 Aug 2025 07:32:54 +0100 Subject: [PATCH 116/127] add minio config to api get route --- app/ro_crates/routes/get_routes.py | 24 ++++++++++++---- tests/test_api_routes.py | 44 +++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/app/ro_crates/routes/get_routes.py b/app/ro_crates/routes/get_routes.py index 4298a79..d6b23c6 100644 --- a/app/ro_crates/routes/get_routes.py +++ b/app/ro_crates/routes/get_routes.py @@ -5,7 +5,8 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester from apiflask import APIBlueprint, Schema -from apiflask.fields import String +from apiflask.fields import String, Boolean +from marshmallow.fields import Nested from flask import Response from app.services.validation_service import get_ro_crate_validation_task @@ -13,8 +14,16 @@ get_routes_bp = APIBlueprint("get_routes", __name__) +class MinioConfig(Schema): + endpoint = String(required=True) + accesskey = String(required=True) + secret = String(required=True) + ssl = Boolean(required=True) + bucket = String(required=True) + + class ValidateResult(Schema): - minio_bucket = String(required=True) + minio_config = Nested(MinioConfig, required=True) root_path = String(required=False) @@ -28,7 +37,12 @@ def get_ro_crate_validation_by_id(json_data, crate_id) -> tuple[Response, int]: - **crate_id**: The RO-Crate ID. _Required_. Request Body Parameters: - - **minio_bucket**: The MinIO bucket containing the RO-Crate. _Required_ + - **minio_config**: The MinIO bucket containing the RO-Crate. _Required_ + - **endpoint**: Endpoint, e.g. 'localhost:9000' + - **accesskey**: Access key / username + - **secret**: Secret / password + - **ssl**: Use SSL encryption? True/False + - **bucket**: The MinIO bucket to access - **root_path**: The root path containing the RO-Crate. _Optional_ Returns: @@ -38,11 +52,11 @@ def get_ro_crate_validation_by_id(json_data, crate_id) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id`) are missing. """ - minio_bucket = json_data["minio_bucket"] + minio_config = json_data["minio_config"] if "root_path" in json_data: root_path = json_data["root_path"] else: root_path = None - return get_ro_crate_validation_task(minio_bucket, crate_id, root_path) + return get_ro_crate_validation_task(minio_config, crate_id, root_path) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 38c7b80..f527501 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -200,13 +200,25 @@ def test_validate_metadata_failure(client: FlaskClient, payload: dict, status_co "crate_id, payload, status_code", [ ( - "",{ - "minio_bucket": "test_bucket", + "", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, "root_path": "base_path" }, 404 ), ( - "crate-123",{ + "crate-123", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + }, "root_path": "base_path" }, 422 ), @@ -221,7 +233,13 @@ def test_get_validation_by_id_failures(client: FlaskClient, crate_id: str, paylo def test_get_validation_by_id_success(client): crate_id = "crate-123" payload = { - "minio_bucket": "test_bucket", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, "root_path": "base_path" } @@ -232,20 +250,26 @@ def test_get_validation_by_id_success(client): assert response.status_code == 200 assert response.json == {"status": "valid"} - mock_get.assert_called_once_with("test_bucket", "crate-123", "base_path") + mock_get.assert_called_once_with(payload["minio_config"], "crate-123", "base_path") def test_get_validation_by_id_missing_root_path(client): crate_id = "crate-123" payload = { - "minio_bucket": "test_bucket", + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + } } with patch("app.ro_crates.routes.get_routes.get_ro_crate_validation_task") as mock_get: - mock_get.return_value = ({"message": "Validation in progress"}, 202) + mock_get.return_value = ({"status": "valid"}, 200) response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) - assert response.status_code == 202 - assert response.json == {"message": "Validation in progress"} - mock_get.assert_called_once_with("test_bucket", "crate-123", None) + assert response.status_code == 200 + assert response.json == {"status": "valid"} + mock_get.assert_called_once_with(payload["minio_config"], "crate-123", None) From 12b729e79440be9767a68898c71d2db93b22bccc Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 5 Aug 2025 07:49:16 +0100 Subject: [PATCH 117/127] add minio_config and minio client to get validation service --- app/services/validation_service.py | 12 ++-- tests/test_services.py | 111 ++++++++++++++++++++--------- 2 files changed, 84 insertions(+), 39 deletions(-) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 223e310..67dde94 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -100,14 +100,14 @@ def queue_ro_crate_metadata_validation_task( def get_ro_crate_validation_task( - minio_bucket: str, + minio_config: dict, crate_id: str, root_path: str, ) -> tuple[Response, int]: """ Retrieves an RO-Crate validation result. - :param minio_bucket: The MinIO bucket containing the RO-Crate. + :param minio_config: Access settings for Minio instance containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. :param root_path: The root path containing the RO-Crate. :return: A tuple containing a JSON response and an HTTP status code. @@ -115,16 +115,18 @@ def get_ro_crate_validation_task( """ logging.info(f"Retrieving validation for: {crate_id}") - if check_ro_crate_exists(minio_bucket, crate_id, root_path): + minio_client = get_minio_client(minio_config) + + if check_ro_crate_exists(minio_client, minio_config["bucket"], crate_id, root_path): logging.info("RO-Crate exists") else: logging.info("RO-Crate does not exist") raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) - if check_validation_exists(minio_bucket, crate_id, root_path): + if check_validation_exists(minio_client, minio_config["bucket"], crate_id, root_path): logging.info("Validation result exists") else: logging.info("Validation does not exist") raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {crate_id}", 400) - return return_ro_crate_validation(minio_bucket, crate_id, root_path), 200 + return return_ro_crate_validation(minio_client, minio_config["bucket"], crate_id, root_path), 200 diff --git a/tests/test_services.py b/tests/test_services.py index 12b7f4d..c7d50c3 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -203,41 +203,84 @@ def test_queue_metadata_json_errors(flask_app, crate_json: str, status_code: int # Test function: get_ro_crate_validation_task @pytest.mark.parametrize( - "crate_id, crate_exists, validation_exists, validation_value, status_code, error_message", + "minio_config, crate_id, crate_exists, validation_exists, " + + "validation_value, status_code, error_message, minio_client", [ - ("crate123", True, True, {"status": "valid"}, 200, None), - ("crate123", False, False, None, 400, "No RO-Crate with prefix: crate123"), - ("crate123", True, False, None, 400, "No validation result yet for RO-Crate: crate123"), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, {"status": "valid"}, 200, None, + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", False, False, None, 400, "No RO-Crate with prefix: crate123", + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, False, None, 400, "No validation result yet for RO-Crate: crate123", + "minio_client" + ), ], ids=["validation_exists", "rocrate_missing", "validation_missing"] ) -def test_get_validation(flask_app, crate_id: str, crate_exists: bool, - validation_exists: bool, validation_value: dict, - status_code: int, error_message: str): - with patch("app.services.validation_service.check_ro_crate_exists", return_value=crate_exists) as mock_rocrate: - with patch("app.services.validation_service.check_validation_exists", return_value=validation_exists) as mock_validation: - with patch("app.services.validation_service.return_ro_crate_validation", return_value=validation_value) as mock_return: - - if crate_exists and validation_exists: - response, status = get_ro_crate_validation_task("test_bucket", crate_id, "base_path") - - mock_return.assert_called_once_with("test_bucket", crate_id, "base_path") - mock_rocrate.assert_called_once_with("test_bucket", crate_id, "base_path") - mock_validation.assert_called_once_with("test_bucket", crate_id, "base_path") - - assert status == status_code - assert response == validation_value - - else: - with pytest.raises(InvalidAPIUsage) as exc_info: - get_ro_crate_validation_task("test_bucket", crate_id, "base_path") - - assert exc_info.value.status_code == status_code - assert error_message in str(exc_info.value.message) - - mock_rocrate.assert_called_once_with("test_bucket", crate_id, "base_path") - if crate_exists: - mock_validation.assert_called_once_with("test_bucket", crate_id, "base_path") - else: - mock_validation.assert_not_called() - mock_return.assert_not_called() +@patch("app.services.validation_service.check_ro_crate_exists") +@patch("app.services.validation_service.check_validation_exists") +@patch("app.services.validation_service.return_ro_crate_validation") +@patch("app.services.validation_service.get_minio_client") +def test_get_validation( + mock_client, + mock_return, + mock_validation, + mock_rocrate, + flask_app, minio_config: dict, crate_id: str, crate_exists: bool, + validation_exists: bool, validation_value: dict, + status_code: int, error_message: str, minio_client: str +): + mock_client.return_value = minio_client + mock_rocrate.return_value = crate_exists + mock_validation.return_value = validation_exists + mock_return.return_value = validation_value + + if crate_exists and validation_exists: + response, status = get_ro_crate_validation_task(minio_config, crate_id, "base_path") + + mock_client.assert_called_once_with(minio_config) + mock_return.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + mock_rocrate.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + mock_validation.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + + assert status == status_code + assert response == validation_value + + else: + with pytest.raises(InvalidAPIUsage) as exc_info: + get_ro_crate_validation_task(minio_config, crate_id, "base_path") + + assert exc_info.value.status_code == status_code + assert error_message in str(exc_info.value.message) + + mock_rocrate.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + if crate_exists: + mock_validation.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + else: + mock_validation.assert_not_called() + mock_return.assert_not_called() From ba62fc882ed0b5ec8a718e5ec2d0655bc69b5b45 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 5 Aug 2025 07:52:16 +0100 Subject: [PATCH 118/127] add minio client to return ro-crate validation --- app/tasks/validation_tasks.py | 4 +++- tests/test_validation_tasks.py | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index b1d93df..0a62b55 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -242,6 +242,7 @@ def check_validation_exists( def return_ro_crate_validation( + minio_client: object, bucket_name: str, crate_id: str, root_path: str, @@ -249,10 +250,11 @@ def return_ro_crate_validation( """ Retrieves the validation result for an RO-Crate using the provided Crate ID. + :param minio_client: The MinIO client :param crate_id: The ID of the RO-Crate that has been validated :return: The validation result """ logging.info(f"Fetching validation result for RO-Crate {crate_id}") - return get_validation_status_from_minio(bucket_name, crate_id, root_path) + return get_validation_status_from_minio(minio_client, bucket_name, crate_id, root_path) diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index f7dad96..afa11c2 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -385,10 +385,10 @@ def test_return_validation_returns_dict(mock_get_status): # Simulate dict result mock_get_status.return_value = {"status": "passed", "errors": []} - result = return_ro_crate_validation("test_bucket", "crate123", None) + result = return_ro_crate_validation("minio_client", "test_bucket", "crate123", None) assert isinstance(result, dict) assert result["status"] == "passed" - mock_get_status.assert_called_once_with("test_bucket", "crate123", None) + mock_get_status.assert_called_once_with("minio_client", "test_bucket", "crate123", None) @mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") @@ -396,10 +396,10 @@ def test_return_validation_returns_string(mock_get_status): # Simulate string result mock_get_status.return_value = "Validation result: OK" - result = return_ro_crate_validation("test_bucket", "crate456", None) + result = return_ro_crate_validation("minio_client", "test_bucket", "crate456", None) assert isinstance(result, str) assert "OK" in result - mock_get_status.assert_called_once_with("test_bucket", "crate456", None) + mock_get_status.assert_called_once_with("minio_client", "test_bucket", "crate456", None) @mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") @@ -408,10 +408,10 @@ def test_return_validation_raises_error(mock_get_status): mock_get_status.side_effect = InvalidAPIUsage("MinIO S3 Error: empty", 500) with pytest.raises(InvalidAPIUsage) as exc_info: - return_ro_crate_validation("test_bucket", "crate789", None) + return_ro_crate_validation("minio_client", "test_bucket", "crate789", None) assert "MinIO S3 Error" in str(exc_info.value.message) - mock_get_status.assert_called_once_with("test_bucket", "crate789", None) + mock_get_status.assert_called_once_with("minio_client", "test_bucket", "crate789", None) # Test function: check_ro_crate_exists From 5b5daf623e0d1d9b84121611bde7ed659f8c2430 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 5 Aug 2025 07:56:02 +0100 Subject: [PATCH 119/127] pass minio_client into get validation from minio --- app/utils/minio_utils.py | 6 ++---- tests/test_minio.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index ef085fb..1612f90 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -109,11 +109,12 @@ def update_validation_status_in_minio(minio_client: object, minio_bucket: str, c ) -def get_validation_status_from_minio(minio_bucket: str, crate_id: str, root_path: str) -> dict: +def get_validation_status_from_minio(minio_client: object, minio_bucket: str, crate_id: str, root_path: str) -> dict: """ Checks for the existence of a validation report for the given RO-Crate in the MinIO bucket. Returns validation message if it exists, or notification that it is missing if not. + :param minio_client: The MinIO client :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate in MinIO :return validation_status: Either the validation status, or note that this does not exist @@ -129,9 +130,6 @@ def get_validation_status_from_minio(minio_bucket: str, crate_id: str, root_path logging.info(f"Getting object {object_name}") try: - - minio_client = get_minio_client() - response = minio_client.get_object( minio_bucket, object_name, diff --git a/tests/test_minio.py b/tests/test_minio.py index 44d7ca4..426d901 100644 --- a/tests/test_minio.py +++ b/tests/test_minio.py @@ -283,10 +283,9 @@ def test_download_s3error( def test_successful_retrieval(mocker, mock_minio_response): mock_client = MagicMock() mock_client.get_object.return_value = mock_minio_response - mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) from app.utils.minio_utils import get_validation_status_from_minio - result = get_validation_status_from_minio("test_bucket", "crate123", None) + result = get_validation_status_from_minio(mock_client, "test_bucket", "crate123", None) assert result == {"status": "valid"} mock_minio_response.close.assert_called_once() @@ -325,11 +324,10 @@ def test_get_validation_error_raised( ): mock_client = MagicMock() mock_client.get_object.side_effect = get_side_effect - mocker.patch("app.utils.minio_utils.get_minio_client", return_value=mock_client) from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage with pytest.raises(InvalidAPIUsage) as exc: - get_validation_status_from_minio(bucket, crateid, root_path) + get_validation_status_from_minio(mock_client, bucket, crateid, root_path) assert exc.value.status_code == status_code assert error_check in str(exc.value.message) From 19e7003cecb77737262552822c7d467c0261addf Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Tue, 5 Aug 2025 08:42:25 +0100 Subject: [PATCH 120/127] update integration tests to pass minio details to api --- tests/test_integration.py | 72 ++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 4d7e5ec..2b4b9a7 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -104,7 +104,13 @@ def test_no_rocrate_for_validation(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates" + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } } response = requests.post(url, json=payload, headers=headers) @@ -130,7 +136,13 @@ def test_no_validation_result_for_missing_crate(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates" + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } } # GET action and tests @@ -156,7 +168,13 @@ def test_get_existing_validation_result(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates" + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } } # GET action and tests @@ -182,7 +200,13 @@ def test_rocrate_not_validated_yet(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates" + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } } # GET action and tests @@ -209,7 +233,13 @@ def test_zipped_rocrate_validation(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates" + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } } # POST action and tests @@ -266,7 +296,13 @@ def test_directory_rocrate_validation(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates" + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } } # POST action and tests @@ -322,7 +358,13 @@ def test_ignore_rocrates_not_on_basepath(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates" + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } } # POST action and tests @@ -350,7 +392,13 @@ def test_zipped_rocrate_in_subdirectory_validation(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates", + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + }, "root_path" : subdir_path } @@ -409,7 +457,13 @@ def test_directory_rocrate_in_subdirectory_validation(): # The API expects the JSON to be passed as a string payload = { - "minio_bucket" : "ro-crates", + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + }, "root_path" : subdir_path } From f10baf60cb1598d589256c4718846bb14a3027cf Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 7 Aug 2025 21:59:32 +0100 Subject: [PATCH 121/127] remove testing reqs file --- requirements.testing.txt | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 requirements.testing.txt diff --git a/requirements.testing.txt b/requirements.testing.txt deleted file mode 100644 index 6bec2e6..0000000 --- a/requirements.testing.txt +++ /dev/null @@ -1,10 +0,0 @@ -celery~=5.4.0 -minio~=7.2.10 -requests~=2.32.3 -Flask~=3.0.3 -Werkzeug~=3.0.4 -redis~=5.2.0 -python-dotenv~=1.0.1 -apiflask~=2.4.0 -roc-validator~=0.6.5 -pytest~=8.4.1 From 2679d27f9e420d7fb489be4e8e0ed0839769f010 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 7 Aug 2025 22:06:56 +0100 Subject: [PATCH 122/127] move requirements to pip-tools input file --- requirements.txt => requirements.in | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename requirements.txt => requirements.in (100%) diff --git a/requirements.txt b/requirements.in similarity index 100% rename from requirements.txt rename to requirements.in From e737b0e93eaeafe77fd67183b2fa035e1b6433dd Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 7 Aug 2025 22:09:07 +0100 Subject: [PATCH 123/127] pip-compile requirements file --- requirements.txt | 196 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f36abd1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,196 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile requirements.in +# +amqp==5.3.1 + # via kombu +apiflask==2.4.0 + # via -r requirements.in +apispec==6.8.2 + # via apiflask +argon2-cffi==25.1.0 + # via minio +argon2-cffi-bindings==25.1.0 + # via argon2-cffi +async-timeout==5.0.1 + # via redis +attrs==25.3.0 + # via + # cattrs + # requests-cache +billiard==4.2.1 + # via celery +blinker==1.9.0 + # via flask +cattrs==25.1.1 + # via requests-cache +celery==5.4.0 + # via -r requirements.in +certifi==2025.8.3 + # via + # minio + # requests +cffi==1.17.1 + # via argon2-cffi-bindings +charset-normalizer==3.4.2 + # via requests +click==8.2.1 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl + # flask + # rich-click + # roc-validator +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1.2 + # via celery +click-repl==0.3.0 + # via celery +colorlog==6.9.0 + # via roc-validator +enum-tools==0.12.0 + # via roc-validator +flask==3.0.3 + # via + # -r requirements.in + # apiflask + # flask-httpauth + # flask-marshmallow +flask-httpauth==4.8.0 + # via apiflask +flask-marshmallow==1.3.0 + # via apiflask +html5lib==1.1 + # via pyshacl +idna==3.10 + # via + # requests + # url-normalize +importlib-metadata==8.7.0 + # via pyshacl +inquirerpy==0.3.4 + # via roc-validator +itsdangerous==2.2.0 + # via flask +jinja2==3.1.6 + # via flask +kombu==5.5.4 + # via celery +markdown-it-py==3.0.0 + # via rich +markupsafe==3.0.2 + # via + # jinja2 + # werkzeug +marshmallow==4.0.0 + # via + # apiflask + # flask-marshmallow + # webargs +mdurl==0.1.2 + # via markdown-it-py +minio==7.2.16 + # via -r requirements.in +owlrl==6.0.2 + # via pyshacl +packaging==25.0 + # via + # apispec + # kombu + # pyshacl + # webargs +pfzy==0.3.4 + # via inquirerpy +platformdirs==4.3.8 + # via requests-cache +prettytable==3.16.0 + # via pyshacl +prompt-toolkit==3.0.51 + # via + # click-repl + # inquirerpy +pycparser==2.22 + # via cffi +pycryptodome==3.23.0 + # via minio +pygments==2.19.2 + # via + # enum-tools + # rich +pyparsing==3.2.3 + # via rdflib +pyshacl==0.26.0 + # via roc-validator +python-dateutil==2.9.0.post0 + # via celery +python-dotenv==1.0.1 + # via -r requirements.in +rdflib==7.1.4 + # via + # owlrl + # pyshacl + # roc-validator +redis==5.2.1 + # via -r requirements.in +requests==2.32.4 + # via + # -r requirements.in + # requests-cache + # roc-validator +requests-cache==1.2.1 + # via roc-validator +rich==13.9.4 + # via + # rich-click + # roc-validator +rich-click==1.8.9 + # via roc-validator +roc-validator==0.6.5 + # via -r requirements.in +six==1.17.0 + # via + # html5lib + # python-dateutil +toml==0.10.2 + # via roc-validator +typing-extensions==4.14.1 + # via + # cattrs + # enum-tools + # minio + # rich-click +tzdata==2025.2 + # via + # celery + # kombu +url-normalize==2.2.1 + # via requests-cache +urllib3==2.5.0 + # via + # minio + # requests + # requests-cache +vine==5.1.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # prettytable + # prompt-toolkit +webargs==8.7.0 + # via apiflask +webencodings==0.5.1 + # via html5lib +werkzeug==3.0.6 + # via + # -r requirements.in + # flask +zipp==3.23.0 + # via importlib-metadata From 8555c1710276b597458a4f8f34a0ef6bbe2fda23 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 7 Aug 2025 22:19:07 +0100 Subject: [PATCH 124/127] update requirements --- requirements.in | 17 ++++++++--------- requirements.txt | 26 ++++++++++---------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/requirements.in b/requirements.in index f828262..93f735b 100644 --- a/requirements.in +++ b/requirements.in @@ -1,9 +1,8 @@ -celery~=5.4.0 -minio~=7.2.10 -requests~=2.32.3 -Flask~=3.0.3 -Werkzeug~=3.0.4 -redis~=5.2.0 -python-dotenv~=1.0.1 -apiflask~=2.4.0 -roc-validator~=0.6.5 \ No newline at end of file +celery==5.4.0 +minio==7.2.16 +requests==2.32.4 +Flask==3.0.3 +Werkzeug==3.0.4 +redis==5.2.0 +apiflask==2.4.0 +roc-validator==0.7.3 diff --git a/requirements.txt b/requirements.txt index f36abd1..16487a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile requirements.in +# pip-compile --output-file=requirements.txt requirements.in # amqp==5.3.1 # via kombu @@ -65,8 +65,8 @@ flask-httpauth==4.8.0 # via apiflask flask-marshmallow==1.3.0 # via apiflask -html5lib==1.1 - # via pyshacl +html5rdf==1.2.1 + # via rdflib idna==3.10 # via # requests @@ -96,7 +96,7 @@ mdurl==0.1.2 # via markdown-it-py minio==7.2.16 # via -r requirements.in -owlrl==6.0.2 +owlrl==7.1.4 # via pyshacl packaging==25.0 # via @@ -124,18 +124,16 @@ pygments==2.19.2 # rich pyparsing==3.2.3 # via rdflib -pyshacl==0.26.0 +pyshacl==0.30.1 # via roc-validator python-dateutil==2.9.0.post0 # via celery -python-dotenv==1.0.1 - # via -r requirements.in -rdflib==7.1.4 +rdflib[html]==7.1.4 # via # owlrl # pyshacl # roc-validator -redis==5.2.1 +redis==5.2.0 # via -r requirements.in requests==2.32.4 # via @@ -150,12 +148,10 @@ rich==13.9.4 # roc-validator rich-click==1.8.9 # via roc-validator -roc-validator==0.6.5 +roc-validator==0.7.3 # via -r requirements.in six==1.17.0 - # via - # html5lib - # python-dateutil + # via python-dateutil toml==0.10.2 # via roc-validator typing-extensions==4.14.1 @@ -186,9 +182,7 @@ wcwidth==0.2.13 # prompt-toolkit webargs==8.7.0 # via apiflask -webencodings==0.5.1 - # via html5lib -werkzeug==3.0.6 +werkzeug==3.0.4 # via # -r requirements.in # flask From c2089909ba4e50c279fcb31b47e8651d7b67de94 Mon Sep 17 00:00:00 2001 From: Douglas Lowe <10961945+douglowe@users.noreply.github.com> Date: Thu, 7 Aug 2025 22:21:16 +0100 Subject: [PATCH 125/127] add dotenv tools back in --- requirements.in | 1 + requirements.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements.in b/requirements.in index 93f735b..ab32e32 100644 --- a/requirements.in +++ b/requirements.in @@ -4,5 +4,6 @@ requests==2.32.4 Flask==3.0.3 Werkzeug==3.0.4 redis==5.2.0 +python-dotenv==1.1.1 apiflask==2.4.0 roc-validator==0.7.3 diff --git a/requirements.txt b/requirements.txt index 16487a2..8a25d22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -128,6 +128,8 @@ pyshacl==0.30.1 # via roc-validator python-dateutil==2.9.0.post0 # via celery +python-dotenv==1.1.1 + # via -r requirements.in rdflib[html]==7.1.4 # via # owlrl From 63cbff1e2deba5613ac5ef6f06879fa69bd59afb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Aug 2025 21:25:10 +0000 Subject: [PATCH 126/127] Update celery requirement from ~=5.4.0 to ~=5.5.3 Updates the requirements on [celery](https://github.com/celery/celery) to permit the latest version. - [Release notes](https://github.com/celery/celery/releases) - [Changelog](https://github.com/celery/celery/blob/main/Changelog.rst) - [Commits](https://github.com/celery/celery/compare/v5.4.0...v5.5.3) --- updated-dependencies: - dependency-name: celery dependency-version: 5.5.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.in | 2 +- requirements.txt | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/requirements.in b/requirements.in index ab32e32..7917ac3 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -celery==5.4.0 +celery==5.5.3 minio==7.2.16 requests==2.32.4 Flask==3.0.3 diff --git a/requirements.txt b/requirements.txt index 8a25d22..7634971 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,6 @@ argon2-cffi==25.1.0 # via minio argon2-cffi-bindings==25.1.0 # via argon2-cffi -async-timeout==5.0.1 - # via redis attrs==25.3.0 # via # cattrs @@ -26,7 +24,7 @@ blinker==1.9.0 # via flask cattrs==25.1.1 # via requests-cache -celery==5.4.0 +celery==5.5.3 # via -r requirements.in certifi==2025.8.3 # via @@ -163,9 +161,7 @@ typing-extensions==4.14.1 # minio # rich-click tzdata==2025.2 - # via - # celery - # kombu + # via kombu url-normalize==2.2.1 # via requests-cache urllib3==2.5.0 From 8e6a4b9ba814ff32dba70786fd410127298ab34e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Aug 2025 21:25:02 +0000 Subject: [PATCH 127/127] Bump werkzeug from 3.0.4 to 3.0.6 Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.4 to 3.0.6. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/3.0.4...3.0.6) --- updated-dependencies: - dependency-name: werkzeug dependency-version: 3.0.6 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 7917ac3..26116dc 100644 --- a/requirements.in +++ b/requirements.in @@ -2,7 +2,7 @@ celery==5.5.3 minio==7.2.16 requests==2.32.4 Flask==3.0.3 -Werkzeug==3.0.4 +Werkzeug==3.0.6 redis==5.2.0 python-dotenv==1.1.1 apiflask==2.4.0 diff --git a/requirements.txt b/requirements.txt index 7634971..760e034 100644 --- a/requirements.txt +++ b/requirements.txt @@ -180,7 +180,7 @@ wcwidth==0.2.13 # prompt-toolkit webargs==8.7.0 # via apiflask -werkzeug==3.0.4 +werkzeug==3.0.6 # via # -r requirements.in # flask