refactor(storage): migrate store-backed flow to S3Backend; drop minio-py

- Move POST/task/GET onto StorageBackend + resolver + runner with server-side credentials. POST resolves before queueing (400/404/409/503 via app error handlers);
- the Celery task (`run_validation_job`) runs fetch->validate->persist->webhook with retries and persists error outcomes; GET reads the results prefix.
- Add s3_crate_prefix/s3_results_prefix to Settings. Delete minio_utils and test_minio, remove minio from requirements and recompile the lock.
- Covers the store-backed validation flow

Closes #177, #178, #179

Author: alexhambley

app/__init__.py

@@ -4,7 +4,10 @@
 
 from apiflask import APIFlask
 
+from app.crates.ids import InvalidCrateId
+from app.crates.resolver import CrateNotFound, AmbiguousCrate
 from app.ro_crates.routes import v1_post_bp, v1_minio_post_bp, v1_minio_get_bp
+from app.storage.errors import StorageError
 from app.utils.config import (
     Settings,
     InvalidAPIUsage,
@@ -58,6 +61,23 @@ def create_app(settings: Settings | None = None) -> APIFlask:
     def invalid_api_usage(e):
         return jsonify(e.to_dict()), e.status_code
 
+    @app.errorhandler(InvalidCrateId)
+    def invalid_crate_id(e):
+        return jsonify({"error": str(e)}), 400
+
+    @app.errorhandler(CrateNotFound)
+    def crate_not_found(e):
+        return jsonify({"error": str(e)}), 404
+
+    @app.errorhandler(AmbiguousCrate)
+    def ambiguous_crate(e):
+        return jsonify({"error": str(e)}), 409
+
+    @app.errorhandler(StorageError)
+    def storage_error(e):
+        logger.error("Storage error: %s", e)
+        return jsonify({"error": "Storage backend unavailable"}), 503
+
     # Integrate Celery
     make_celery(app)
 

app/ro_crates/routes/__init__.py

@@ -10,6 +10,6 @@
 # Always registered:
 v1_post_bp = post_routes_bp
 
-# Registered only when MinIO is enabled:
+# Registered only when object storage is enabled:
 v1_minio_post_bp = minio_post_routes_bp
 v1_minio_get_bp = get_routes_bp

app/ro_crates/routes/get_routes.py

@@ -1,62 +1,24 @@
-"""Defines get API endpoints for validating RO-Crates using their IDs from MinIO."""
+"""GET endpoint for retrieving a stored RO-Crate validation result by ID."""
 
-# Author: Alexander Hambley
-# License: MIT
-# Copyright (c) 2025 eScience Lab, The University of Manchester
-
-from apiflask import APIBlueprint, Schema
-from apiflask.fields import String, Boolean
-from marshmallow.fields import Nested
+from apiflask import APIBlueprint
 from flask import Response
 
 from app.services.validation_service import get_ro_crate_validation_task
 
 get_routes_bp = APIBlueprint("get_routes", __name__)
 
 
-class MinioConfig(Schema):
-    endpoint = String(required=True)
-    accesskey = String(required=True)
-    secret = String(required=True)
-    ssl = Boolean(required=True)
-    bucket = String(required=True)
-
-
-class ValidateResult(Schema):
-    minio_config = Nested(MinioConfig, required=True)
-    root_path = String(required=False)
-
-
 @get_routes_bp.get("<string:crate_id>/validation")
-@get_routes_bp.input(ValidateResult(partial=False), location='json')
-def get_ro_crate_validation_by_id(json_data, crate_id) -> tuple[Response, int]:
+def get_ro_crate_validation_by_id(crate_id) -> tuple[Response, int]:
     """
-    Endpoint to obtain an RO-Crate validation result using its ID from MinIO.
+    Obtain a stored RO-Crate validation result by its ID.
 
     Path Parameters:
     - **crate_id**: The RO-Crate ID. _Required_.
 
-    Request Body Parameters:
-    - **minio_config**: The MinIO bucket containing the RO-Crate. _Required_
-      - **endpoint**: Endpoint, e.g. 'localhost:9000'
-      - **accesskey**: Access key / username
-      - **secret**: Secret / password
-      - **ssl**: Use SSL encryption? True/False
-      - **bucket**: The MinIO bucket to access 
-    - **root_path**: The root path containing the RO-Crate. _Optional_
-
     Returns:
-    - A tuple containing the validation result and an HTTP status code.
-
-    Raises:
-    - KeyError: If required parameters (`crate_id`) are missing.
+    - A tuple containing the stored validation result and an HTTP status code.
+      Returns 404 if no result has been stored for the crate yet.
     """
 
-    minio_config = json_data["minio_config"]
-
-    if "root_path" in json_data:
-        root_path = json_data["root_path"]
-    else:
-        root_path = None
-
-    return get_ro_crate_validation_task(minio_config, crate_id, root_path)
+    return get_ro_crate_validation_task(crate_id)

app/ro_crates/routes/post_routes.py

@@ -1,8 +1,7 @@
-"""Defines post API endpoints for validating RO-Crates using their IDs from MinIO."""
+"""POST endpoints for validating RO-Crates by stored ID or by inline metadata."""
 
 from apiflask import APIBlueprint, Schema
-from apiflask.fields import String, Boolean
-from marshmallow.fields import Nested
+from apiflask.fields import String
 from flask import Response, current_app
 
 from app.services.validation_service import (
@@ -13,22 +12,12 @@
 # Always-on blueprint:
 post_routes_bp = APIBlueprint("post_routes", __name__)
 
-# MinIO blueprint. Only registered when MINIO_ENABLED is true
+# Store-backed blueprint. Only registered when storage is enabled
 # (see app.create_app), so the ID-based routes are unreachable by default.
 minio_post_routes_bp = APIBlueprint("minio_post_routes", __name__)
 
 
-class MinioConfig(Schema):
-    endpoint = String(required=True)
-    accesskey = String(required=True)
-    secret = String(required=True)
-    ssl = Boolean(required=True)
-    bucket = String(required=True)
-
-
 class ValidateCrate(Schema):
-    minio_config = Nested(MinioConfig, required=True)
-    root_path = String(required=False)
     profile_name = String(required=False)
     webhook_url = String(required=False)
 
@@ -42,51 +31,26 @@ class ValidateJSON(Schema):
 @minio_post_routes_bp.input(ValidateCrate(partial=False), location="json")
 def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]:
     """
-    Endpoint to validate an RO-Crate using its ID from MinIO.
+    Validate a stored RO-Crate by its ID.
+
+    Storage credentials and layout are configured server-side; the request body
+    carries only optional fields.
 
     Path Parameters:
     - **crate_id**: The RO-Crate ID. _Required_.
 
     Request Body Parameters:
-    - **minio_config**: The MinIO bucket containing the RO-Crate. _Required_
-      - **endpoint**: Endpoint, e.g. 'localhost:9000'
-      - **accesskey**: Access key / username
-      - **secret**: Secret / password
-      - **ssl**: Use SSL encryption? True/False
-      - **bucket**: The MinIO bucket to access
-    - **root_path**: The root path containing the RO-Crate. _Optional_
     - **profile_name**: The profile name for validation. _Optional_.
-    - **webhook_url**: The webhook URL where validation results will be sent. _Optional_.
+    - **webhook_url**: The webhook URL where the validation result will be sent. _Optional_.
 
     Returns:
     - A tuple containing the validation task's response and an HTTP status code.
-
-    Raises:
-    - KeyError: If required parameters (`crate_id` or `webhook_url`) are missing.
     """
 
-    minio_config = json_data["minio_config"]
-
-    if "root_path" in json_data:
-        root_path = json_data["root_path"]
-    else:
-        root_path = None
-
-    if "webhook_url" in json_data:
-        webhook_url = json_data["webhook_url"]
-    else:
-        webhook_url = None
+    profile_name = json_data.get("profile_name")
+    webhook_url = json_data.get("webhook_url")
 
-    if "profile_name" in json_data:
-        profile_name = json_data["profile_name"]
-    else:
-        profile_name = None
-
-    profiles_path = current_app.config["PROFILES_PATH"]
-
-    return queue_ro_crate_validation_task(
-        minio_config, crate_id, root_path, profile_name, webhook_url, profiles_path
-    )
+    return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url)
 
 
 @post_routes_bp.post("/validate_metadata")

app/services/validation_service.py

@@ -1,71 +1,60 @@
-"""Service methods to queue RO-Crates for validation using the CRS4 validator and Celery."""
+"""Service layer for RO-Crate validation requests."""
 
-import logging
 import json
+import logging
 
-from flask import jsonify, Response
-
-from app.tasks.validation_tasks import (
-    process_validation_task_by_id,
-    return_ro_crate_validation,
-    check_ro_crate_exists,
-    check_validation_exists,
-)
+from flask import jsonify, Response, current_app
 
+from app.crates.ids import validate_crate_id
+from app.crates.layout import result_key
+from app.crates.resolver import resolve_crate
+from app.storage.errors import ObjectNotFound
+from app.storage.s3 import S3Backend
+from app.tasks.validation_tasks import process_validation_task_by_id
 from app.utils.config import InvalidAPIUsage
-from app.utils.minio_utils import get_minio_client
-from app.validation.runner import validate_metadata
 from app.validation.results import ValidationStatus
+from app.validation.runner import validate_metadata
 
 logger = logging.getLogger(__name__)
 
 
+def _build_storage() -> S3Backend:
+    """Build the storage backend from server-side settings."""
+    settings = current_app.config["SETTINGS"]
+    return S3Backend.from_settings(settings)
+
+
 def queue_ro_crate_validation_task(
-    minio_config,
-    crate_id,
-    root_path=None,
-    profile_name=None,
-    webhook_url=None,
-    profiles_path=None,
+    crate_id: str, profile_name=None, webhook_url=None
 ) -> tuple[Response, int]:
     """
-    Queues an RO-Crate for validation with Celery.
+    Resolve a crate by ID and queue it for asynchronous validation.
+
+    Credentials and layout are server-side; the request carries only the ID and
+    optional profile/webhook. Resolution happens before queueing so a bad or
+    missing crate is reported immediately. ``InvalidCrateId`` / ``CrateNotFound``
+    / ``AmbiguousCrate`` / ``StorageError`` propagate to the app error handlers.
 
-    :param minio_config: Access settings for Minio instance containing the RO-Crate.
     :param crate_id: The ID of the RO-Crate to validate.
-    :param root_path: The root path containing the RO-Crate.
     :param profile_name: The profile to validate against.
-    :param webhook_url: The URL to POST the validation results to.
-    :return: A tuple containing a JSON response and an HTTP status code.
-    :raises: Exception: If an error occurs whilst queueing the task.
+    :param webhook_url: The URL to POST the validation result to.
+    :return: A JSON response and HTTP status code.
     """
+    settings = current_app.config["SETTINGS"]
+    storage = _build_storage()
 
-    logging.info(f"Processing: {crate_id}, {profile_name}, {webhook_url}")
-    logging.info(f"Minio Bucket: {minio_config['bucket']}; Root path: {root_path}")
-
-    minio_client = get_minio_client(minio_config)
+    # Raises if the crate is missing/ambiguous/invalid -> handled as 4xx.
+    resolve_crate(storage, crate_id, settings.s3_crate_prefix)
 
-    if check_ro_crate_exists(minio_client, minio_config["bucket"], crate_id, root_path):
-        logging.info("RO-Crate exists")
-    else:
-        logging.info("RO-Crate does not exist")
-        raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400)
-
-    try:
-        process_validation_task_by_id.delay(
-            minio_config, crate_id, root_path, profile_name, webhook_url, profiles_path
-        )
-        return jsonify({"message": "Validation in progress"}), 202
-
-    except Exception as e:
-        return jsonify({"error": str(e)}), 500
+    process_validation_task_by_id.delay(crate_id, profile_name, webhook_url)
+    return jsonify({"message": "Validation in progress"}), 202
 
 
 def run_metadata_validation(
     crate_json: str, profile_name=None, profiles_path=None
 ) -> tuple[Response, int]:
     """
-    Validates RO-Crate metadata synchronously and returns the result inline.
+    Validate RO-Crate metadata synchronously and return the result inline.
 
     Metadata-only validation is fast and stateless, so it runs in the request
     rather than via Celery. Returns 200 for a valid/invalid outcome and 422
@@ -74,9 +63,8 @@ def run_metadata_validation(
     :param crate_json: The RO-Crate JSON-LD metadata, as a string.
     :param profile_name: The profile to validate against.
     :param profiles_path: A path to the profile definition directory.
-    :return: A tuple containing a JSON response and an HTTP status code.
+    :return: A JSON response and HTTP status code.
     """
-
     if not crate_json:
         return jsonify({"error": "Missing required parameter: crate_json"}), 422
 
@@ -95,41 +83,25 @@ def run_metadata_validation(
     return jsonify(outcome.to_dict()), status_code
 
 
-def get_ro_crate_validation_task(
-    minio_config: dict,
-    crate_id: str,
-    root_path: str,
-) -> tuple[Response, int]:
+def get_ro_crate_validation_task(crate_id: str) -> tuple[Response, int]:
     """
-    Retrieves an RO-Crate validation result.
+    Return a crate's stored validation result.
 
-    :param minio_config: Access settings for Minio instance containing the RO-Crate.
-    :param crate_id: The ID of the RO-Crate to validate.
-    :param root_path: The root path containing the RO-Crate.
-    :return: A tuple containing a JSON response and an HTTP status code.
-    :raises Exception: If an error occurs whilst retreiving validation result
+    Reads the result object from the results prefix. A missing result yields a
+    404; the stored outcome (including a persisted ``error`` outcome) is returned
+    as-is otherwise.
+
+    :param crate_id: The ID of the RO-Crate whose result is requested.
+    :return: A JSON response and HTTP status code.
     """
-    logging.info(f"Retrieving validation for: {crate_id}")
-
-    minio_client = get_minio_client(minio_config)
-
-    if check_ro_crate_exists(minio_client, minio_config["bucket"], crate_id, root_path):
-        logging.info("RO-Crate exists")
-    else:
-        logging.info("RO-Crate does not exist")
-        raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400)
-
-    if check_validation_exists(
-        minio_client, minio_config["bucket"], crate_id, root_path
-    ):
-        logging.info("Validation result exists")
-    else:
-        logging.info("Validation does not exist")
-        raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {crate_id}", 400)
-
-    return (
-        return_ro_crate_validation(
-            minio_client, minio_config["bucket"], crate_id, root_path
-        ),
-        200,
-    )
+    settings = current_app.config["SETTINGS"]
+    storage = _build_storage()
+
+    validate_crate_id(crate_id)  # raises InvalidCrateId -> 400
+
+    try:
+        data = storage.get_bytes(result_key(settings.s3_results_prefix, crate_id))
+    except ObjectNotFound:
+        raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {crate_id}", 404)
+
+    return jsonify(json.loads(data)), 200