Merge pull request #161 from eScienceLab/feature/optional-minio-156

feat: Make MinIO object storage optional (#156)

Author: alexhambley

README.md

@@ -2,6 +2,19 @@
 
 This project presents a Flask-based API for validating RO-Crates.
 
+### Optional MinIO object storage
+
+The RO-Crate Validation Service can validate an RO-Crate's metadata directly from a JSON payload (the `POST v1/ro_crates/validate_metadata` endpoint) without storing anything. This is the default mode.
+
+Optionally, the service can read crates from — and write validation results
+back to — a [MinIO](https://min.io/) object store. This is disabled by
+default and controlled by the `MINIO_ENABLED` environment variable:
+
+- `MINIO_ENABLED=false` (default): only a stateless validation endpoint is available and nothing is stored.
+- `MINIO_ENABLED=true`: the ID endpoints (`POST`/`GET v1/ro_crates/{crate_id}/validation`) are also registered, and a MinIO instance is required. With Docker Compose, start MinIO with its opt-in profile: `docker compose --profile minio up`.
+
+When MinIO is disabled the ID-based endpoints are not registered and return `404`.
+
 ## API
 
 #### Request Validation of RO-Crate
@@ -184,8 +197,14 @@ curl -X 'POST' \
     ```bash
    docker compose up --build
    ```
+   This runs in the default (metadata-only) mode. To enable the MinIO-backed
+   endpoints, set `MINIO_ENABLED=true` in your `.env` and start the `minio`
+   profile:
+    ```bash
+   docker compose --profile minio up --build
+   ```
 
-5. Set up the MinIO bucket
+5. **(Only when `MINIO_ENABLED=true`)** Set up the MinIO bucket
    1. Open the MinIO web interface at `http://localhost:9000`.  
    2. Log in with your MinIO credentials.  
    3. Create a new bucket named `ro-crates`.  

app/__init__.py

@@ -4,14 +4,22 @@
 # License: MIT
 # Copyright (c) 2025 eScience Lab, The University of Manchester
 
+import logging
 import os
 
 from apiflask import APIFlask
 
-from app.ro_crates.routes import v1_post_bp, v1_get_bp
-from app.utils.config import DevelopmentConfig, ProductionConfig, InvalidAPIUsage, make_celery
+from app.ro_crates.routes import v1_post_bp, v1_minio_post_bp, v1_minio_get_bp
+from app.utils.config import (
+    DevelopmentConfig,
+    ProductionConfig,
+    InvalidAPIUsage,
+    make_celery,
+)
 from flask import jsonify
 
+logger = logging.getLogger(__name__)
+
 
 def create_app() -> APIFlask:
     """
@@ -20,23 +28,36 @@ def create_app() -> APIFlask:
     :return: Flask: A configured Flask application instance.
     """
     app = APIFlask(__name__)
-    app.register_blueprint(v1_post_bp, url_prefix="/v1/ro_crates")
-    app.register_blueprint(v1_get_bp, url_prefix="/v1/ro_crates")
 
-    @app.errorhandler(InvalidAPIUsage)
-    def invalid_api_usage(e):
-        return jsonify(e.to_dict()), e.status_code
-
-    # Load configuration:
+    # Load config before registering blueprints, so MINIO_ENABLED can
+    # decide whether the backed endpoints are exposed.
     if os.getenv("FLASK_ENV") == "production":
         app.config.from_object(ProductionConfig)
     else:
         # Development environment:
         app.debug = True
+        app.config.from_object(DevelopmentConfig)
+
+    # Always available:
+    app.register_blueprint(v1_post_bp, url_prefix="/v1/ro_crates")
+
+    # MinIO is optional and disabled by default. Only register
+    # the MinIO ID routes when enabled:
+    if app.config.get("MINIO_ENABLED"):
+        app.register_blueprint(v1_minio_post_bp, url_prefix="/v1/ro_crates")
+        app.register_blueprint(v1_minio_get_bp, url_prefix="/v1/ro_crates")
+        logger.info("MinIO storage enabled: ID-based validation endpoints registered.")
+    else:
+        logger.info("MinIO storage disabled: only metadata validation is available.")
+
+    if app.debug:
         print("URL Map:")
         for rule in app.url_map.iter_rules():
             print(rule)
-        app.config.from_object(DevelopmentConfig)
+
+    @app.errorhandler(InvalidAPIUsage)
+    def invalid_api_usage(e):
+        return jsonify(e.to_dict()), e.status_code
 
     # Integrate Celery
     make_celery(app)

app/ro_crates/routes/__init__.py

@@ -4,8 +4,12 @@
 # License: MIT
 # Copyright (c) 2025 eScience Lab, The University of Manchester
 
-from app.ro_crates.routes.post_routes import post_routes_bp
+from app.ro_crates.routes.post_routes import post_routes_bp, minio_post_routes_bp
 from app.ro_crates.routes.get_routes import get_routes_bp
 
+# Always registered:
 v1_post_bp = post_routes_bp
-v1_get_bp = get_routes_bp
+
+# Registered only when MinIO is enabled:
+v1_minio_post_bp = minio_post_routes_bp
+v1_minio_get_bp = get_routes_bp

app/ro_crates/routes/post_routes.py

@@ -11,11 +11,16 @@
 
 from app.services.validation_service import (
     queue_ro_crate_validation_task,
-    queue_ro_crate_metadata_validation_task
+    queue_ro_crate_metadata_validation_task,
 )
 
+# Always-on blueprint:
 post_routes_bp = APIBlueprint("post_routes", __name__)
 
+# MinIO blueprint. Only registered when MINIO_ENABLED is true
+# (see app.create_app), so the ID-based routes are unreachable by default.
+minio_post_routes_bp = APIBlueprint("minio_post_routes", __name__)
+
 
 class MinioConfig(Schema):
     endpoint = String(required=True)
@@ -37,8 +42,8 @@ class ValidateJSON(Schema):
     profile_name = String(required=False)
 
 
-@post_routes_bp.post("<string:crate_id>/validation")
-@post_routes_bp.input(ValidateCrate(partial=False), location='json')
+@minio_post_routes_bp.post("<string:crate_id>/validation")
+@minio_post_routes_bp.input(ValidateCrate(partial=False), location="json")
 def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]:
     """
     Endpoint to validate an RO-Crate using its ID from MinIO.
@@ -52,7 +57,7 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]:
       - **accesskey**: Access key / username
       - **secret**: Secret / password
       - **ssl**: Use SSL encryption? True/False
-      - **bucket**: The MinIO bucket to access 
+      - **bucket**: The MinIO bucket to access
     - **root_path**: The root path containing the RO-Crate. _Optional_
     - **profile_name**: The profile name for validation. _Optional_.
     - **webhook_url**: The webhook URL where validation results will be sent. _Optional_.
@@ -83,12 +88,13 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]:
 
     profiles_path = current_app.config["PROFILES_PATH"]
 
-    return queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name,
-                                          webhook_url, profiles_path)
+    return queue_ro_crate_validation_task(
+        minio_config, crate_id, root_path, profile_name, webhook_url, profiles_path
+    )
 
 
 @post_routes_bp.post("/validate_metadata")
-@post_routes_bp.input(ValidateJSON(partial=False), location='json')  # -> json_data
+@post_routes_bp.input(ValidateJSON(partial=False), location="json")  # -> json_data
 def validate_ro_crate_metadata(json_data) -> tuple[Response, int]:
     """
     Endpoint to validate an RO-Crate JSON file uploaded to the Service.
@@ -113,4 +119,6 @@ def validate_ro_crate_metadata(json_data) -> tuple[Response, int]:
 
     profiles_path = current_app.config["PROFILES_PATH"]
 
-    return queue_ro_crate_metadata_validation_task(crate_json, profile_name, profiles_path=profiles_path)
+    return queue_ro_crate_metadata_validation_task(
+        crate_json, profile_name, profiles_path=profiles_path
+    )

app/utils/config.py

@@ -17,6 +17,13 @@ def get_env(name: str, default=None, required=False):
     return value
 
 
+def get_bool_env(name: str, default: bool = False) -> bool:
+    value = get_env(name)
+    if value is None:
+        return default
+    return value.strip().lower() in ("true", "1", "yes", "on")
+
+
 class Config:
     """Base configuration class for the Flask application."""
 
@@ -27,14 +34,20 @@ class Config:
     # rocrate validator configuration:
     PROFILES_PATH = get_env("PROFILES_PATH", required=False)
 
+    # Optional MinIO storage. Disabled by default - when False the
+    # ID validation endpoints are not registered:
+    MINIO_ENABLED = get_bool_env("MINIO_ENABLED", default=False)
+
 
 class DevelopmentConfig(Config):
     """Development configuration class."""
+
     DEBUG = True
 
 
 class ProductionConfig(Config):
     """Production configuration class."""
+
     DEBUG = False
 
 
@@ -50,7 +63,7 @@ def __init__(self, message, status_code=None, payload=None):
 
     def to_dict(self):
         rv = dict(self.payload or ())
-        rv['message'] = self.message
+        rv["message"] = self.message
         return rv
 
 

docker-compose-develop.yml

@@ -12,14 +12,16 @@ services:
       - FLASK_ENV=development
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      # Optional object storage. Set MINIO_ENABLED=true and start the "minio"
+      # profile (docker compose --profile minio up) to use 
+      - MINIO_ENABLED=${MINIO_ENABLED:-false}
       - MINIO_ENDPOINT=${MINIO_ENDPOINT}
       - MINIO_ROOT_USER=${MINIO_ROOT_USER}
       - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
       - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME}
       - PROFILES_PATH=/app/profiles
     depends_on:
       - redis
-      - minio
 
   celery_worker:
     build:
@@ -29,9 +31,9 @@ services:
     environment:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - MINIO_ENABLED=${MINIO_ENABLED:-false}
     depends_on:
       - redis
-      - minio
     volumes:
       - ./tests/data/rocrate_validator_profiles:/app/profiles:ro
 
@@ -42,6 +44,9 @@ services:
 
   minio:
     image: "minio/minio"
+    # Started with `docker compose --profile minio up`.
+    profiles:
+      - minio
     ports:
       - "9000:9000"
       - "9001:9001"

docker-compose.yml

@@ -11,13 +11,15 @@ services:
       - FLASK_ENV=development
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      # Optional object storage. Set MINIO_ENABLED=true and start the "minio"
+      # profile (docker compose --profile minio up) to use 
+      - MINIO_ENABLED=${MINIO_ENABLED:-false}
       - MINIO_ENDPOINT=${MINIO_ENDPOINT}
       - MINIO_ROOT_USER=${MINIO_ROOT_USER}
       - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
       - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME}
     depends_on:
       - redis
-      - minio
 
   celery_worker:
     platform: linux/x86_64
@@ -26,9 +28,9 @@ services:
     environment:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - MINIO_ENABLED=${MINIO_ENABLED:-false}
     depends_on:
       - redis
-      - minio
 
   redis:
     image: "redis:alpine"
@@ -37,6 +39,9 @@ services:
 
   minio:
     image: "minio/minio"
+    # Started with `docker compose --profile minio up`.
+    profiles:
+      - minio
     ports:
       - "9000:9000"
       - "9001:9001"

example.env

@@ -1,3 +1,8 @@
+# MinIO is off by default; only the stateless validation endpoint is exposed when 
+# disabled. The MINIO_* vars below and the "minio" docker-compose profile are 
+#only needed when this is true.
+MINIO_ENABLED=false
+
 MINIO_ROOT_USER=minioadmin
 MINIO_ROOT_PASSWORD=minioadmin
 MINIO_BUCKET_NAME=ro-crates