feat: add /healthz and /readyz endpoints

- Adds endpoints to check if the S3 is reachable and Celery / broker connected;
- Returns 503 with a per-check breakdown when a dependency is down.
- When storage is disabled both checks report "disabled".

Closes #181

Author: alexhambley

app/__init__.py

@@ -6,6 +6,7 @@
 
 from app.crates.ids import InvalidCrateId
 from app.crates.resolver import CrateNotFound, AmbiguousCrate
+from app.health import health_bp
 from app.ro_crates.routes import v1_post_bp, v1_minio_post_bp, v1_minio_get_bp
 from app.services.logging_service import (
     new_request_id,
@@ -48,6 +49,7 @@ def create_app(settings: Settings | None = None) -> APIFlask:
     app.config["PROFILES_PATH"] = settings.profiles_path
 
     # Always available:
+    app.register_blueprint(health_bp)
     app.register_blueprint(v1_post_bp, url_prefix="/v1/ro_crates")
 
     # Object storage is optional and disabled by default. Only register the

app/health.py

@@ -0,0 +1,70 @@
+"""Liveness and readiness endpoints for orchestration.
+
+``/healthz`` reports that the process is up. ``/readyz`` reports whether the
+service can actually serve S3 requests, by checking the object store and the
+Celery broker. When storage is disabled, those dependencies are not required
+and report ``disabled``.
+"""
+
+import logging
+
+from apiflask import APIBlueprint
+from flask import current_app, jsonify
+
+from app.storage.s3 import S3Backend
+
+logger = logging.getLogger(__name__)
+
+health_bp = APIBlueprint("health", __name__)
+
+# Short connection timeout (seconds) so readiness checks fail quickly.
+_BROKER_TIMEOUT = 3
+
+
+def check_storage(settings) -> tuple[bool, str]:
+    """Return whether the object store is reachable, with a detail string."""
+    if not settings.storage_enabled:
+        return True, "disabled"
+    try:
+        S3Backend.from_settings(settings).health_check()
+        return True, "ok"
+    except Exception as error:  # noqa: BLE001 - any failure means not ready
+        logger.warning("Storage readiness check failed: %s", error)
+        return False, str(error)
+
+
+def check_broker(settings) -> tuple[bool, str]:
+    """Return whether the Celery broker is reachable, with a detail string."""
+    if not settings.storage_enabled:
+        return True, "disabled"
+    try:
+        from kombu import Connection
+
+        with Connection(settings.celery_broker_url) as connection:
+            connection.ensure_connection(max_retries=1, timeout=_BROKER_TIMEOUT)
+        return True, "ok"
+    except Exception as error:  # noqa: BLE001 - any failure means not ready
+        logger.warning("Broker readiness check failed: %s", error)
+        return False, str(error)
+
+
+@health_bp.get("/healthz")
+def healthz():
+    """Liveness: the process is running."""
+    return jsonify({"status": "ok"}), 200
+
+
+@health_bp.get("/readyz")
+def readyz():
+    """Readiness: dependencies needed to serve requests are reachable."""
+    settings = current_app.config["SETTINGS"]
+
+    storage_ok, storage_detail = check_storage(settings)
+    broker_ok, broker_detail = check_broker(settings)
+    ready = storage_ok and broker_ok
+
+    body = {
+        "status": "ready" if ready else "not ready",
+        "checks": {"storage": storage_detail, "broker": broker_detail},
+    }
+    return jsonify(body), (200 if ready else 503)

app/ro_crates/routes/__init__.py

@@ -1,9 +1,5 @@
 """Defines main Blueprint and registers sub-Blueprints for organising related routes."""
 
-# Author: Alexander Hambley
-# License: MIT
-# Copyright (c) 2025 eScience Lab, The University of Manchester
-
 from app.ro_crates.routes.post_routes import post_routes_bp, minio_post_routes_bp
 from app.ro_crates.routes.get_routes import get_routes_bp
 

app/storage/s3.py

@@ -92,6 +92,15 @@ def download_tree(self, prefix: str, dest_dir: str) -> None:
             with open(local_path, "wb") as handle:
                 handle.write(self.get_bytes(key))
 
+    def health_check(self) -> None:
+        """Verify the bucket is reachable; raise ``StorageError`` if not."""
+        try:
+            self._client.head_bucket(Bucket=self.bucket)
+        except (ClientError, BotoCoreError) as error:
+            raise StorageError(
+                f"Bucket {self.bucket} not reachable: {error}"
+            ) from error
+
     @staticmethod
     def _translate(error: ClientError, key: str) -> StorageError:
         """Map a botocore ClientError to the storage error vocabulary.

tests/test_health.py

@@ -0,0 +1,71 @@
+"""Tests for the health and readiness endpoints."""
+
+from unittest import mock
+
+import pytest
+
+from app import create_app
+from app.utils.config import Settings
+
+
+def _storage_env() -> dict:
+    return {
+        "STORAGE_ENABLED": "true",
+        "S3_ENDPOINT": "minio:9000",
+        "S3_ACCESS_KEY": "a",
+        "S3_SECRET_KEY": "b",
+        "S3_BUCKET": "ro-crates",
+        "CELERY_BROKER_URL": "redis://r/0",
+        "CELERY_RESULT_BACKEND": "redis://r/1",
+    }
+
+
+@pytest.fixture
+def disabled_client():
+    return create_app(settings=Settings.from_env({})).test_client()
+
+
+@pytest.fixture
+def storage_client():
+    return create_app(settings=Settings.from_env(_storage_env())).test_client()
+
+
+def test_healthz_is_always_ok(disabled_client):
+    response = disabled_client.get("/healthz")
+    assert response.status_code == 200
+    assert response.json["status"] == "ok"
+
+
+def test_readyz_ready_when_storage_disabled(disabled_client):
+    response = disabled_client.get("/readyz")
+    assert response.status_code == 200
+    assert response.json["status"] == "ready"
+    assert response.json["checks"]["storage"] == "disabled"
+
+
+def test_readyz_ok_when_all_checks_pass(storage_client):
+    with mock.patch("app.health.check_storage", return_value=(True, "ok")), \
+         mock.patch("app.health.check_broker", return_value=(True, "ok")):
+        response = storage_client.get("/readyz")
+
+    assert response.status_code == 200
+    assert response.json["status"] == "ready"
+
+
+def test_readyz_503_when_storage_unreachable(storage_client):
+    with mock.patch("app.health.check_storage", return_value=(False, "bucket down")), \
+         mock.patch("app.health.check_broker", return_value=(True, "ok")):
+        response = storage_client.get("/readyz")
+
+    assert response.status_code == 503
+    assert response.json["status"] == "not ready"
+    assert response.json["checks"]["storage"] == "bucket down"
+
+
+def test_readyz_503_when_broker_unreachable(storage_client):
+    with mock.patch("app.health.check_storage", return_value=(True, "ok")), \
+         mock.patch("app.health.check_broker", return_value=(False, "broker down")):
+        response = storage_client.get("/readyz")
+
+    assert response.status_code == 503
+    assert response.json["checks"]["broker"] == "broker down"

tests/test_storage_s3.py

@@ -78,6 +78,16 @@ def test_non_missing_client_error_becomes_storage_error(s3_backend):
     assert not isinstance(exc_info.value, ObjectNotFound)
 
 
+def test_health_check_passes_for_existing_bucket(s3_backend):
+    s3_backend.health_check()  # must not raise
+
+
+def test_health_check_fails_for_missing_bucket(s3_backend):
+    broken = S3Backend(s3_backend._client, "nonexistent-bucket")
+    with pytest.raises(StorageError):
+        broken.health_check()
+
+
 def test_s3_backend_satisfies_protocol(s3_backend):
     assert isinstance(s3_backend, StorageBackend)