Merge pull request #186 from eScienceLab/refactor-production-readiness

Production-readiness refactor: drop MinIO, make Crate IDs more robust, add async validation and offline cache

Author: alexhambley

.github/workflows/build-with-profiles.yml

@@ -19,33 +19,38 @@ jobs:
       id-token: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build and push Docker image
         id: push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
         with:
           context: .
-          file: ./Dockerfile.fivesafes-profile
           push: true
+          # Add the five-safes profile (via EXTRA_PROFILES_PATH) and warm its
+          # cache, using the shared Dockerfile's build args.
+          build-args: |
+            FIVE_SAFES_PROFILE_VERSION=five-safes-0.7.4-beta
+            PROFILES_ARCHIVE_URL=https://github.com/eScienceLab/rocrate-validator/archive/refs/tags/five-safes-0.7.4-beta.tar.gz
+            EXTRA_PROFILES_PATH=/app/extra-profiles
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
       - name: Generate artifact attestation
-        uses: actions/attest-build-provenance@v2
+        uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2.4.0
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
           subject-digest: ${{ steps.push.outputs.digest }}

.github/workflows/build.yml

@@ -21,33 +21,33 @@ jobs:
       id-token: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
       - name: Log in to the Container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
       - name: Build and push Docker image
         id: push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
         with:
           context: .
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
       # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds).
       - name: Generate artifact attestation
-        uses: actions/attest-build-provenance@v2
+        uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2.4.0
         with:
           subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
           subject-digest: ${{ steps.push.outputs.digest }}

.github/workflows/lint.yml

@@ -0,0 +1,30 @@
+name: Lint
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.11'
+
+      - name: Install ruff
+        run: |
+          python -m pip install --upgrade pip
+          # Keep in sync with requirements-dev.txt
+          pip install ruff==0.15.17
+
+      - name: Lint
+        run: ruff check .
+
+      - name: Format check
+        run: ruff format --check .

.github/workflows/test_docker.yml

@@ -10,26 +10,26 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: '3.11'
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest requests minio docker
+          # Keep in sync with requirements-dev.txt
+          pip install pytest==9.1.0 requests==2.33.1 boto3==1.43.29
 
-      - name: Build Docker Compose Containers
+      - name: Run integration tests (brings up the compose stack)
         run: |
           cp example.env .env
-          docker compose -f docker-compose-develop.yml build 
+          pytest -s -v tests/test_integration.py
 
-      - name: Spin Up Docker Compose and Run Tests
-        run: pytest -s -v tests/test_integration.py
-
-      - name: Ensure that Docker Compose is Shutdown
+      - name: Ensure Docker Compose is shut down
         if: always()
-        run: docker compose down
+        run: >
+          docker compose -f docker-compose-develop.yml -p cratey_integration
+          --profile objectstore down -v || true

.github/workflows/unit_tests.yml

@@ -10,18 +10,17 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: '3.11'
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install pytest pytest-mock
+          pip install -r requirements-dev.txt
 
       - name: Run tests (excluding integration tests)
         run: |

Dockerfile

@@ -1,17 +1,53 @@
+# syntax=docker/dockerfile:1
 FROM python:3.11-slim
 
-# Install required system packages, including git
-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+# git is needed by some dependencies; wget is only used when baking a profile.
+RUN apt-get update && apt-get install -y git wget && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
 COPY requirements.txt .
 RUN pip install --upgrade pip
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY cratey.py LICENSE /app/
+COPY wsgi.py LICENSE /app/
 COPY app /app/app
 
+# Optionally fetch an extra RO-Crate profile into a normal directory. It is
+# *added* to the bundled profiles at runtime via EXTRA_PROFILES_PATH.
+# A plain build leaves PROFILES_ARCHIVE_URL empty
+# and skips this; the "with profiles" image build passes it as --build-arg.
+ARG PROFILES_ARCHIVE_URL=""
+ARG FIVE_SAFES_PROFILE_VERSION=""
+# Set EXTRA_PROFILES_PATH only for the profiles build (passed as a build arg).
+ARG EXTRA_PROFILES_PATH=""
+ENV EXTRA_PROFILES_PATH=${EXTRA_PROFILES_PATH}
+ENV CACHE_PATH=/app/.rocrate-cache
+RUN <<EOF_PROFILES
+set -e
+if [ -n "$PROFILES_ARCHIVE_URL" ]; then
+    mkdir -p /app/extra-profiles
+    wget -O /tmp/profiles.tar.gz "$PROFILES_ARCHIVE_URL"
+    tar -xzf /tmp/profiles.tar.gz \
+        -C /app/extra-profiles \
+        --strip-components=3 \
+        "rocrate-validator-${FIVE_SAFES_PROFILE_VERSION}/rocrate_validator/profiles/five-safes-crate"
+    rm /tmp/profiles.tar.gz
+fi
+EOF_PROFILES
+
+# Pre-populate the HTTP cache so opt-in offline validation
+# (VALIDATION_OFFLINE=true) works without network at runtime.
+RUN <<EOF_CACHE_WARM
+set -e
+if [ -n "$EXTRA_PROFILES_PATH" ]; then
+    rocrate-validator cache warm --all-profiles \
+        --extra-profiles-path "$EXTRA_PROFILES_PATH" --cache-path "$CACHE_PATH"
+else
+    rocrate-validator cache warm --all-profiles --cache-path "$CACHE_PATH"
+fi
+EOF_CACHE_WARM
+
 RUN useradd -ms /bin/bash flaskuser
 RUN chown -R flaskuser:flaskuser /app
 
@@ -21,4 +57,5 @@ EXPOSE 5000
 
 CMD ["flask", "run", "--host=0.0.0.0"]
 
-LABEL org.opencontainers.image.source="https://github.com/eScienceLab/Cratey-Validator"
+LABEL org.opencontainers.image.source="https://github.com/eScienceLab/RO-Crate-Validation-Service"
+LABEL org.ro-crate-validation-service.five-safes-profile-version="${FIVE_SAFES_PROFILE_VERSION}"