test_publish.yml

name: Test, lint and publish

on:
  push:
    branches:
      - main
    tags:
      - "v*"
  pull_request:
    branches:
concurrency:
  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
  cancel-in-progress: true
defaults:
  run:
    shell: bash
env:
  DEVCONTAINER_REGISTRY: ghcr.io
  DEVCONTAINER_IMAGE_NAME: ${{ github.repository }}/devcontainer
  DEVCONTAINER_IMAGE_TAG_MAIN: "cache-main"
  CACHE_KEY: main-branch-poetry-cache-ubuntu
  CACHE_PATH: .devcontainer/.poetry_cache

jobs:
  build-devcontainer:
    runs-on: ubuntu-latest
    outputs:
      image_repository: ${{ steps.docker_image.outputs.image_repository }}
      image_tag: ${{ steps.docker_image.outputs.image_tag }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - name: Docker image metadata
        id: meta
        uses: docker/metadata-action@v6
        with:
          images: ${{ env.DEVCONTAINER_REGISTRY }}/${{ env.DEVCONTAINER_IMAGE_NAME }}
          tags: |
            type=ref,event=pr,prefix=cache-pr-,priority=600
            type=ref,event=branch,prefix=cache-,priority=500
            type=ref,event=tag,prefix=cache-,priority=500
          flavor: |
            latest=false
      - name: Extract Docker image name
        id: docker_image
        env:
          IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
        run: |
          IMAGE=$(echo "$IMAGE_TAGS" | cut -d" " -f1)
          IMAGE_REPOSITORY=$(echo "$IMAGE" | cut -d":" -f1)
          IMAGE_TAG=$(echo "$IMAGE" | cut -d":" -f2)
          echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
          echo "image_repository=$IMAGE_REPOSITORY" >> "$GITHUB_OUTPUT"
          echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
      - uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Pre-build devcontainer
        uses: devcontainers/ci@v0.3
        continue-on-error: true
        with:
          push: always
          skipContainerUserIdUpdate: false
          imageName: ${{ steps.docker_image.outputs.image_repository }}
          imageTag: ${{ steps.docker_image.outputs.image_tag }}
          cacheFrom: |
            ${{ steps.docker_image.outputs.image_repository }}:${{ steps.docker_image.outputs.image_tag }}
            ${{ steps.docker_image.outputs.image_repository }}:${{ env.DEVCONTAINER_IMAGE_TAG_MAIN }}
  style-checks:
    runs-on: ubuntu-latest
    needs:
      - build-devcontainer
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: actions/cache/restore@v5
        name: Restore cache
        with:
          path: ${{ env.CACHE_PATH }}
          key: ${{ env.CACHE_KEY }}
      - name: Set Git config
        shell: bash
        run: |
          git config --add user.name "Renku Bot"
          git config --add user.email "renku@datascience.ch"
      - name: Style checks
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make style_checks && poetry run pre-commit run --all-files --show-diff-on-failure --hook-stage manual
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
  test-main:
    runs-on: ubuntu-latest
    needs:
      - build-devcontainer
      - style-checks
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: actions/cache/restore@v5
        name: Restore cache
        with:
          path: ${{ env.CACHE_PATH }}
          key: ${{ env.CACHE_KEY }}
      - name: Set Git config
        shell: bash
        run: |
          git config --add user.name "Renku Bot"
          git config --add user.email "renku@datascience.ch"
      - name: Test setup
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make test_setup
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Main tests
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make main_tests
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Combine coverage data
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make collect_coverage
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Coveralls Parallel
        uses: coverallsapp/github-action@v2
        continue-on-error: true
        with:
          flag-name: run-main-tests
          parallel: true
          path-to-lcov: coverage.lcov
  test-schemathesis:
    runs-on: ubuntu-latest
    needs:
      - build-devcontainer
      - style-checks
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: actions/cache/restore@v5
        name: Restore cache
        with:
          path: ${{ env.CACHE_PATH }}
          key: ${{ env.CACHE_KEY }}
      - name: Set Git config
        shell: bash
        run: |
          git config --add user.name "Renku Bot"
          git config --add user.email "renku@datascience.ch"
      - name: Test setup
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make test_setup
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Schemathesis Tests
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make schemathesis_tests
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
          env: |
            HYPOTHESIS_PROFILE=ci
      - name: Combine coverage data
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make collect_coverage
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Coveralls Parallel
        uses: coverallsapp/github-action@v2
        continue-on-error: true
        with:
          flag-name: run-schemathesis-tests
          parallel: true
          path-to-lcov: coverage.lcov
  coveralls:
    needs:
      - test-main
      - test-schemathesis
    runs-on: ubuntu-latest
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@v2
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          COVERALLS_SERVICE_NAME: gihub-action
        with:
          parallel-finished: true
          carryforward: "run-main-tests,run-schemathesis-tests"
        continue-on-error: true

  publish:
    runs-on: ubuntu-latest
    needs:
      - test-main
      - test-schemathesis
    strategy:
      matrix:
        include:
          - service: "Data Service API"
            image-name: renku/renku-data-service
            dockerfile: ./projects/renku_data_service/Dockerfile
            name: renku-data-services
          - service: "Secrets Storage"
            image-name: renku/secrets-storage
            dockerfile: ./projects/secrets_storage/Dockerfile
            name: secrets-storage
          - service: "K8s Watcher"
            image-name: renku/data-service-k8s-watcher
            dockerfile: ./projects/k8s_watcher/Dockerfile
            name: k8s-watcher
          - service: "Data Tasks"
            image-name: renku/data-service-data-tasks
            dockerfile: ./projects/renku_data_tasks/Dockerfile
            name: data-tasks
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Docker meta ${{ matrix.service }}
        id: meta
        uses: docker/metadata-action@v6
        with:
          # list of Docker images to use as base name for tags
          images: |
            ${{ matrix.image-name }}
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.RENKU_DOCKER_USERNAME }}
          password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
      - name: Build and push ${{ matrix.service }}
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ${{ matrix.dockerfile }}
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=registry,ref=${{ matrix.image-name }}:buildcache
          cache-to: type=registry,ref=${{ matrix.image-name}}:buildcache,mode=max
          provenance: false
      - name: Scan image ${{ matrix.service }}
        uses: snyk/actions/docker@master
        continue-on-error: true
        with:
          image: ${{ steps.meta.outputs.tags}}
          args: --file=${{ matrix.dockerfile }} --severity-threshold=high --sarif-file-output=${{ matrix.name }}.sarif
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
      - name: Upload Snyk report
        uses: github/codeql-action/upload-sarif@v4
        continue-on-error: true
        with:
          sarif_file: ${{ matrix.name }}.sarif
          category: snyk_${{ matrix.name }}