ci.yml

name: CI tests
on:
  pull_request:

  push:
    branches:
      - main

env:
  FORCE_COLOR: true
  UV_VERSION: "0.11.8"
  NODE_VERSION: "22.22"

concurrency:
  group: ${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  pre-commit:
    name: Run linters and other pre-commit hooks
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6

      - name: Set up Python 3.10
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"

      - name: Set up uv
        uses: astral-sh/setup-uv@v7
        with:
          version: ${{ env.UV_VERSION }}
          enable-cache: true

      - name: Install dependencies
        working-directory: ./pydatalab
        run: |
          uv sync --locked --all-extras --dev

      - name: Run pre-commit
        working-directory: ./pydatalab
        run: |
          uv run pre-commit run --all-files --show-diff-on-failure

  pytest:
    name: Run Python unit tests

    services:
      mongo:
        image: mongo:8
        ports:
          - 27017:27017

    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        python-version: ["3.10", "3.11"]

    steps:
      - uses: actions/checkout@v6
        with:
          # tests need an unshallowed version of the repository to check the version
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install MongoDB tools binaries
        run: |
          wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.9.0.deb && sudo apt install ./mongodb-database-tools-*-100.9.0.deb

      - name: Set up uv
        uses: astral-sh/setup-uv@v7
        with:
          version: ${{ env.UV_VERSION }}
          working-directory: "./pydatalab"
          enable-cache: true

      - name: Install locked versions of dependencies
        working-directory: ./pydatalab
        run: |
          uv sync --locked --all-extras --dev

      - name: Run all tests
        working-directory: ./pydatalab
        run: |
          uv run pytest -rs --cov=pydatalab --cov-report=term --cov-report=xml -vvv ./tests

      - name: Upload coverage to Codecov
        if: matrix.python-version == '3.10' && github.repository == 'datalab-org/datalab'
        uses: codecov/codecov-action@v6
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: datalab-org/datalab

  webapp:
    name: Test the app build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Set up Node
        uses: actions/setup-node@v6
        env:
          FORCE_COLOR: 0
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "yarn"
          cache-dependency-path: ./webapp/yarn.lock

      - name: Install web app
        working-directory: ./webapp
        run: yarn install --frozen-lockfile

      - name: Build web app
        working-directory: ./webapp
        run: yarn build

  plugins:
    name: Test plugin builds
    runs-on: ubuntu-latest
    steps:

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Set up uv
        uses: astral-sh/setup-uv@v7
        with:
          version: ${{ env.UV_VERSION }}
          enable-cache: false
          working-directory: "./pydatalab"

      - name: Clone and mount template plugin to test local plugin installation
        run: |
          mkdir -p pydatalab/plugins

          cp .github/workflows/plugins.ci-template.toml plugins.toml

          # Initialise copier with default/dummy options
          uvx copier@9.8.0 copy -l \
            -d author_email=test@test.com  \
            -d author_fullname=test \
            -d project_name=datalab-app-plugin-template \
            -d repository_namespace=datalab-org \
            "https://github.com/datalab-org/datalab-app-plugin-template" \
            pydatalab/plugins/datalab-app-plugin-template


      - name: Set some environment variables for CI containers pre-build
        run: |
          echo "PYDATALAB_TESTING=true" >> .env
          echo "PYDATALAB_TESTING=true" >> pydatalab/.env

      - name: Build Docker images
        uses: docker/bake-action@v7
        with:
          files: docker-compose.yml
          load: true
          source: .
          targets: 'api'
          set: |
            api.cache-from=type=gha,scope=${{ github.ref_name }}-build-api
            api.cache-from=type=gha,scope=main-build-api
            api.cache-to=type=gha,scope=${{ github.ref_name }}-build-api,mode=max
            api.args.SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0+ci
            api.args.WEB_CONCURRENCY=1
            api.tags=datalab-api:latest

      - name: Start services
        run: |
          # Launch production container profiles and wait for them to come up
          if ! docker compose up api --force-recreate -d --wait; then
            echo "=== Docker compose startup failed ==="
            echo "=== Container logs: ==="
            docker compose logs database api app
            echo "=== Container status: ==="
            docker compose ps database api app
            exit 1
          fi

      - name: Test that plugin is loaded
        run: |
          # Test that plugin block is listed in info endpoint
          curl -s http://localhost:5000/info/blocks | jq '.data | any(.id == "example")'

  e2e:
    name: e2e tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 3
      matrix:
        browser-backend: ["electron", "chrome"] # , "firefox"]  # Temporarily disable Firefox until migration to Cypress v14 (https://github.com/datalab-org/datalab/issues/1289)

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Set up Node
        uses: actions/setup-node@v6
        env:
          FORCE_COLOR: 0
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "yarn"
          cache-dependency-path: ./webapp/yarn.lock

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Set some environment variables for CI containers pre-build
        run: |
          cp webapp/.env.test_e2e .env
          echo "PYDATALAB_TESTING=true" >> .env
          echo "PYDATALAB_TESTING=true" >> pydatalab/.env

      - name: Build Docker images
        uses: docker/bake-action@v7
        with:
          files: docker-compose.yml
          load: true
          source: .
          targets: 'app,api,database'
          set: |
            app.cache-from=type=gha,scope=${{ github.ref_name }}-build-app
            app.cache-from=type=gha,scope=main-build-app
            api.cache-from=type=gha,scope=${{ github.ref_name }}-build-api
            api.cache-from=type=gha,scope=main-build-api
            database.cache-from=type=gha,scope=${{ github.ref_name }}-build-database
            database.cache-from=type=gha,scope=main-build-database
            api.cache-to=type=gha,scope=${{ github.ref_name }}-build-api,mode=max
            app.cache-to=type=gha,scope=${{ github.ref_name }}-build-app,mode=max
            database.cache-to=type=gha,scope=${{ github.ref_name }}-build-database,mode=max
            api.args.SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0+ci
            api.args.WEB_CONCURRENCY=1
            app.args.VUE_APP_GIT_VERSION=0.0.0+ci
            app.tags=datalab-app:latest
            api.tags=datalab-api:latest
            database.tags=datalab-database:latest

      - name: Start services
        run: |
          # Launch production container profiles and wait for them to come up
          if ! docker compose up database api app --no-build --force-recreate -d --wait; then
            echo "=== Docker compose startup failed ==="
            echo "=== Container logs: ==="
            docker compose logs database api app
            echo "=== Container status: ==="
            docker compose ps database api app
            exit 1
          fi

      - name: Run end-to-end tests
        uses: cypress-io/github-action@v7
        with:
          config: baseUrl=http://localhost:8081
          working-directory: ./webapp
          record: ${{ github.repository == 'datalab-org/datalab' }}
          browser: ${{ matrix.browser-backend }}
          group: "End-to-end tests (${{ matrix.browser-backend }})"
        env:
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}

  component:
    # Cypress component tests don't need the datalab stack — they exercise Vue
    # components in isolation. Splitting them out means they run in parallel
    # with the e2e job instead of serially after it.
    name: Component tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        browser-backend: ["electron", "chrome"]

    steps:
      - uses: actions/checkout@v6

      - name: Set up Node
        uses: actions/setup-node@v6
        env:
          FORCE_COLOR: 0
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "yarn"
          cache-dependency-path: ./webapp/yarn.lock

      - name: Run component tests
        uses: cypress-io/github-action@v7
        with:
          working-directory: ./webapp
          record: ${{ github.repository == 'datalab-org/datalab' }}
          component: true
          publish-summary: false
          browser: ${{ matrix.browser-backend }}
          group: "Component tests (${{ matrix.browser-backend }})"
        env:
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}

  liccheck:
    name: Check OSS license compatibility for dependencies
    uses: ./.github/workflows/license-check.yml