core.yml

# Main Firedrake CI workflow run for pull requests or pushes to main or release

on:
  workflow_call:
    inputs:
      trigger:
        description: Whether the workflow was triggered by a PR or push ('pr' or 'push')
        type: string
        required: true
      source_ref:
        description: The input Git ref to checkout
        type: string
        required: true
      base_ref:
        description: The base branch ('main' or 'release')
        type: string
        required: true
      deploy_website:
        description: Whether to deploy the website
        type: boolean
        default: false
      test_complex:
        description: Whether to test a complex build
        type: boolean
        required: true
      test_cuda:
        description: Whether to test using CUDA-enabled PETSc
        type: boolean
        required: true
      test_macos:
        description: Whether to test using macOS
        type: boolean
        required: true

  workflow_dispatch:
    inputs:
      trigger:
        description: Whether the workflow was triggered by a PR or push ('pr' or 'push')
        type: string
        required: true
      source_ref:
        description: The input Git ref to checkout
        type: string
        required: true
      base_ref:
        description: The base branch ('main' or 'release')
        type: string
        required: true
      deploy_website:
        description: Whether to deploy the website
        type: boolean
        default: false
      test_complex:
        description: Whether to test a complex build
        type: boolean
        required: true
      test_cuda:
        description: Whether to test using CUDA-enabled PETSc
        type: boolean
        required: true
      test_macos:
        description: Whether to test using macOS
        type: boolean
        required: true

concurrency:
  # Cancel running jobs if new commits are pushed
  group: >
    ${{ github.workflow }}-
    ${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  OMPI_ALLOW_RUN_AS_ROOT: 1
  OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
  OMP_NUM_THREADS: 1
  OPENBLAS_NUM_THREADS: 1
  PYTEST_MPI_MAX_NPROCS: 8
  PYOP2_SPMD_STRICT: 1
  FIREDRAKE_CI: 1
  # NOTE: One should occasionally update test_durations.json by running
  # 'make test_durations' inside a 'firedrake:latest' Docker image.
  EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json --durations=50

jobs:

  # Test suite

  test_default:
    name: Test default
    runs-on: [self-hosted, Linux]
    container:
      image: ubuntu:latest
    env:
      # Disable some compiler optimisations as they cause runner-specific bugs
      PYOP2_CFLAGS: -O1 -fno-fast-math
    steps:
      - name: Pre-run cleanup
        # Make sure the current directory is empty
        run: find . -delete

      - uses: actions/checkout@v5
        with:
          path: firedrake-repo
          ref: ${{ inputs.source_ref }}

      - name: Install Firedrake
        id: install
        uses: ./firedrake-repo/.github/actions/install
        with:
          os: linux
          source_ref: ${{ inputs.source_ref }}
          base_ref: ${{ inputs.base_ref }}
          deps: ci

      - name: Run TSFC tests
        # Run even if earlier tests failed
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          : # Use pytest-xdist here so we can have a single collated output (not possible
          : # for parallel tests)
          firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/tsfc
        timeout-minutes: 10

      - name: Run PyOP2 tests
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          : # Use pytest-xdist here so we can have a single collated output (not possible
          : # for parallel tests)
          firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
          firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
          firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
          firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
        timeout-minutes: 10


      - name: Run Firedrake tests (nprocs = 1)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          : # Use pytest-xdist here so we can have a single collated output (not possible
          : # for parallel tests)
          firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 90

      - name: Run tests (nprocs = 2)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 60

      - name: Run tests (nprocs = 3)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 60

      - name: Run tests (nprocs = 4)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 15

      - name: Run tests (nprocs = 5)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 5 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 15

      - name: Run tests (nprocs = 6)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 6 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 15

      - name: Run tests (nprocs = 7)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 7 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 15

      - name: Run tests (nprocs = 8)
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 8 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 15

      - name: Run Gusto smoke tests
        # Only test Gusto in real mode
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          if [ ${{ inputs.base_ref }} = 'release' ]; then
            GUSTO_BRANCH='main'
          else
            GUSTO_BRANCH='future'
          fi
          git clone --depth 1 https://github.com/firedrakeproject/gusto.git gusto-repo --branch $GUSTO_BRANCH
          pip install --verbose ./gusto-repo
          python -m pytest -n 8 --verbose \
            gusto-repo/integration-tests/balance/test_saturated_balance.py \
            gusto-repo/integration-tests/equations/test_thermal_sw.py \
            gusto-repo/integration-tests/transport/test_embedded_dg_advection.py
        timeout-minutes: 10

      - name: Run Thetis smoke tests
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          git clone --depth 1 https://github.com/thetisproject/thetis.git thetis-repo --branch ${{ inputs.base_ref }}
          pip install --verbose ./thetis-repo
          python -m pytest -n 8 --verbose thetis-repo/test_adjoint/test_swe_adjoint.py
        timeout-minutes: 10

      - name: Run spyro smoke tests
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          git clone --depth 1 https://github.com/NDF-Poli-USP/spyro.git spyro-repo
          pip install --verbose ./spyro-repo
          mpiexec -n 6 python -m pytest spyro-repo/test_integration/ -m parallel[6]
        timeout-minutes: 5

      - name: Run G-ADOPT smoke tests
        if: success() || steps.install.conclusion == 'success'
        run: |
          . venv/bin/activate
          git clone --depth 1 https://github.com/g-adopt/g-adopt.git g-adopt-repo --branch ${{ inputs.base_ref }}
          pip install --verbose ./g-adopt-repo
          make -C g-adopt-repo/demos/mantle_convection/base_case check
        timeout-minutes: 5

      - name: Upload log files
        uses: actions/upload-artifact@v4
        if: success() || steps.install.conclusion == 'success'
        with:
          name: firedrake-logs-default
          path: pytest_*.log

      - name: Post-run cleanup
        if: always()
        run: find . -delete

  # Since we can't skip matrix entries based on 'inputs.test_complex' etc
  # we have to build the matrix as a separate step
  gen_linux_extra_matrix:
    name: Prepare matrix for Linux extra tests
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.generate.outputs.matrix }}
    steps:
      - name: Generate matrix
        id: generate
        run: |
          if [ ${{ inputs.test_complex }} == 'true' ]; then
            matrix='{"scalar_type": "complex", "gpu": "none"}'
          fi
          if [ ${{ inputs.test_cuda }} == 'true' ]; then
            arch='{"scalar_type": "default", "gpu": "cuda"}'
            if [ "$matrix" ]; then
              matrix="$matrix, $arch"
            else
              matrix="$arch"
            fi
          fi

          echo "Matrix is:"
          echo "$matrix"
          echo "matrix=[$matrix]" >> "$GITHUB_OUTPUT"

  test_linux_extra:
    name: Test Linux extra
    needs: gen_linux_extra_matrix
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.gen_linux_extra_matrix.outputs.matrix) }}
    runs-on: [self-hosted, Linux, gpu]
    container:
      # TODO: set to 'ubuntu:latest' for GPU case
      image: ${{ case(matrix.gpu == 'cuda', 'ubuntu:noble', 'ubuntu:latest') }}
      options: --gpus all
    env:
      # Disable some compiler optimisations as they cause runner-specific bugs
      PYOP2_CFLAGS: -O1 -fno-fast-math
      # Prevent PETSc from exiting with an error due to using non-GPU aware system MPI
      PETSC_OPTIONS: -use_gpu_aware_mpi 0
    steps:
      - name: Pre-run cleanup
        # Make sure the current directory is empty
        run: find . -delete

      - name: Confirm NVIDIA GPUs are enabled
        if: matrix.gpu == 'cuda'
        # The presence of the nvidia-smi command indicates that the Nvidia drivers have
        # successfully been imported into the container, there is no point continuing
        # if nvidia-smi is not present
        run: nvidia-smi

      - uses: actions/checkout@v5
        with:
          path: firedrake-repo
          ref: ${{ inputs.source_ref }}

      - name: Install Firedrake
        id: install
        uses: ./firedrake-repo/.github/actions/install
        with:
          os: linux
          source_ref: ${{ inputs.source_ref }}
          base_ref: ${{ inputs.base_ref }}
          scalar_type: ${{ matrix.scalar_type }}
          gpu: ${{ matrix.gpu }}
          deps: ci

      - name: Confirm GPU usage in OffloadPC test
        shell: bash
        if: matrix.gpu == 'cuda'
        run: |
          . venv/bin/activate
          : # We expect >80% of compute in SNESSolve to be performed on the GPU with GAMG pc and CG ksp. When
          : # -log_view -log_view_gpu_time is enabled, the final column of the PETSc profiling output is the
          : # percentage of FLOPs performed on the GPU. This test extracts the line beginning with SNESSolve (example below)
          : # into an array and tests if the final element of that array is above 80.
          : # -----
          : # SNESSolve 1 1.0 8.6306e-01 1.0 2.50e+05 1.0 0.0e+00 0.0e+00 0.0e+00 22 89 0 0 0 22 89 0 0 0 0 7 27 4.99e-02 21 2.63e-02 87
          while read -r line; do
            if [[ "${line}" =~ ^SNESSolve ]]; then
              readarray -d' ' arr <<<"${line}"
              echo "SNESSolve PETSc log_view line"
              echo "============================="
              echo "${line}"
              [[ "${arr[-1]}" -gt 80 ]] && exit 0 || exit 1
            fi
          done < <( python firedrake-repo/tests/firedrake/offload/test_poisson_offloading_pc.py -log_view -log_view_gpu_time )
          echo "SNESSolve not found in PETSc log_view output!"
          exit 1

      # TODO: We should run all tests that are specific to a particular configuration
      # (e.g. run all tests marked 'skipnogpu')
      - name: Run tests subset
        if: inputs.trigger == 'push'
        run: |
          . venv/bin/activate

          : # Run 10% of the whole test suite (nprocs = {1, 2, 3})
          PYTEST_ARGS="--splits 10 --group 1 -m parallel[match] $EXTRA_PYTEST_ARGS firedrake-repo/tests/firedrake"

          python3 -m pytest -n 8 $PYTEST_ARGS
          mpiexec -n 2 python3 -m pytest $PYTEST_ARGS
          mpiexec -n 3 python3 -m pytest $PYTEST_ARGS
        timeout-minutes: 30

      - name: Run TSFC tests
        if: inputs.trigger == 'pr'
        run: |
          . venv/bin/activate
          : # Use pytest-xdist here so we can have a single collated output (not possible
          : # for parallel tests)
          firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/tsfc
        timeout-minutes: 10

      - name: Run PyOP2 tests
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
          firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
          firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
          firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2
        timeout-minutes: 10

      - name: Run Firedrake tests (nprocs = 1)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 90

      - name: Run Firedrake tests (nprocs = 2)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 60

      - name: Run Firedrake tests (nprocs = 3)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 60

      - name: Run Firedrake tests (nprocs = 4)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 20

      - name: Run Firedrake tests (nprocs = 5)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 5 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 20

      - name: Run Firedrake tests (nprocs = 6)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 6 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 20

      - name: Run Firedrake tests (nprocs = 7)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 7 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 20

      - name: Run Firedrake tests (nprocs = 8)
        if: inputs.trigger == 'pr' && (success() || steps.install.conclusion == 'success')
        run: |
          . venv/bin/activate
          firedrake-run-split-tests 8 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
        timeout-minutes: 20

      - name: Post-run cleanup
        if: always()
        run: find . -delete

  # TODO: run full test suite on PR and subset on push like above
  test_macos:
    name: Test macOS
    runs-on: [self-hosted, macOS]
    if: inputs.test_macos
    steps:
      - name: Pre-run cleanup
        # Make sure the current directory is empty
        run: find . -delete

      - name: Add homebrew to PATH
        # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#adding-a-system-path
        run: echo "/opt/homebrew/bin" >> "$GITHUB_PATH"

      - uses: actions/checkout@v5
        with:
          path: firedrake-repo
          ref: ${{ inputs.source_ref }}

      - name: Install Firedrake
        id: install
        uses: ./firedrake-repo/.github/actions/install
        with:
          os: macos
          source_ref: ${{ inputs.source_ref }}
          base_ref: ${{ inputs.base_ref }}
          deps: ci

      - name: Post-run cleanup
        if: always()
        run: find . -delete

  # Quick checks

  check:
    name: Basic repository checks
    runs-on: ubuntu-latest
    container:
      image: ubuntu:latest
    steps:
      - name: Install system dependencies
        run: |
          apt-get update
          apt-get -y install python3

      - uses: actions/checkout@v5
        with:
          path: firedrake-repo
          ref: ${{ inputs.source_ref }}

      - name: Validate single source of truth
        run: ./firedrake-repo/scripts/check-config

      # Check that the Dockerfile is using the latest Ubuntu version.
      # The version is hardcoded into the Dockerfile so that the OS
      # for each release is fixed.
      - name: Check Dockerfile Ubuntu version
        run: |
          latest_version=$(grep "VERSION_ID=" /etc/os-release | cut -d '"' -f 2)
          docker_version=$(grep FROM firedrake-repo/docker/Dockerfile.vanilla | cut -d ':' -f 2)
          echo "Latest version: $latest_version"
          echo "Docker version: $docker_version"
          if [[ "$docker_version" != "$latest_version" ]]; then
            echo "Ubuntu version ${docker_version} in Dockerfile is out of date with latest version ${latest_version}"
            exit 1
          fi

  lint:
    name: Lint codebase
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v5
        with:
          ref: ${{ inputs.source_ref }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.13'
      - name: Setup flake8 annotations
        uses: rbialon/flake8-annotations@v1
      - name: Install linting packages
        run: pip install flake8 pylint
      - name: Lint codebase
        run: make lint GITHUB_ACTIONS_FORMATTING=1

  # Docs

  build_docs:
    name: Build documentation
    runs-on: [self-hosted, Linux]
    container:
      image: firedrakeproject/firedrake-vanilla-default:dev-${{ inputs.base_ref }}
    outputs:
      conclusion: ${{ steps.report_docs.outputs.conclusion }}
    steps:
      - name: Install system dependencies
        run: |
          apt-get update
          apt-get -y install inkscape texlive-full

      - uses: actions/checkout@v5
        with:
          path: firedrake-repo
          ref: ${{ inputs.source_ref }}

      - name: Install Firedrake
        id: install
        run: |
          pip install --verbose -r ./firedrake-repo/requirements-build.txt
          CC=mpicc CXX=mpicxx \
            pip install --verbose --no-build-isolation './firedrake-repo[docs]'
          firedrake-clean
          pip list

      - name: Build documentation
        id: build_docs
        working-directory: firedrake-repo/docs
        run: |
          make SPHINXOPTS="-t ${{ inputs.base_ref }}" html
          make latex
          make latexpdf
          # : Copy manual to HTML tree
          cp build/latex/Firedrake.pdf build/html/_static/manual.pdf

      - name: Check bibtex
        if: success() || steps.install.conclusion == 'success'
        run: make -C firedrake-repo/docs validate-bibtex

      - name: Check documentation links
        if: success() || steps.install.conclusion == 'success'
        run: make -C firedrake-repo/docs linkcheck

      - name: Upload documentation
        uses: actions/upload-pages-artifact@v3
        id: upload_docs
        if: success() || steps.build_docs.conclusion == 'success'
        with:
          name: github-pages
          path: firedrake-repo/docs/build/html
          retention-days: 1

      # Some files are automatically generated when we install or build the
      # docs (e.g. AUTHORS.rst). These files are committed to the repository
      # so uncommitted changes at this point reflect changes in the automatically
      # generated files that ought to be committed.
      - name: Check no untracked files
        if: always()
        working-directory: firedrake-repo
        run: |
          if [ -z "$( git status --porcelain )" ]; then
            exit 0
          else
            echo Error: there are untracked files in the repository
            echo The output from 'git status' is:
            git status
            echo The output from 'git diff' is:
            git diff
            exit 1
          fi

      - name: Report docs build status
        id: report_docs
        if: success() || steps.upload_docs.conclusion == 'success'
        run: echo "conclusion=success" >> "$GITHUB_OUTPUT"

  deploy_website:
    name: Deploy GitHub pages (push to main only)
    needs: build_docs
    # Only deploy the website for main here, the release branch is managed
    # by https://github.com/firedrakeproject/firedrakeproject.github.io
    if: |
      always() &&
      inputs.deploy_website &&
      inputs.base_ref == 'main' &&
      needs.build_docs.outputs.conclusion == 'success'
    permissions:
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: http://firedrakeproject.github.io/firedrake
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to GitHub Pages
        uses: actions/deploy-pages@v4