Merge remote-tracking branch 'origin/main' into connorjward/fix-zenodo-ratelimit

Author: connorjward

.github/actionlint.yaml

@@ -0,0 +1,4 @@
+self-hosted-runner:
+  labels:
+    # Custom label for GPU-enabled self-hosted runners
+    - gpu

.github/workflows/core.yml

@@ -23,6 +23,10 @@ on:
         description: Whether to test using macOS
         type: boolean
         default: false
+      test_gpu:
+        description: Whether to test using CUDA-enabled PETSc
+        type: boolean
+        default: false
       deploy_website:
         description: Whether to deploy the website
         type: boolean
@@ -54,6 +58,10 @@ on:
         description: Whether to test using macOS
         type: boolean
         default: false
+      test_gpu:
+        description: Whether to test using CUDA-enabled PETSc
+        type: boolean
+        default: false
       deploy_website:
         description: Whether to deploy the website
         type: boolean
@@ -319,7 +327,12 @@ jobs:
           matrix.arch == 'default'
         run: |
           . venv/bin/activate
-          git clone --depth 1 https://github.com/firedrakeproject/gusto.git gusto-repo
+          if [ ${{ inputs.target_branch }} = 'release' ]; then
+            GUSTO_BRANCH='main'
+          else
+            GUSTO_BRANCH='future'
+          fi
+          git clone --depth 1 https://github.com/firedrakeproject/gusto.git gusto-repo --branch $GUSTO_BRANCH
           pip install --verbose ./gusto-repo
           python -m pytest -n 8 --verbose \
             gusto-repo/integration-tests/balance/test_saturated_balance.py \
@@ -460,6 +473,161 @@ jobs:
         run: |
           find . -delete
 
+  test_gpu:
+    name: Build and test Firedrake (Linux CUDA)
+    runs-on: [self-hosted, Linux, gpu]
+    container:
+      image: ubuntu:latest
+      options: --gpus all
+    if: inputs.test_gpu
+    env:
+      OMPI_ALLOW_RUN_AS_ROOT: 1
+      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+      OMP_NUM_THREADS: 1
+      OPENBLAS_NUM_THREADS: 1
+      FIREDRAKE_CI: 1
+      PYOP2_SPMD_STRICT: 1
+      # Disable fast math as it exposes compiler bugs
+      PYOP2_CFLAGS: -fno-fast-math
+      # NOTE: One should occasionally update test_durations.json by running
+      # 'make test_durations' inside a 'firedrake:latest' Docker image.
+      EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json --durations=50
+      PYTEST_MPI_MAX_NPROCS: 8
+      # Prevent PETSc from exiting with an error due to using non-GPU aware system MPI
+      PETSC_OPTIONS: -use_gpu_aware_mpi 0
+    steps:
+      - name: Confirm Nvidia GPUs are enabled
+        # The presence of the nvidia-smi command indicates that the Nvidia drivers have
+        # successfully been imported into the container, there is no point continuing
+        # if nvidia-smi is not present
+        run: nvidia-smi
+
+      - name: Fix HOME
+        # For unknown reasons GitHub actions overwrite HOME to /github/home
+        # which will break everything unless fixed
+        # (https://github.com/actions/runner/issues/863)
+        run: echo "HOME=/root" >> "$GITHUB_ENV"
+
+
+      # Git is needed for actions/checkout and Python for firedrake-configure
+      # curl needed for adding new deb repositories to ubuntu
+      - name: Install system dependencies (1)
+        run: |
+          apt-get update
+          apt-get -y install git python3 curl
+
+
+      - name: Pre-run cleanup
+        # Make sure the current directory is empty
+        run: find . -delete
+
+      - uses: actions/checkout@v5
+        with:
+          path: firedrake-repo
+          ref: ${{ inputs.source_ref }}
+
+      - name: Add Nvidia CUDA deb repositories
+        run: |
+          deburl=$( python3 ./firedrake-repo/scripts/firedrake-configure --show-extra-repo-pkg-url --gpu-arch cuda )
+          debfile=$( basename "${deburl}" )
+          curl -fsSLO "${deburl}"
+          dpkg -i "${debfile}"
+          apt-get update
+
+      - name: Install system dependencies (2)
+        run: |
+          apt-get -y install \
+            $(python3 ./firedrake-repo/scripts/firedrake-configure --arch default --gpu-arch cuda --show-system-packages)
+          apt-get -y install python3-venv
+          : # Dependencies needed to run the test suite
+          apt-get -y install fonts-dejavu graphviz graphviz-dev parallel poppler-utils
+
+      - name: Install PETSc
+        env:
+          EXTRA_OPTIONS: -use_gpu_aware_mpi 0
+        run: |
+          if [ ${{ inputs.target_branch }} = 'release' ]; then
+            git clone --depth 1 \
+              --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --gpu-arch cuda --show-petsc-version) \
+              https://gitlab.com/petsc/petsc.git
+          else
+            git clone --depth 1 https://gitlab.com/petsc/petsc.git
+          fi
+          cd petsc
+          python3 ../firedrake-repo/scripts/firedrake-configure \
+            --arch default --gpu-arch cuda --show-petsc-configure-options | \
+            xargs -L1 ./configure --with-make-np=4
+          make
+          make check
+          {
+            echo "PETSC_DIR=/__w/firedrake/firedrake/petsc"
+            echo "PETSC_ARCH=arch-firedrake-default-cuda"
+            echo "SLEPC_DIR=/__w/firedrake/firedrake/petsc/arch-firedrake-default-cuda"
+          } >> "$GITHUB_ENV"
+
+      - name: Install Firedrake
+        id: install
+        run: |
+          export $(python3 ./firedrake-repo/scripts/firedrake-configure --arch default --gpu-arch cuda --show-env)
+          python3 -m venv venv
+          . venv/bin/activate
+
+          : # Empty the pip cache to ensure that everything is compiled from scratch
+          pip cache purge
+
+          if [ ${{ inputs.target_branch }} = 'release' ]; then
+            EXTRA_PIP_FLAGS=''
+          else
+            : # Install build dependencies
+            pip install "$PETSC_DIR"/src/binding/petsc4py
+            pip install -r ./firedrake-repo/requirements-build.txt
+
+            : # We have to pass '--no-build-isolation' to use a custom petsc4py
+            EXTRA_PIP_FLAGS='--no-build-isolation'
+          fi
+
+          pip install --verbose $EXTRA_PIP_FLAGS \
+            --no-binary h5py \
+            --extra-index-url https://download.pytorch.org/whl/cu130 \
+            './firedrake-repo[ci]'
+
+          firedrake-clean
+          pip list
+
+      - name: Run smoke tests
+        run: |
+          . venv/bin/activate
+          firedrake-check
+        timeout-minutes: 10
+
+      - name: Confirm GPU usage in OffloadPC test
+        shell: bash
+        run: |
+          . venv/bin/activate
+          # We expect >80% of compute in SNESSolve to be performed on the GPU with GAMG pc and CG ksp. When
+          # -log_view -log_view_gpu_time is enabled, the final column of the PETSc profiling output is the
+          # percentage of FLOPs performed on the GPU. This test extracts the line beginning with SNESSolve (example below)
+          # into an array and tests if the final element of that array is above 80.
+          # -----
+          # SNESSolve 1 1.0 8.6306e-01 1.0 2.50e+05 1.0 0.0e+00 0.0e+00 0.0e+00 22 89 0 0 0 22 89 0 0 0 0 7 27 4.99e-02 21 2.63e-02 87
+          #
+          while read -r line; do
+            if [[ "${line}" =~ ^SNESSolve ]]; then
+              readarray -d' ' arr <<<"${line}"
+              echo "SNESSolve PETSc log_view line"
+              echo "============================="
+              echo "${line}"
+              [[ "${arr[-1]}" -gt 80 ]] && exit 0 || exit 1
+            fi
+          done < <( python firedrake-repo/tests/firedrake/offload/test_poisson_offloading_pc.py -log_view -log_view_gpu_time )
+          echo "SNESSolve not found in PETSc log_view output!"
+          exit 1
+
+      - name: Post-run cleanup
+        if: always()
+        run: |
+          find . -delete
+
   lint:
     name: Lint codebase
     runs-on: ubuntu-latest
@@ -505,15 +673,7 @@ jobs:
           firedrake-clean
           pip list
 
-      - name: Check bibtex
-        run: make -C firedrake-repo/docs validate-bibtex
-
-      - name: Check documentation links
-        if: success() || steps.install.conclusion == 'success'
-        run: make -C firedrake-repo/docs linkcheck
-
       - name: Build documentation
-        if: success() || steps.install.conclusion == 'success'
         id: build_docs
         working-directory: firedrake-repo/docs
         run: |
@@ -523,6 +683,14 @@ jobs:
           # : Copy manual to HTML tree
           cp build/latex/Firedrake.pdf build/html/_static/manual.pdf
 
+      - name: Check bibtex
+        if: success() || steps.install.conclusion == 'success'
+        run: make -C firedrake-repo/docs validate-bibtex
+
+      - name: Check documentation links
+        if: success() || steps.install.conclusion == 'success'
+        run: make -C firedrake-repo/docs linkcheck
+
       - name: Upload documentation
         uses: actions/upload-pages-artifact@v3
         id: upload_docs

.github/workflows/pr.yml

@@ -12,4 +12,6 @@ jobs:
       target_branch: ${{ github.base_ref }}
       # Only run macOS tests if the PR is labelled 'macOS'
       test_macos: ${{ contains(github.event.pull_request.labels.*.name, 'macOS') }}
+      # Only run GPU tests if the PR is labelled 'gpu'
+      test_gpu: ${{ contains(github.event.pull_request.labels.*.name, 'gpu') }}
     secrets: inherit

.github/workflows/push.yml

@@ -13,6 +13,7 @@ jobs:
       source_ref: ${{ github.ref_name }}
       target_branch: ${{ github.ref_name }}
       test_macos: true
+      test_gpu: true
       deploy_website: true
     secrets: inherit
 

.github/workflows/release.yml

@@ -11,6 +11,10 @@ on:
         description: The version number to release
         type: string
         required: true
+      prev_version:
+        description: The previously most up-to-date version number
+        type: string
+        required: true
 
 jobs:
   check:
@@ -62,7 +66,7 @@ jobs:
       - name: Create release
         env:
           GH_TOKEN: ${{ github.token }}
-        run: gh release create --title ${{ inputs.version }} --target ${{ inputs.branch }} --generate-notes ${{ inputs.version }}
+        run: gh release create --title ${{ inputs.version }} --target ${{ inputs.branch }} --generate-notes ${{ inputs.version }} --notes-start-tag ${{ inputs.prev_version }}
 
   docker:
     name: Build Docker containers

MANIFEST.in

@@ -1,10 +1,12 @@
-.PHONY: all
-all: modules
-
-.PHONY: modules
-modules:
+.PHONY: ext
+ext:
 	@echo "    Building extension modules"
-	@python setup.py build_ext --inplace > build.log 2>&1 || cat build.log
+	@python setup.py build_ext --inplace 2>&1 | tee cat build.log
+
+.PHONY: extforce
+extforce:
+	@echo "    Force building all extension modules"
+	@python setup.py build_ext --inplace --force 2>&1 | tee cat build.log
 
 .PHONY: lint
 lint: srclint actionlint dockerlint
@@ -55,39 +57,6 @@ dockerlint:
 			|| exit 1; \
 	done
 
-.PHONY: clean
-clean:
-	@echo "    Cleaning extension modules"
-	@python setup.py clean > /dev/null 2>&1
-	@echo "    RM firedrake/cython/dmplex.*.so"
-	-@rm -f firedrake/cython/dmplex.so > /dev/null 2>&1
-	@echo "    RM firedrake/cython/dmplex.c"
-	-@rm -f firedrake/cython/dmplex.c > /dev/null 2>&1
-	@echo "    RM firedrake/cython/extrusion_numbering.*.so"
-	-@rm -f firedrake/cython/extrusion_numbering.so > /dev/null 2>&1
-	@echo "    RM firedrake/cython/extrusion_numbering.c"
-	-@rm -f firedrake/cython/extrusion_numbering.c > /dev/null 2>&1
-	@echo "    RM firedrake/cython/hdf5interface.*.so"
-	-@rm -f firedrake/cython/hdf5interface.so > /dev/null 2>&1
-	@echo "    RM firedrake/cython/hdf5interface.c"
-	-@rm -f firedrake/cython/hdf5interface.c > /dev/null 2>&1
-	@echo "    RM firedrake/cython/spatialindex.*.so"
-	-@rm -f firedrake/cython/spatialindex.so > /dev/null 2>&1
-	@echo "    RM firedrake/cython/spatialindex.c"
-	-@rm -f firedrake/cython/spatialindex.c > /dev/null 2>&1
-	@echo "    RM firedrake/cython/supermeshimpl.*.so"
-	-@rm -f firedrake/cython/supermeshimpl.so > /dev/null 2>&1
-	@echo "    RM firedrake/cython/supermeshimpl.c"
-	-@rm -f firedrake/cython/supermeshimpl.c > /dev/null 2>&1
-	@echo "    RM firedrake/cython/mg/impl.*.so"
-	-@rm -f firedrake/cython/mg/impl.so > /dev/null 2>&1
-	@echo "    RM firedrake/cython/mg/impl.c"
-	-@rm -f firedrake/cython/mg/impl.c > /dev/null 2>&1
-	@echo "    RM pyop2/*.so"
-	-@rm -f pyop2/*.so > /dev/null 2>&1
-	@echo "    RM tinyasm/*.so"
-	-@rm -f tinyasm/*.so > /dev/null 2>&1
-
 # NOTE: It is recommended to run this command from inside the 'firedrake'
 # Docker image to reduce the likelihood of test failures.
 .PHONY: test_durations