Merge tag 'v3.14.1-pdaf-upstream' into tsmp-pdaf-patched

Author: jjokella

.github/workflows/linux.yml

@@ -1,10 +1,12 @@
-name: ParFlow CI Test
+name: Linux CI Tests
 
 on: [push, pull_request]
 
 jobs:
   style-check:
-     runs-on: ubuntu-22.04
+     # Runs a check on C and Python coding style.  Code
+     # should not be merged unless the style check passes.
+     runs-on: ubuntu-24.04
      name: Check Style
      steps:
        - name: Checkout commit
@@ -23,7 +25,7 @@ jobs:
            cache-parflow-style-hit: ${{steps.cache-parflow-style-dependencies.outputs.cache-hit}}
          with:
            path: "~/depend"
-           key: cache-check-style-${{ secrets.CACHE_VERSION }}
+           key: cache-check-style-${{ hashfiles('**/.github/workflows/linux.yml') }}
 
        - name: Directory Setup
          run: |
@@ -54,13 +56,13 @@ jobs:
            CACHE_HIT: ${{steps.cache-parflow-style-dependencies.outputs.cache-hit}}
          run: |
             if [[ "$CACHE_HIT" != 'true' ]]; then
-              echo "cache-check-style-${{ secrets.CACHE_VERSION }}" > $HOME/depend/cache-key
+              echo "cache-check-style-${{ hashfiles('**/.github/workflows/linux.yml') }}" > $HOME/depend/cache-key
               curl -L -o uncrustify-0.79.0.tar.gz https://github.com/uncrustify/uncrustify/archive/uncrustify-0.79.0.tar.gz
               tar -xf uncrustify-0.79.0.tar.gz
               cd uncrustify-uncrustify-0.79.0
               mkdir build
               cd build
-              cmake -DCMAKE_INSTALL_PREFIX=/usr/local ..
+              cmake -DCMAKE_INSTALL_PREFIX=${PARFLOW_DEP_DIR} ..
               make
               sudo make -j 2 install
               cd ../..
@@ -75,7 +77,7 @@ jobs:
                echo "Installing"
                mkdir -p $HOME/depend/{include,lib}
                cd $HOME/depend
-               echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}" > $HOME/depend/cache-key
+               echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ hashfiles('**/.github/workflows/linux.yml') }}" > $HOME/depend/cache-key
                wget -nv --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-x86_64.tar.gz
                tar -xf cmake-3.25.1-linux-x86_64.tar.gz
                $HOME/depend/cmake-3.25.1-linux-x86_64/bin/cmake --version
@@ -97,6 +99,17 @@ jobs:
     strategy:
       # fail-fast: true
       matrix:
+        # Basic philosophy is to check builds with different compilers, backends
+        # and OS.
+        # 1) Should retain checks for GCC and LLVM
+        # 2) Should at minimum check OMP, CUDA, and Kokkos backends
+        # 3) Most checks an be run on the same version of Ubuntu but at least one older
+        #    version should be retained to verify we still run on older systems.
+        # 4) Should check Python build, currently we retain non-Python build but
+        #    the non-Python build may be deprecated
+        #
+        # Other builds maybe included to test other configuration options as deemed
+        # useful.
         config:
         - {
             name: "Ubuntu 24.04 GCC",
@@ -107,7 +120,8 @@ jobs:
             backend: "none",
             amps_layer: mpi1,
             netcdf: "true",
-            memory_manager: "none"
+            memory_manager: "none",
+            pdi: "false"
           }
         - {
             name: "Ubuntu 24.04 LLVM",
@@ -118,7 +132,8 @@ jobs:
             backend: "none",
             amps_layer: mpi1,
             netcdf: "true",
-            memory_manager: "none"
+            memory_manager: "none",
+            pdi: "false"
           }
         - {
             name: "Ubuntu 22.04 GCC Python",
@@ -129,7 +144,8 @@ jobs:
             backend: "none",
             amps_layer: mpi1,
             netcdf: "true",
-            memory_manager: "none"
+            memory_manager: "none",
+            pdi: "false"
           }
         - {
             name: "Ubuntu 24.04 GCC Python",
@@ -140,29 +156,32 @@ jobs:
             backend: "none",
             amps_layer: mpi1,
             netcdf: "false",
-            memory_manager: "none"
+            memory_manager: "none",
+            pdi: "false"
           }
         - {
-            name: "Ubuntu 22.04 GCC OMP",
-            os: ubuntu-22.04,
-            cudaos: 'ubuntu2204',
+            name: "Ubuntu 24.04 GCC OMP",
+            os: ubuntu-24.04,
+            cudaos: 'ubuntu2404',
             cc: "gcc", cxx: "g++", fc: "gfortran",
             python: "false",
             backend: "omp",
             amps_layer: mpi1,
             netcdf: "false",
-            memory_manager: "none"
+            memory_manager: "none",
+            pdi: "false"
           }
         - {
             name: "Ubuntu 24.04 LLVM OASIS3-MCT Build",
             os: ubuntu-24.04,
-            cudaos: 'ubuntu2204',
+            cudaos: 'ubuntu2404',
             cc: "clang-18", cxx: "clang++-18", fc: "gfortran",
             python: "false",
             backend: "omp",
             amps_layer: oas3,
             netcdf: "true",
-            memory_manager: "none"
+            memory_manager: "none",
+            pdi: "false"
           }
 #        - {
 #            name: "Ubuntu 24.04 GCC CUDA/RMM Build",
@@ -181,10 +200,11 @@ jobs:
             cc: "gcc", cxx: "g++", fc: "gfortran",
             python: "false",
             backend: "cuda",
-            cudaos: 'ubuntu2204',
+            cudaos: 'ubuntu2404',
             amps_layer: mpi1,
             netcdf: "false",
-            memory_manager: "umpire"
+            memory_manager: "umpire",
+            pdi: "false"
           }
 #        - {
 #            name: "Ubuntu 24.04 GCC Kokkos/RMM Build",
@@ -203,10 +223,23 @@ jobs:
             cc: "gcc", cxx: "g++", fc: "gfortran",
             python: "false",
             backend: "kokkos",
-            cudaos: 'ubuntu2204',
+            cudaos: 'ubuntu2404',
             amps_layer: mpi1,
             netcdf: "false",
-            memory_manager: "umpire"
+            memory_manager: "umpire",
+            pdi: "false"
+          }
+        - {
+            name: "Ubuntu 24.04 GCC PDI Build",
+            os: ubuntu-24.04,
+            cudaos: 'ubuntu2404',
+            cc: "gcc", cxx: "g++", fc: "gfortran",
+            python: "false",
+            backend: "none",
+            amps_layer: mpi1,
+            netcdf: "false",
+            memory_manager: "none",
+            pdi: "true"
           }
 
     steps:
@@ -241,10 +274,6 @@ jobs:
         sudo apt-get -qq update
         sudo apt -qq install python3 python3-venv
 
-    # Cache dependencies so don't have to rebuild on each test.
-    # Can flush caches by resetting the CACHE_VERSION secret on GitHub
-    # settings for the project (using date-timestamp for secret).
-
     - name: Cache dependencies
       uses: actions/cache@v4
       id: cache-parflow-dependencies
@@ -253,7 +282,7 @@ jobs:
         cache-parflow-hit: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
       with:
         path: "~/depend"
-        key: cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ matrix.config.memory_manager }}-${{ secrets.CACHE_VERSION }}
+        key: cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ matrix.config.memory_manager }}-${{ matrix.config.pdi }}-${{ hashfiles('**/.github/workflows/linux.yml') }}
 
     - name: Directory Setup
       run: |
@@ -298,34 +327,20 @@ jobs:
        if [[ "$CACHE_HIT" != 'true' ]]; then
           echo "Installing CMake v3.26.5"
           cd $HOME/depend
-          echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}" > $HOME/depend/cache-key
+          echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ hashfiles('**/.github/workflows/linux.yml') }}" > $HOME/depend/cache-key
           wget -nv --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.26.5/cmake-3.26.5-linux-x86_64.tar.gz
           tar -xf cmake-3.26.5-linux-x86_64.tar.gz
           $HOME/depend/cmake-3.26.5-linux-x86_64/bin/cmake --version
         fi
         echo "$HOME/depend/cmake-3.26.5-linux-x86_64/bin" >> $GITHUB_PATH
 
-    - name: NVidia Driver Install
+    - name: NVidia compiler and CUDA toolkit Install
       if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
-      run: |
-        sudo add-apt-repository -y ppa:graphics-drivers
-        sudo apt-get update -qq
-        sudo apt-get install -y nvidia-driver-535
-
-    - name: CUDA Install
-      if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
-      run: |
-        export CUDA_VERSION_MAJOR_MINOR=12.4.0
-        export CUDA_SHORT=12.4
-        export CUDA_OS=${{ matrix.config.cudaos }}
-        ./bin/install-cuda.sh
-        export CUDA_HOME=/usr/local/cuda-${CUDA_SHORT}
-        echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
-        export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
-        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-        export LIBRARY_PATH=${CUDA_HOME}/lib64/stubs
-        echo "LIBRARY_PATH=${LIBRARY_PATH}" >> $GITHUB_ENV
-        echo "${CUDA_HOME}/bin" >> $GITHUB_PATH
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: nvidia-cuda-toolkit nvidia-cuda-dev
+        version: 1.0
+        execute_install_scripts: true
 
     - name: UCX
       if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
@@ -364,13 +379,14 @@ jobs:
       if: (matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos') && matrix.config.memory_manager == 'umpire'
       run: |
         if [[ "$CACHE_HIT" != 'true' ]]; then
-          git clone --recursive https://github.com/LLNL/Umpire.git
+          git clone -b v2025.03.1 --recursive https://github.com/LLNL/Umpire.git
           cd Umpire
-          mkdir build
-          cd build
-          cmake ../ -DCMAKE_INSTALL_PREFIX=$PARFLOW_DEP_DIR/Umpire -DENABLE_CUDA=On
-          make
-          make install
+          cmake -S . -B build                                             \
+                     -DCMAKE_CXX_COMPILER=mpicxx                        \
+                     -DCMAKE_INSTALL_PREFIX="${PARFLOW_DEP_DIR}/Umpire" \
+                     -DENABLE_CUDA="ON"
+          cmake --build build --parallel 4
+          cmake --install build
         fi
         echo "UMPIRE_FLAGS=-Dumpire_ROOT=$PARFLOW_DEP_DIR/Umpire" >> $GITHUB_ENV
 
@@ -469,15 +485,54 @@ jobs:
         CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
       run: |
         if [[ "$CACHE_HIT" != 'true' ]]; then
-          URL=https://github.com/hypre-space/hypre/archive/v2.26.0.tar.gz
-          mkdir hypre
-          cd hypre
+          # Download
+          wget https://github.com/hypre-space/hypre/archive/v2.33.0.tar.gz
+          tar xf v2.33.0.tar.gz && cd hypre-2.33.0
+
+          # Set Hypre accelerator backend
+          if [[ "${{ matrix.config.backend }}" == "cuda" ]]; then
+            HAS_CUDA="ON"
+            BACKEND="CUDA"
+          else
+            HAS_CUDA="OFF"
+            BACKEND=""
+          fi
+
+          # Install Hypre. See https://hypre.readthedocs.io/en/latest/ch-misc.html#build-system-options
+          cmake -S src -B build                                \
+                -DCMAKE_C_COMPILER=mpicc                     \
+                -DCMAKE_CXX_COMPILER=mpicxx                  \
+                -DCMAKE_INSTALL_PREFIX="${PARFLOW_DEP_DIR}"  \
+                -DHYPRE_ENABLE_MPI="ON"                      \
+                -DHYPRE_ENABLE_CUDA="${HAS_CUDA}"
+          cmake --build build --parallel 4
+          cmake --install build
+          echo "" && echo "Successfully built Hypre ${BACKEND}"
+        fi
+
+    - name: PDI Install
+      env:
+        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
+      if: matrix.config.pdi == 'true'
+      run: |
+        if [[ "$CACHE_HIT" != 'true' ]]; then
+          PDI_VERSION="1.8.1"
+          URL="https://github.com/pdidev/pdi/archive/refs/tags/1.8.1.tar.gz"
+          export curr_pdi=$(pwd)
+          mkdir -p pdi
+          cd pdi
           curl -L $URL | tar --strip-components=1 -xz
-          cd src
-          ./configure --prefix=$PARFLOW_DEP_DIR
+          # Configure and install PDI
+          mkdir build install
+          cd build
+          cmake .. -DCMAKE_INSTALL_PREFIX=${PARFLOW_DEP_DIR}
           make -j 2 install
         fi
 
+        export LD_LIBRARY_PATH=${PARFLOW_DEP_DIR}/pdi/install/lib:${LD_LIBRARY_PATH}
+        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+        echo "PDI_FLAGS=-DPDI_ROOT=${PARFLOW_DEP_DIR}/pdi/install" >> $GITHUB_ENV
+
     - name: ParFlow CMake Configure
       run: |
         cat $HOME/depend/cache-key
@@ -496,21 +551,23 @@ jobs:
            -DPARFLOW_AMPS_LAYER=${{ matrix.config.amps_layer }}         \
            -DMPIEXEC_POSTFLAGS='--oversubscribe'                        \
            -DPARFLOW_ACCELERATOR_BACKEND=${{ matrix.config.backend }}   \
+           -DCMAKE_CUDA_ARCHITECTURES="60;70;75;80"                     \
            -DPARFLOW_AMPS_SEQUENTIAL_IO=true                            \
            -DPARFLOW_HAVE_CLM=${HAVE_CLM}                               \
            -DHYPRE_ROOT=$PARFLOW_DEP_DIR                                \
            -DOAS3_ROOT=$PARFLOW_DEP_DIR                                 \
            -DSILO_ROOT=$PARFLOW_DEP_DIR                                 \
            -DPARFLOW_ENABLE_PYTHON=${{ matrix.config.python }}          \
            -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR                          \
-           $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS $UMPIRE_FLAGS
+           $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS $UMPIRE_FLAGS $PDI_FLAGS
 
     - name: ParFlow CMake Build
       run: (cd build; make -j 2 install)
 
     # Can't test with GPU since have no GPU hardware on testing nodes, GPU test is build only
+    # PDI testing requires a configuration file for specification tree, so for now, it is only being built and not tested.
     - name: ParFlow CTest
-      if: matrix.config.backend != 'cuda' && matrix.config.backend != 'kokkos'
+      if: matrix.config.backend != 'cuda' && matrix.config.backend != 'kokkos' && matrix.config.pdi != 'true'
       run: |
         export OMPI_MCA_rmaps_base_oversubscribe=1
         export OMP_NUM_THREADS=1

.github/workflows/macos.yml

@@ -1,4 +1,4 @@
-name: ParFlow MacOS CI Test
+name: MacOS CI Tests
 
 #
 # Notes
@@ -45,10 +45,6 @@ jobs:
       run: |
         brew link tcl-tk@8
 
-# Cache dependencies so don't have to rebuild on each test.
-# Can flush caches by resetting the CACHE_VERSION secret on GitHub
-# settings for the project (using date-timestamp for secret).
-
     - name: Cache dependencies
       uses: actions/cache@v4
       id: cache-parflow-dependencies
@@ -57,7 +53,7 @@ jobs:
         cache-parflow-hit: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
       with:
         path: "~/depend"
-        key: cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}
+        key: cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ hashfiles('**/.github/workflows/macos.yml') }}
 
     - name: Directory Setup
       run: |
@@ -82,7 +78,7 @@ jobs:
        if [[ "$CACHE_HIT" != 'true' ]]; then
           echo "Installing"
           cd ~/depend
-          echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}" > ~/depend/cache-key
+          echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ hashfiles('**/.github/workflows/macos.yml') }}" > ~/depend/cache-key
           wget -nv --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-x86_64.tar.gz
           tar -xf cmake-3.25.1-linux-x86_64.tar.gz
           $HOME/depend/cmake-3.25.1-linux-x86_64/bin/cmake --version