Merge branch 'hypre-cuda-fixes' into tsmp-pdaf-patched

Author: jjokella

.github/workflows/linux.yml

@@ -4,7 +4,7 @@ on: [push, pull_request]
 
 jobs:
   style-check:
-     runs-on: ubuntu-22.04
+     runs-on: ubuntu-24.04
      name: Check Style
      steps:
        - name: Checkout commit
@@ -121,9 +121,9 @@ jobs:
             memory_manager: "none"
           }
         - {
-            name: "Ubuntu 22.04 GCC Python",
-            os: ubuntu-22.04,
-            cudaos: 'ubuntu2204',
+            name: "Ubuntu 24.04 GCC Python",
+            os: ubuntu-24.04,
+            cudaos: 'ubuntu2404',
             cc: "gcc", cxx: "g++", fc: "gfortran",
             python: "true",
             backend: "none",
@@ -143,9 +143,9 @@ jobs:
             memory_manager: "none"
           }
         - {
-            name: "Ubuntu 22.04 GCC OMP",
-            os: ubuntu-22.04,
-            cudaos: 'ubuntu2204',
+            name: "Ubuntu 24.04 GCC OMP",
+            os: ubuntu-24.04,
+            cudaos: 'ubuntu2404',
             cc: "gcc", cxx: "g++", fc: "gfortran",
             python: "false",
             backend: "omp",
@@ -156,7 +156,7 @@ jobs:
         - {
             name: "Ubuntu 24.04 LLVM OASIS3-MCT Build",
             os: ubuntu-24.04,
-            cudaos: 'ubuntu2204',
+            cudaos: 'ubuntu2404',
             cc: "clang-18", cxx: "clang++-18", fc: "gfortran",
             python: "false",
             backend: "omp",
@@ -181,7 +181,7 @@ jobs:
             cc: "gcc", cxx: "g++", fc: "gfortran",
             python: "false",
             backend: "cuda",
-            cudaos: 'ubuntu2204',
+            cudaos: 'ubuntu2404',
             amps_layer: mpi1,
             netcdf: "false",
             memory_manager: "umpire"
@@ -203,7 +203,7 @@ jobs:
             cc: "gcc", cxx: "g++", fc: "gfortran",
             python: "false",
             backend: "kokkos",
-            cudaos: 'ubuntu2204',
+            cudaos: 'ubuntu2404',
             amps_layer: mpi1,
             netcdf: "false",
             memory_manager: "umpire"
@@ -217,15 +217,7 @@ jobs:
         sudo apt-get -qq update
         sudo apt-get -qq install curl cmake build-essential tcl-dev tk-dev gfortran hdf5-helpers libcurl4 libcurl4-gnutls-dev
 
-    - name: HDF5 and OpenMPI Install on Ubuntu 22.04
-      if: matrix.config.os == 'ubuntu-22.04'
-      run: |
-        sudo apt-get -qq update
-        sudo apt -qq install libhdf5-openmpi-103 libhdf5-openmpi-dev
-
-    # libhdf5-openmpi-103 has been renamed in Ubuntu 24.04.
-    - name: HDF5 and OpenMPI Install on Ubuntu 24.04
-      if: matrix.config.os == 'ubuntu-24.04'
+    - name: HDF5 and OpenMPI Install
       run: |
         sudo apt-get -qq update
         sudo apt -qq install libhdf5-openmpi-103-1t64 libhdf5-openmpi-dev
@@ -305,27 +297,13 @@ jobs:
         fi
         echo "$HOME/depend/cmake-3.26.5-linux-x86_64/bin" >> $GITHUB_PATH
 
-    - name: NVidia Driver Install
+    - name: NVidia compiler and CUDA toolkit Install
       if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
-      run: |
-        sudo add-apt-repository -y ppa:graphics-drivers
-        sudo apt-get update -qq
-        sudo apt-get install -y nvidia-driver-535
-
-    - name: CUDA Install
-      if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
-      run: |
-        export CUDA_VERSION_MAJOR_MINOR=12.4.0
-        export CUDA_SHORT=12.4
-        export CUDA_OS=${{ matrix.config.cudaos }}
-        ./bin/install-cuda.sh
-        export CUDA_HOME=/usr/local/cuda-${CUDA_SHORT}
-        echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
-        export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
-        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-        export LIBRARY_PATH=${CUDA_HOME}/lib64/stubs
-        echo "LIBRARY_PATH=${LIBRARY_PATH}" >> $GITHUB_ENV
-        echo "${CUDA_HOME}/bin" >> $GITHUB_PATH
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: nvidia-cuda-toolkit nvidia-cuda-dev
+        version: 1.0
+        execute_install_scripts: true
 
     - name: UCX
       if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
@@ -366,11 +344,12 @@ jobs:
         if [[ "$CACHE_HIT" != 'true' ]]; then
           git clone --recursive https://github.com/LLNL/Umpire.git
           cd Umpire
-          mkdir build
-          cd build
-          cmake ../ -DCMAKE_INSTALL_PREFIX=$PARFLOW_DEP_DIR/Umpire -DENABLE_CUDA=On
-          make
-          make install
+          cmake -S . -B bld                                             \
+                     -DCMAKE_CXX_COMPILER=mpicxx                        \
+                     -DCMAKE_INSTALL_PREFIX="${PARFLOW_DEP_DIR}/Umpire" \
+                     -DENABLE_CUDA="ON"
+          cmake --build bld --parallel 4
+          cmake --install bld
         fi
         echo "UMPIRE_FLAGS=-Dumpire_ROOT=$PARFLOW_DEP_DIR/Umpire" >> $GITHUB_ENV
 
@@ -469,13 +448,27 @@ jobs:
         CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
       run: |
         if [[ "$CACHE_HIT" != 'true' ]]; then
-          URL=https://github.com/hypre-space/hypre/archive/v2.26.0.tar.gz
-          mkdir hypre
-          cd hypre
-          curl -L $URL | tar --strip-components=1 -xz
-          cd src
-          ./configure --prefix=$PARFLOW_DEP_DIR
-          make -j 2 install
+          # Download
+          wget https://github.com/hypre-space/hypre/archive/v2.33.0.tar.gz
+          tar xf v2.33.0.tar.gz && cd hypre-2.33.0
+
+          # Set Hypre accelerator backend
+          if [[ "${{ matrix.config.backend }}" == "cuda" ]]; then
+            BACKEND="CUDA"    # GPU
+          else
+            BACKEND="OPENMP"  # CPU
+          fi
+
+          # Install Hypre. See https://hypre.readthedocs.io/en/latest/ch-misc.html#build-system-options
+          cmake -S src -B bld                                \
+                -DCMAKE_C_COMPILER=mpicc                     \
+                -DCMAKE_CXX_COMPILER=mpicxx                  \
+                -DCMAKE_INSTALL_PREFIX="${PARFLOW_DEP_DIR}"  \
+                -DHYPRE_ENABLE_MPI="ON"                      \
+                -DHYPRE_ENABLE_${BACKEND}="ON"
+          cmake --build bld --parallel 4
+          cmake --install bld
+          echo "" && echo "Successfully built Hypre-${BACKEND}"
         fi
 
     - name: ParFlow CMake Configure

CMakeLists.txt

@@ -225,6 +225,30 @@ if ( ${PARFLOW_AMPS_LAYER} IN_LIST PARFLOW_AMPS_LAYER_REQUIRE_MPI )
     set(PARFLOW_HAVE_MPI "yes")
     set(HAVE_MPI ${PARFLOW_HAVE_MPI})
     message(STATUS "Found MPIEXEC: ${MPIEXEC}")    
+
+    # Need to pass MPI include header to nvcc to avoid missing <mpi.h> error at amps.h.
+    if (${PARFLOW_HAVE_CUDA})
+      if (MPI_C_INCLUDE_DIRS)
+        # Construct the compiler string for the include directories.
+        foreach(dir ${MPI_C_INCLUDE_DIRS})
+          string(APPEND CUDA_MPI_INCLUDES "-I${dir} ")
+        endforeach()
+      else()
+        # Guess MPI include dir from MPI_HOME (if it exists)
+        # https://gitlab.kitware.com/cmake/cmake/-/blob/master/Modules/FindMPI.cmake?ref_type=heads#L120
+        if(DEFINED CACHE{MPI_HOME})
+          set(CUDA_MPI_INCLUDES -I${MPI_HOME}/include)
+       elseif(DEFINED ENV{MPI_HOME})
+          set(CUDA_MPI_INCLUDES -I$ENV{MPI_HOME}/include)
+        endif()
+      endif()
+
+      if(CUDA_MPI_INCLUDES)
+        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_MPI_INCLUDES}")
+      else()
+        message(WARNING "MPI include dir not found. This might cause compile problems when building CUDA sources with MPI headers.")
+      endif()
+    endif()
   endif (${MPI_C_FOUND})
 
 endif ( ${PARFLOW_AMPS_LAYER} IN_LIST PARFLOW_AMPS_LAYER_REQUIRE_MPI )

cmake/modules/FindHypre.cmake

@@ -149,6 +149,12 @@ if (Hypre_FOUND AND NOT TARGET Hypre::Hypre)
           list(APPEND _cuda_targets CUDA::cusparse)
         endif()
 
+        if(TARGET CUDA::cusolver_static)
+          list(APPEND _cuda_targets CUDA::cusolver_static)
+        elseif(TARGET CUDA::cusolver)
+          list(APPEND _cuda_targets CUDA::cusolver)
+        endif()
+
         string(JOIN ", " _hypre_cuda_targets ${_cuda_targets})
         target_link_libraries(Hypre::Hypre INTERFACE ${_cuda_targets})
         message(STATUS "Found Hypre with CUDA backend. The ff. CUDA targets will be added to the linker options: ${_hypre_cuda_targets}")