Add option to use Sundials library instead of the embedded Kinsol (parflow#657)

The embedded Kinsol version is very old and has not been updated since
the early versions of ParFlow. A compile time option has been added to
use the Sundials library version of Kinsol.

Some tests were modified to use EtaConst as this is the more stable
option but possibly less performant. This provided consistency between
the old and new Kinsol results.

Some tests were setting EtaValue when using Walker1 and Walker2
EtaChoice options. The key is ignored for those options so removed for
better clarity.

A bug in Walker1 was backported to the embedded Kinsol version from the
new Sundials. This provides better consistency between the old and new
Kinsol and should provide better solver performance for Walker1.

Walker2 results were the most inconsistent between the old and new
kinsol. Setting EtaGammer to a tighter constraint improves consistency.

---------

Co-authored-by: Daniel Osei-Kuffuor <oseikuffuor1@llnl.gov>

Author: smithsg84

.github/workflows/linux.yml

@@ -121,7 +121,21 @@ jobs:
             amps_layer: mpi1,
             netcdf: "true",
             memory_manager: "none",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
+          }
+        - {
+            name: "Ubuntu 24.04 GCC Sundials",
+            os: ubuntu-24.04,
+            cudaos: 'ubuntu2404',
+            cc: "gcc", cxx: "g++", fc: "gfortran",
+            python: "false",
+            backend: "none",
+            amps_layer: mpi1,
+            netcdf: "true",
+            memory_manager: "none",
+            pdi: "false",
+            sundials: "external"
           }
         - {
             name: "Ubuntu 24.04 LLVM",
@@ -133,7 +147,8 @@ jobs:
             amps_layer: mpi1,
             netcdf: "true",
             memory_manager: "none",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
           }
         - {
             name: "Ubuntu 22.04 GCC Python",
@@ -145,7 +160,8 @@ jobs:
             amps_layer: mpi1,
             netcdf: "true",
             memory_manager: "none",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
           }
         - {
             name: "Ubuntu 24.04 GCC Python",
@@ -157,7 +173,8 @@ jobs:
             amps_layer: mpi1,
             netcdf: "false",
             memory_manager: "none",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
           }
         - {
             name: "Ubuntu 24.04 GCC OMP",
@@ -169,7 +186,8 @@ jobs:
             amps_layer: mpi1,
             netcdf: "false",
             memory_manager: "none",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
           }
         - {
             name: "Ubuntu 24.04 LLVM OASIS3-MCT Build",
@@ -181,7 +199,8 @@ jobs:
             amps_layer: oas3,
             netcdf: "true",
             memory_manager: "none",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
           }
 #        - {
 #            name: "Ubuntu 24.04 GCC CUDA/RMM Build",
@@ -204,7 +223,8 @@ jobs:
             amps_layer: mpi1,
             netcdf: "false",
             memory_manager: "umpire",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
           }
 #        - {
 #            name: "Ubuntu 24.04 GCC Kokkos/RMM Build",
@@ -227,7 +247,8 @@ jobs:
             amps_layer: mpi1,
             netcdf: "false",
             memory_manager: "umpire",
-            pdi: "false"
+            pdi: "false",
+            sundials: "embedded"
           }
         - {
             name: "Ubuntu 24.04 GCC PDI Build",
@@ -239,7 +260,8 @@ jobs:
             amps_layer: mpi1,
             netcdf: "false",
             memory_manager: "none",
-            pdi: "true"
+            pdi: "true",
+            sundials: "embedded"
           }
 
     steps:
@@ -499,7 +521,7 @@ jobs:
           fi
 
           # Install Hypre. See https://hypre.readthedocs.io/en/latest/ch-misc.html#build-system-options
-          cmake -S src -B build                                \
+          cmake -S src -B build                              \
                 -DCMAKE_C_COMPILER=mpicc                     \
                 -DCMAKE_CXX_COMPILER=mpicxx                  \
                 -DCMAKE_INSTALL_PREFIX="${PARFLOW_DEP_DIR}"  \
@@ -510,6 +532,25 @@ jobs:
           echo "" && echo "Successfully built Hypre ${BACKEND}"
         fi
 
+    - name: Sundials Install
+      env:
+        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
+      if: matrix.config.sundials == 'external'
+      run: |
+        if [[ "$CACHE_HIT" != 'true' ]]; then
+          # Download
+          wget https://github.com/llnl/sundials/archive/v7.4.0.tar.gz
+          tar xf v7.4.0.tar.gz
+          cd sundials-7.4.0
+          cmake -S . -B build                   \
+                -DCMAKE_C_COMPILER=mpicc                     \
+                -DCMAKE_CXX_COMPILER=mpicxx                  \
+                -DCMAKE_INSTALL_PREFIX="${PARFLOW_DEP_DIR}"
+          cmake --build build --parallel 4
+          cmake --install build
+          echo "SUNDIALS_FLAGS=-DSUNDIALS_ROOT=${PARFLOW_DEP_DIR}" >> $GITHUB_ENV
+        fi
+
     - name: PDI Install
       env:
         CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
@@ -559,7 +600,7 @@ jobs:
            -DSILO_ROOT=$PARFLOW_DEP_DIR                                 \
            -DPARFLOW_ENABLE_PYTHON=${{ matrix.config.python }}          \
            -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR                          \
-           $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS $UMPIRE_FLAGS $PDI_FLAGS
+           $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS $UMPIRE_FLAGS $PDI_FLAGS $SUNDIALS_FLAGS
 
     - name: ParFlow CMake Build
       run: (cd build; make -j 2 install)

CMakeLists.txt

@@ -373,11 +373,13 @@ endif (${PARFLOW_ENABLE_SZLIB} OR DEFINED SZLIB_ROOT)
 #-----------------------------------------------------------------------------
 option(PARFLOW_ENABLE_SUNDIALS "Build with SUNDIALS" "OFF")
 if (${PARFLOW_ENABLE_SUNDIALS} OR DEFINED SUNDIALS_ROOT)
-  find_package(SUNDIALS COMPONENTS sundials_cvode sundials_kinsol)
+  find_package(SUNDIALS COMPONENTS sundials_cvode sundials_kinsol sundials_core sundials_sunlinsolspgmr)
   if (${SUNDIALS_FOUND})
     set(PARFLOW_HAVE_SUNDIALS "yes")
     set(HAVE_SUNDIALS ${PARFLOW_HAVE_SUNDIALS})
   endif (${SUNDIALS_FOUND})
+else ()
+    message (STATUS "Found KINSOL: embedded")
 endif (${PARFLOW_ENABLE_SUNDIALS} OR DEFINED SUNDIALS_ROOT)
 
 #-----------------------------------------------------------------------------

README.md

@@ -350,6 +350,21 @@ Here are some common packages:
 - __SZLIB__: builds ParFlow with SZlib compression library
 - __ZLIB__: builds ParFlow with Zlib compression library
 
+### KINSOL Solver
+
+There are two compile options for the KINSOL solver used by ParFlow.
+The current default is to use an old version of KINSOL that is
+embedded in ParFlow.  The newer option is to use KINSOL from the
+SUNDIALS package.   To use the SUNDIALS package you will need to have a 
+SUNDIALS install that includes KINSOL and specify during the CMake configuration:
+
+```shell
+   -DSUNDIALS_ROOT=<directory of the location of sundials install>
+```
+
+The GitHub repository for [SUNDIALS](https://github.com/LLNL/sundials)
+has downloads and build instructions.
+
 ### How to specify the launcher command used to run MPI applications
 
 There are multiple ways to run MPI applications such as mpiexec,

bin/valgrind.sup

@@ -199,6 +199,14 @@
    ...
 }
 
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Cond
+   fun:__intel_sse2_strncmp
+   ...
+   fun:MPIDI_CH3I_set_affinity
+}
+
 {
    Clone
    Memcheck:Leak
@@ -253,6 +261,37 @@
    fun:PMPI_Init
 }
 
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:calloc
+   obj:/usr/lib64/libibverbs.so.1.14.48.0
+   fun:nl_recvmsgs_report
+   fun:nl_recvmsgs
+   obj:/usr/lib64/libibverbs.so.1.14.48.0
+   obj:/usr/lib64/libibverbs.so.1.14.48.0
+   obj:/usr/lib64/libibverbs.so.1.14.48.0
+   fun:ibv_get_device_list
+   ...
+   fun:rdma_cm_get_hca_type
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:calloc
+   obj:/usr/lib64/libmlx4.so.1.0.48.0
+   obj:/usr/lib64/libibverbs.so.1.14.48.0
+   obj:/usr/lib64/libibverbs.so.1.14.48.0
+   obj:/usr/lib64/libibverbs.so.1.14.48.0
+   fun:ibv_get_device_list
+   ...
+   fun:MPID_Init
+}
+
+
 {
    <insert_a_suppression_name_here>
    Memcheck:Cond
@@ -269,7 +308,6 @@
    ...
 }
 
-
 {
    <insert_a_suppression_name_here>
    Memcheck:Cond
@@ -732,3 +770,4 @@
    fun:cudaMemPrefetchAsync
    ...
 }
+

pfsimulator/CMakeLists.txt

@@ -8,7 +8,11 @@ string(APPEND CMAKE_C_FLAGS " ${PARFLOW_ETRACE_COMPILE_FLAGS}")
 string(APPEND CMAKE_CXX_FLAGS " ${PARFLOW_ETRACE_COMPILE_FLAGS}")
 
 add_subdirectory (amps)
-add_subdirectory (kinsol)
+
+if (NOT PARFLOW_HAVE_SUNDIALS)
+  add_subdirectory (kinsol)
+endif ()
+
 add_subdirectory (parflow_lib)
 add_subdirectory (parflow_exe)
 add_subdirectory (third_party)