Merge pull request #4126 from lindsayad/revert-revert-hypre-memory-location

Revert the revert of setting the hypre memory location

Author: lindsayad

src/solvers/petsc_linear_solver.C

@@ -36,6 +36,11 @@
 #include "libmesh/enum_solver_type.h"
 #include "libmesh/enum_convergence_flags.h"
 
+#if defined(LIBMESH_HAVE_PETSC_HYPRE) && PETSC_VERSION_LESS_THAN(3, 23, 0) &&                      \
+    !PETSC_VERSION_LESS_THAN(3, 12, 0) && defined(PETSC_HAVE_HYPRE_DEVICE)
+#include <HYPRE_utilities.h>
+#endif
+
 // C++ includes
 #include <memory>
 #include <string.h>
@@ -571,6 +576,20 @@ PetscLinearSolver<T>::solve_base (SparseMatrix<T> * matrix,
   // Allow command line options to override anything set programmatically.
   LibmeshPetscCall(KSPSetFromOptions(_ksp));
 
+#if defined(LIBMESH_HAVE_PETSC_HYPRE) && PETSC_VERSION_LESS_THAN(3, 23, 0) &&                      \
+    !PETSC_VERSION_LESS_THAN(3, 12, 0) && defined(PETSC_HAVE_HYPRE_DEVICE)
+  {
+    // Make sure hypre has been initialized
+    LibmeshPetscCallExternal(HYPRE_Initialize);
+    PetscScalar * dummyarray;
+    PetscMemType mtype;
+    LibmeshPetscCall(VecGetArrayAndMemType(solution->vec(), &dummyarray, &mtype));
+    LibmeshPetscCall(VecRestoreArrayAndMemType(solution->vec(), &dummyarray));
+    if (PetscMemTypeHost(mtype))
+      LibmeshPetscCallExternal(HYPRE_SetMemoryLocation, HYPRE_MEMORY_HOST);
+  }
+#endif
+
   // If the SolverConfiguration object is provided, use it to override
   // solver options.
   if (this->_solver_configuration)

src/solvers/petsc_nonlinear_solver.C

@@ -34,6 +34,11 @@
 #include "libmesh/petscdmlibmesh.h"
 #include "libmesh/petsc_mffd_matrix.h"
 
+#if defined(LIBMESH_HAVE_PETSC_HYPRE) && PETSC_VERSION_LESS_THAN(3, 23, 0) &&                      \
+    !PETSC_VERSION_LESS_THAN(3, 12, 0) && defined(PETSC_HAVE_HYPRE_DEVICE)
+#include <HYPRE_utilities.h>
+#endif
+
 namespace libMesh
 {
 class ResidualContext
@@ -1069,6 +1074,20 @@ PetscNonlinearSolver<T>::solve (SparseMatrix<T> &  pre_in,  // System Preconditi
 #endif
   LibmeshPetscCall(SNESSetFromOptions(_snes));
 
+#if defined(LIBMESH_HAVE_PETSC_HYPRE) && PETSC_VERSION_LESS_THAN(3, 23, 0) &&                      \
+    !PETSC_VERSION_LESS_THAN(3, 12, 0) && defined(PETSC_HAVE_HYPRE_DEVICE)
+  {
+    // Make sure hypre has been initialized
+    LibmeshPetscCallExternal(HYPRE_Initialize);
+    PetscScalar * dummyarray;
+    PetscMemType mtype;
+    LibmeshPetscCall(VecGetArrayAndMemType(x->vec(), &dummyarray, &mtype));
+    LibmeshPetscCall(VecRestoreArrayAndMemType(x->vec(), &dummyarray));
+    if (PetscMemTypeHost(mtype))
+      LibmeshPetscCallExternal(HYPRE_SetMemoryLocation, HYPRE_MEMORY_HOST);
+  }
+#endif
+
   if (this->user_presolve)
     this->user_presolve(this->system());