Merge branch 'env-cholqr' into 'master'

Add the option to enable/disable CholQR through ChASE config

See merge request SLai/ChASE!30

Author: brunowu

ChASE-MPI/CMakeLists.txt

@@ -5,11 +5,6 @@ cmake_minimum_required( VERSION 3.8 )
 
 # project( ChASE-MPI LANGUAGES C CXX CUDA )
 
-find_package(OpenMP)
-if(OpenMP_FOUND)
-  target_compile_definitions( chase_algorithm  INTERFACE "HAS_OMP" )
-endif()
-
 add_library( chase_seq INTERFACE )
 target_link_libraries(chase_seq INTERFACE chase_algorithm)
 target_compile_definitions(chase_seq INTERFACE NO_MPI=1 )
@@ -60,12 +55,6 @@ if(SCALAPACK_FOUND)
   	)
 endif()
 
-if(OpenMP_FOUND)
-    target_link_libraries( chase_seq INTERFACE
-	    OpenMP::OpenMP_CXX
-        )
-endif()
-
 target_compile_options( chase_seq INTERFACE
   ${CMAKE_C_FLAGS}
   ${MPI_CXX_COMPILE_FLAGS}  
@@ -94,16 +83,10 @@ if(SCALAPACK_FOUND)
 	)
 endif()
 
-  target_compile_definitions( chase_mpi INTERFACE
+target_compile_definitions( chase_mpi INTERFACE
 	  "-DUSE_MPI"
         )
 
-if(OpenMP_FOUND)
-    target_link_libraries( chase_mpi INTERFACE
-            OpenMP::OpenMP_CXX
-        )
-endif()
-
 include(CheckLanguage)
 check_language(CUDA)
 if(CMAKE_CUDA_COMPILER)

ChASE-MPI/chase_mpi.hpp

@@ -15,10 +15,6 @@
 
 #include <random>
 
-#ifdef HAS_OMP
-#include <omp.h>
-#endif
-
 #include "algorithm/chase.hpp"
 
 #include "blas_templates.hpp"
@@ -471,13 +467,23 @@ class ChaseMpi : public chase::Chase<T>
 #ifdef USE_MPI        
         MPI_Comm_rank(MPI_COMM_WORLD, &grank);
 #endif        
+        int diasable = 0;
+
+        if(config_.DoCholQR())
+        {
+            diasable = 0;
+        }else
+        {
+            diasable = 1;
+        }
+
         char* cholddisable;
         cholddisable = getenv("CHASE_DISABLE_CHOLQR");
-        int diasable = 0;
         if (cholddisable)
         {
             diasable = std::atoi(cholddisable);
         }
+
         if (diasable == 1)
         {
 #ifdef CHASE_OUTPUT
@@ -573,16 +579,7 @@ class ChaseMpi : public chase::Chase<T>
 #ifdef USE_NSIGHT
         nvtxRangePop();
 #endif
-#ifdef HAS_OMP
-        char* omp_threads;
-        omp_threads = getenv("OMP_NUM_THREADS");
-        int num_threads = 1;
-        if (omp_threads)
-        {
-            num_threads = std::atoi(omp_threads);
-        }
-        omp_set_num_threads(1);
-#endif
+
 #ifdef USE_NSIGHT
         nvtxRangePushA("Lanczos: loop");
 #endif
@@ -643,9 +640,7 @@ class ChaseMpi : public chase::Chase<T>
 #ifdef USE_NSIGHT
         nvtxRangePop();
 #endif
-#ifdef HAS_OMP
-        omp_set_num_threads(num_threads);
-#endif	
+	
         delete[] ritzv;
         delete[] isuppz;
         delete[] d;
@@ -693,16 +688,6 @@ class ChaseMpi : public chase::Chase<T>
 #ifdef USE_NSIGHT
         nvtxRangePop();
 #endif
-#ifdef HAS_OMP
-        char* omp_threads;
-        omp_threads = getenv("OMP_NUM_THREADS");
-        int num_threads = 1;
-        if (omp_threads)
-        {
-            num_threads = std::atoi(omp_threads);
-        }
-        omp_set_num_threads(1);
-#endif	
         // ENSURE that v1 has one norm
 #ifdef USE_NSIGHT
         nvtxRangePushA("Lanczos: loop");
@@ -762,9 +747,7 @@ class ChaseMpi : public chase::Chase<T>
         {
             Tau[k] = std::abs(ritzV[k * m]) * std::abs(ritzV[k * m]);
         }
-#ifdef HAS_OMP
-        omp_set_num_threads(num_threads);
-#endif	
+
         delete[] isuppz;
         delete[] d;
         delete[] e;

ChASE-MPI/impl/chase_mpidla_blaslapack.hpp

@@ -265,28 +265,15 @@ class ChaseMpiDLABlaslapack : public ChaseMpiDLAInterface<T>
     //!      within the row communicator.
     void Resd(Base<T>* ritzv, Base<T>* resid, std::size_t locked,
               std::size_t unconverged) override
-    {
-#ifdef HAS_OMP
-        char* omp_threads;
-        omp_threads = getenv("OMP_NUM_THREADS");
-        int num_threads = 1;
-        if (omp_threads)
-        {
-            num_threads = std::atoi(omp_threads);
-        }
-        omp_set_num_threads(1);
-#endif   	    
+    { 	    
         for (auto i = 0; i < unconverged; i++)
         {
             T alpha = -ritzv[i];
             t_axpy(n_, &alpha, B2_ + locked * n_ + i * n_, 1,
                    B_ + locked * n_ + i * n_, 1);
 
             resid[i] = t_norm_p2(n_, B_ + locked * n_ + i * n_);
-        }
-#ifdef HAS_OMP
-        omp_set_num_threads(num_threads);
-#endif   	
+        } 	
     }
 
     //! - This function performs the local computation for ChaseMpiDLA::heevd()

ChASE-MPI/impl/chase_mpidla_mgpu.hpp

@@ -636,16 +636,6 @@ class ChaseMpiDLAMultiGPU : public ChaseMpiDLAInterface<T>
     void Resd(Base<T>* ritzv, Base<T>* resid, std::size_t locked,
               std::size_t unconverged) override
     {
-#ifdef HAS_OMP
-        char* omp_threads;
-        omp_threads = getenv("OMP_NUM_THREADS");
-        int num_threads = 1;
-        if (omp_threads)
-        {
-            num_threads = std::atoi(omp_threads);
-        }
-        omp_set_num_threads(1);
-#endif
         for (auto i = 0; i < unconverged; i++)
         {
             T alpha = -ritzv[i];
@@ -655,9 +645,6 @@ class ChaseMpiDLAMultiGPU : public ChaseMpiDLAInterface<T>
             resid[i] = t_norm_p2(n_, B_ + locked * n_ + i * n_);
 
         }
-#ifdef HAS_OMP
-        omp_set_num_threads(num_threads);
-#endif
     }
     //! - This function performs the local computation for ChaseMpiDLA::heevd()
     //! - It is implemented based on `cuBLAS`'s `xgemm` and cuSOLVER's

algorithm/configuration.hpp

@@ -427,6 +427,21 @@ class ChaseConfig
      */
     std::size_t GetNex() const { return nex_; }
 
+    //! Sets the `cholqr_` flag to either `true` or `false`.
+    /*! This function is used to change the value of `cholqr_` so
+        either flexible CholQR (`true`) or Househoulder QR (`false`)
+        will be used. 
+        \param flag A boolean parameter which admits either a `true` or `false`
+       value.
+     */
+    void SetCholQR(bool flag){
+        cholqr_ = flag;
+    }
+    //! Return the value of `cholqr_`
+    bool DoCholQR(){
+        return cholqr_;
+    }
+
 private:
     ///////////////////////////////////////////////////
     // General parameters of the eigensolver
@@ -542,6 +557,9 @@ class ChaseConfig
         default value is set to *4*.
      */
     std::size_t num_lanczos_;
+
+    //! Optional parameter indicating if CholeksyQR is disabled
+    bool cholqr_ = true;
 };
 
 template <typename T>

examples/4_interface/4_c_dist_chase.c

@@ -23,7 +23,7 @@ void pzchase_init_(int* N, int* nev, int* nex, int* m, int* n,
                    double* ritzv, int* dim0, int* dim1, char* grid_major,
                    MPI_Comm* comm, int* init);
 void pzchase_finalize_(int* flag);
-void pzchase_(int* deg, double* tol, char* mode, char* opt);
+void pzchase_(int* deg, double* tol, char* mode, char* opt, char *qr);
 
 int main(int argc, char** argv)
 {
@@ -52,6 +52,7 @@ int main(int argc, char** argv)
     double tol = 1e-10;
     char mode = 'R';
     char opt = 'S';
+    char qr = 'C';
 
     if (N % dims[0] == 0)
     {
@@ -108,7 +109,7 @@ int main(int argc, char** argv)
         }
     }
 
-    pzchase_(&deg, &tol, &mode, &opt);
+    pzchase_(&deg, &tol, &mode, &opt, &qr);
 
     pzchase_finalize_(&init);
 

examples/4_interface/4_c_serial_chase.c

@@ -27,7 +27,7 @@ double random_normal()
 int zchase_init_(int* N, int* nev, int* nex, double _Complex* H, int *ldh,
                  double _Complex* V, double* ritzv, int* init);
 void zchase_finalize_();
-void zchase_(int* deg, double* tol, char* mode, char* opt);
+void zchase_(int* deg, double* tol, char* mode, char* opt, char *qr);
 
 int main(int argc, char** argv)
 {
@@ -47,6 +47,7 @@ int main(int argc, char** argv)
     double tol = 1e-10;
     char mode = 'R';
     char opt = 'S';
+    char qr = 'C';
 
     if (rank == 0)
         printf("ChASE C example driver\n");
@@ -88,7 +89,7 @@ int main(int argc, char** argv)
             }
         }
 
-        zchase_(&deg, &tol, &mode, &opt);
+        zchase_(&deg, &tol, &mode, &opt, &qr);
 
         // Perturb Full Clement matrix
         for (int i = 1; i < N; ++i)

examples/4_interface/4_f_dist_chase.f90

@@ -16,7 +16,7 @@ PROGRAM main
 real(8) :: tol
 complex(8) :: cv
 integer :: deg
-character        :: mode, opt, major
+character        :: mode, opt, major, qr
 complex(8),  allocatable :: h(:,:), v(:,:), hh(:, :)
 real(8), allocatable :: lambda(:)
 
@@ -35,6 +35,7 @@ PROGRAM main
 mode = 'R'
 opt = 'S'
 major = 'C'
+qr = 'C'
 
 dims(1) = 0
 dims(2) = 0
@@ -85,7 +86,7 @@ PROGRAM main
 	end do 
 end do
 
-call pzchase(deg, tol, mode, opt)
+call pzchase(deg, tol, mode, opt, qr)
 
 call pzchase_finalize(init)
 

examples/4_interface/4_f_serial_chase.f90

@@ -14,7 +14,7 @@ PROGRAM main
 real(8) :: tol
 complex(8) :: cv
 integer :: deg
-character        :: mode, opt
+character        :: mode, opt, qr
 complex(8),  allocatable :: h(:,:), v(:,:)
 real(8), allocatable :: lambda(:)
 
@@ -33,6 +33,7 @@ PROGRAM main
 tol = 1e-10
 mode = 'R'
 opt = 'S'
+qr = 'C'
 
 if(rank == 0) then
 	print *, "ChASE Fortran example driver"
@@ -68,7 +69,7 @@ PROGRAM main
 		end if
 	end if
 
-	call zchase(deg, tol, mode, opt)
+	call zchase(deg, tol, mode, opt, qr)
 
 	do i = 2, N
 		do j = 2, i