Fix GPU cases, verified on frontier with cce-acc, cce-omp and amdflang

Author: Anand

CMakeLists.txt

@@ -710,8 +710,15 @@ exit 0
                         PRIVATE -DFRONTIER_UNIFIED)
                 endif()
 
-                find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
-                target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
+		        find_library(HIP_LIB amdhip64
+                    HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
+                find_library(HIPFORT_AMDGCN_LIB hipfort-amdgcn
+                    HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
+                target_include_directories(${a_target} PRIVATE
+                    "$ENV{OLCF_AFAR_ROOT}/include/hipfort/amdgcn")
+                target_link_libraries(${a_target} PRIVATE
+                    ${HIP_LIB} ${HIPFORT_AMDGCN_LIB})
+
             endif()
         elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
             target_compile_options(${a_target} PRIVATE "SHELL:-h noacc" "SHELL:-x acc")

src/simulation/m_rhs.fpp

@@ -110,6 +110,9 @@ module m_rhs
     $:GPU_DECLARE(create='[qL_rsx_vf, qR_rsx_vf]')
     $:GPU_DECLARE(create='[dqL_rsx_vf, dqR_rsx_vf]')
 
+    integer :: iglob
+    $:GPU_DECLARE(create='[iglob]')
+
 contains
 
     !> Initialize the RHS module
@@ -616,14 +619,16 @@ contains
                         iv%beg = 1; iv%end = eqn_idx%cont%end
                         call s_reconstruct_cell_boundary_values(q_prim_qp%vf(iv%beg:iv%end), qL_rsx_vf, qR_rsx_vf, id)
 
-                        iv%beg = eqn_idx%mom%beg; iv%end = eqn_idx%mom%end
-                        $:GPU_PARALLEL_LOOP(collapse=4)
+                        iv%beg = eqn_idx%mom%beg; iv%end = eqn_idx%mom%end; iglob = id
+                        $:GPU_UPDATE(device='[iv, iglob]')
+
+                        $:GPU_PARALLEL_LOOP(collapse=4, private='[i, j, k, l]')
                         do i = iv%beg, iv%end
                             do l = idwbuff(3)%beg, idwbuff(3)%end
                                 do k = idwbuff(2)%beg, idwbuff(2)%end
                                     do j = idwbuff(1)%beg, idwbuff(1)%end
-                                        qL_rsx_vf(j, k, l, i) = qL_prim(id)%vf(i)%sf(j, k, l)
-                                        qR_rsx_vf(j, k, l, i) = qR_prim(id)%vf(i)%sf(j, k, l)
+                                        qL_rsx_vf(j, k, l, i) = qL_prim(iglob)%vf(i)%sf(j, k, l)
+                                        qR_rsx_vf(j, k, l, i) = qR_prim(iglob)%vf(i)%sf(j, k, l)
                                     end do
                                 end do
                             end do
@@ -654,14 +659,16 @@ contains
                         iv%beg = 1; iv%end = eqn_idx%cont%end
                         call s_reconstruct_cell_boundary_values(q_prim_qp%vf(iv%beg:iv%end), qL_rsx_vf, qR_rsx_vf, id)
 
-                        iv%beg = eqn_idx%mom%beg; iv%end = eqn_idx%mom%end
-                        $:GPU_PARALLEL_LOOP(collapse=4)
+                        iv%beg = eqn_idx%mom%beg; iv%end = eqn_idx%mom%end; iglob = id
+                        $:GPU_UPDATE(device='[iv, iglob]')
+
+                        $:GPU_PARALLEL_LOOP(collapse=4, private='[i, j, k, l]')
                         do i = iv%beg, iv%end
                             do l = idwbuff(3)%beg, idwbuff(3)%end
                                 do k = idwbuff(2)%beg, idwbuff(2)%end
                                     do j = idwbuff(1)%beg, idwbuff(1)%end
-                                        qL_rsx_vf(j, k, l, i) = qL_prim(id)%vf(i)%sf(j, k, l)
-                                        qR_rsx_vf(j, k, l, i) = qR_prim(id)%vf(i)%sf(j, k, l)
+                                        qL_rsx_vf(j, k, l, i) = qL_prim(iglob)%vf(i)%sf(j, k, l)
+                                        qR_rsx_vf(j, k, l, i) = qR_prim(iglob)%vf(i)%sf(j, k, l)
                                     end do
                                 end do
                             end do

src/simulation/m_surface_tension.fpp

@@ -290,7 +290,9 @@ contains
         integer, intent(in)                                                                       :: norm_dir
         integer                                                                                   :: i, j, k, l
 
-        $:GPU_PARALLEL_LOOP(collapse=4)
+        $:GPU_UPDATE(device='[iv]')
+
+        $:GPU_PARALLEL_LOOP(collapse=4, private='[i, j, k, l]')
         do i = iv%beg, iv%end
             do l = idwbuff(3)%beg, idwbuff(3)%end
                 do k = idwbuff(2)%beg, idwbuff(2)%end