processor now takes into account the domain size when computing the number of IB force communications. This limits the number of transmits and prevents double-counting in periodic cases.

Author: danieljvickers

src/common/m_global_parameters_common.fpp

@@ -109,8 +109,11 @@ module m_global_parameters_common
 
     !> @name Processor coordinates and parallel-IO addressing (identical declaration across all three targets)
     !> @{
-    integer, allocatable, dimension(:) :: proc_coords  !< Processor coordinates in MPI_CART_COMM
-    integer, allocatable, dimension(:) :: start_idx    !< Starting cell-center index of local processor in global grid
+    integer, allocatable, dimension(:) :: proc_coords      !< Processor coordinates in MPI_CART_COMM
+    integer, allocatable, dimension(:) :: start_idx        !< Starting cell-center index of local processor in global grid
+    integer                            :: num_procs_x = 1  !< Number of MPI ranks in x-direction
+    integer                            :: num_procs_y = 1  !< Number of MPI ranks in y-direction
+    integer                            :: num_procs_z = 1  !< Number of MPI ranks in z-direction
     !> @}
 
     !> @name MPI info for parallel IO with Lustre file systems (identical across all three targets)

src/common/m_mpi_common.fpp

@@ -1021,7 +1021,6 @@ contains
     subroutine s_mpi_decompose_computational_domain
 
 #ifdef MFC_MPI
-        integer :: num_procs_x, num_procs_y, num_procs_z  !< Optimal number of processors in the x-, y- and z-directions
         !> Non-optimal number of processors in the x-, y- and z-directions
         real(wp) :: tmp_num_procs_x, tmp_num_procs_y, tmp_num_procs_z
         real(wp) :: fct_min        !< Processor factorization (fct) minimization parameter

src/simulation/m_ibm.fpp

@@ -1252,7 +1252,7 @@ contains
                 recv_torques_snap = 0._wp
                 tag = 300
 
-                do k = 1, 2*ib_neighborhood_radius
+                do k = 1, min(2*ib_neighborhood_radius, num_procs_${X}$ - 1)
                     ! send forces to +${X}$ neighbor; receive from -${X}$ neighbor. Add received values then
                     pack_pos = 0
                     $:GPU_PARALLEL_LOOP(private='[i]', copyin='[forces, torques]')
@@ -1300,7 +1300,7 @@ contains
                 send_neighbor = merge(bc_${X}$%beg, MPI_PROC_NULL, bc_${X}$%beg >= 0)
                 recv_neighbor = merge(bc_${X}$%end, MPI_PROC_NULL, bc_${X}$%end >= 0)
 
-                do k = 1, 2*ib_neighborhood_radius
+                do k = 1, min(2*ib_neighborhood_radius, num_procs_${X}$ - 1)
                     pack_pos = 0
                     $:GPU_PARALLEL_LOOP(private='[i]', copyin='[forces, torques]')
                     do i = 1, num_ibs