Added profiling and increased maximum num IBs to 1000

Author: danieljvickers

src/common/m_constants.fpp

@@ -22,7 +22,7 @@ module m_constants
     integer, parameter :: fourier_rings = 5                       !< Fourier filter ring limit
     integer, parameter :: num_fluids_max = 10                     !< Maximum number of fluids in the simulation
     integer, parameter :: num_probes_max = 10                     !< Maximum number of flow probes in the simulation
-    integer, parameter :: num_patches_max = 10
+    integer, parameter :: num_patches_max = 1000
     integer, parameter :: num_bc_patches_max = 10
     integer, parameter :: pathlen_max = 400
     integer, parameter :: nnode = 4    !< Number of QBMM nodes

src/simulation/m_ibm.fpp

@@ -84,6 +84,8 @@ contains
         integer :: i, j, k
         integer :: max_num_gps, max_num_inner_gps
 
+        call nvtxStartRange("SETUP-IBM-MODULE")
+
         ! do all set up for moving immersed boundaries
         moving_immersed_boundary_flag = .false.
         do i = 1, num_ibs
@@ -108,7 +110,7 @@ contains
         ib_markers%sf = 0._wp
         $:GPU_UPDATE(device='[ib_markers%sf]')
         call s_apply_ib_patches(ib_markers)
-        ! call s_populate_ib_buffers()
+        call s_populate_ib_buffers()
         $:GPU_UPDATE(host='[ib_markers%sf]')
         do i = 1, num_ibs
             if (patch_ib(i)%moving_ibm /= 0) call s_compute_centroid_offset(i) ! offsets are computed after IB markers are generated
@@ -141,6 +143,8 @@ contains
         call s_compute_interpolation_coeffs(ghost_points)
         $:GPU_UPDATE(device='[ghost_points]')
 
+        call nvtxEndRange
+
     end subroutine s_ibm_setup
 
     subroutine s_populate_ib_buffers()
@@ -989,6 +993,8 @@ contains
 
         integer :: i, ierr
 
+        call nvtxStartRange("UPDATE-MIB")
+
         ! Clears the existing immersed boundary indices
         ib_markers%sf = 0._wp
 
@@ -1002,9 +1008,12 @@ contains
         $:GPU_UPDATE(device='[patch_ib]')
 
         ! recompute the new ib_patch locations and broadcast them.
-         $:GPU_UPDATE(device='[ib_markers%sf]')
+        call nvtxStartRange("COMPUTE-IB-MARKERS")
+        $:GPU_UPDATE(device='[ib_markers%sf]')
         call s_apply_ib_patches(ib_markers)
+        call s_populate_ib_buffers()
         $:GPU_UPDATE(host='[ib_markers%sf]')
+        call nvtxEndRange
 
         ! recalculate the ghost point locations and coefficients
         call s_find_num_ghost_points(num_gps, num_inner_gps)
@@ -1013,15 +1022,19 @@ contains
         call s_find_ghost_points(ghost_points, inner_points)
         $:GPU_UPDATE(device='[ghost_points, inner_points]')
 
+        call nvtxStartRange("APPLY-LEVELSET")
         call s_apply_levelset(ghost_points, num_gps)
         $:GPU_UPDATE(device='[ghost_points]')
+        call nvtxEndRange
 
         call s_compute_image_points(ghost_points)
         $:GPU_UPDATE(device='[ghost_points]')
 
         call s_compute_interpolation_coeffs(ghost_points)
         $:GPU_UPDATE(device='[ghost_points]')
 
+        call nvtxEndRange
+
     end subroutine s_update_mib
 
     ! compute the surface integrals of the IB via a volume integraion method described in

src/simulation/m_time_steppers.fpp

@@ -802,6 +802,8 @@ contains
         integer :: i
         logical :: forces_computed
 
+        call nvtxStartRange("PROPAGATE-IMERSED-BOUNDARIES")
+
         forces_computed = .false.
 
         do i = 1, num_ibs
@@ -849,6 +851,8 @@ contains
 
         call s_update_mib(num_ibs)
 
+        call nvtxEndRange
+
     end subroutine s_propagate_immersed_boundaries
 
     !> This subroutine saves the temporary q_prim_vf vector

toolchain/mfc/case_validator.py

@@ -684,7 +684,7 @@ def check_ibm(self):
 
         self.prohibit(ib and n <= 0,
                      "Immersed Boundaries do not work in 1D (requires n > 0)")
-        self.prohibit(ib and (num_ibs <= 0 or num_ibs > 10),
+        self.prohibit(ib and (num_ibs <= 0 or num_ibs > 1000),
                      "num_ibs must be between 1 and num_patches_max (10)")
         self.prohibit(not ib and num_ibs > 0,
                      "num_ibs is set, but ib is not enabled")

toolchain/mfc/params/definitions.py

@@ -11,7 +11,7 @@
 from .registry import REGISTRY
 
 # Index limits
-NP, NF, NI, NA, NPR, NB = 10, 10, 10, 4, 10, 10  # patches, fluids, ibs, acoustic, probes, bc_patches
+NP, NF, NI, NA, NPR, NB = 10, 10, 1000, 4, 10, 10  # patches, fluids, ibs, acoustic, probes, bc_patches
 
 
 # =============================================================================
@@ -672,7 +672,7 @@ def get_value_label(param_name: str, value: int) -> str:
     # Counts (must be positive)
     "num_fluids": {"min": 1, "max": 10},
     "num_patches": {"min": 0, "max": 10},
-    "num_ibs": {"min": 0, "max": 10},
+    "num_ibs": {"min": 0, "max": 1000},
     "num_source": {"min": 1},
     "num_probes": {"min": 1},
     "num_integrals": {"min": 1},