Fix MPI Divergent Grid Stretching (#1625)

* Added a global domain length to pre-process so that we can normalize by the global length

* Empty commit to refresh PR mergeability check

---------

Co-authored-by: Daniel Vickers <danieljvickers@login10.frontier.olcf.ornl.gov>
Co-authored-by: Spencer Bryngelson <sbryngelson@gmail.com>
Co-authored-by: Spencer Bryngelson <sbryngelson@login10.frontier.olcf.ornl.gov>

Author: 4 people

src/pre_process/m_global_parameters.fpp

@@ -38,6 +38,9 @@ module m_global_parameters
     real(wp), allocatable, dimension(:) :: x_cb, y_cb, z_cb
     real(wp) :: dx, dy, dz                             !< Minimum cell-widths in the x-, y- and z-coordinate directions
     type(bounds_info) :: x_domain, y_domain, z_domain  !< Locations of the domain bounds in the x-, y- and z-coordinate directions
+    !> Global (pre-decomposition) domain bounds, needed by s_generate_serial_grid to stretch the grid using the full domain length
+    !! rather than a local processor's sub-domain length
+    type(bounds_info) :: x_domain_glb, y_domain_glb, z_domain_glb
 
     ! Simulation Algorithm Parameters
     ! sys_size, eqn_idx, b_size, tensor_size, chemistry, elasticity, shear_*: in m_global_parameters_common

src/pre_process/m_grid.f90

@@ -49,7 +49,7 @@ impure subroutine s_generate_serial_grid
 
         ! Hyperbolic tangent grid stretching
         if (stretch_x) then
-            length = abs(x_cb(m) - x_cb(-1))
+            length = abs(x_domain_glb%end - x_domain_glb%beg)
             x_cb = x_cb/length
             x_a = x_a/length
             x_b = x_b/length
@@ -96,7 +96,7 @@ impure subroutine s_generate_serial_grid
 
         ! Hyperbolic tangent grid stretching in y-direction
         if (stretch_y) then
-            length = abs(y_cb(n) - y_cb(-1))
+            length = abs(y_domain_glb%end - y_domain_glb%beg)
             y_cb = y_cb/length
             y_a = y_a/length
             y_b = y_b/length
@@ -130,7 +130,7 @@ impure subroutine s_generate_serial_grid
 
         ! Hyperbolic tangent grid stretching in z-direction
         if (stretch_z) then
-            length = abs(z_cb(p) - z_cb(-1))
+            length = abs(z_domain_glb%end - z_domain_glb%beg)
             z_cb = z_cb/length
             z_a = z_a/length
             z_b = z_b/length

src/pre_process/m_start_up.fpp

@@ -622,6 +622,12 @@ contains
         ! Neither procedure has to be carried out if pre-process is in fact not truly executed in parallel.
         call s_mpi_bcast_user_inputs()
         call s_initialize_parallel_io()
+
+        ! Save the global domain bounds before decomposition overwrites x/y/z_domain with each processor's local sub-domain bounds
+        x_domain_glb = x_domain
+        y_domain_glb = y_domain
+        z_domain_glb = z_domain
+
         call s_mpi_decompose_computational_domain()
 
     end subroutine s_initialize_mpi_domain