merge: AMR patch-slot infrastructure (SP12a T1) into PR branch

Author: sbryngelson

docs/documentation/case.md

@@ -683,6 +683,7 @@ To restart the simulation from $k$-th time step, see @ref running "Restarting Ca
 | `amr_tag_eps`           | Real    | Relative density-gradient threshold for AMR refinement tagging (default 0.1) |
 | `amr_buf`               | Integer | Coarse-cell padding around tagged cells when regridding (default 3) |
 | `amr_subcycle`          | Logical | Advance the coarse level at the case dt and the fine level at dt/2 (two substeps; Berger-Colella refluxing). Requires `amr`; incompatible with `cfl_dt`. |
+| `amr_max_patches`       | Integer | Number of fixed refined-patch slots preallocated (each max-patch sized; ~N x device memory); must be >= 1 (default 4) |
 | `hybrid_weno`           | Logical | Use linear-optimal reconstruction in smooth cells, full WENO only at flagged discontinuities (requires WENO reconstruction) |
 | `hybrid_weno_eps`       | Real    | Smoothness threshold for hybrid WENO shock flagging; must be > 0 (default 1e-2) |
 | `hybrid_riemann`        | Logical | Use a cheap central/Rusanov flux in smooth cells, full HLLC only at flagged discontinuities (requires HLLC, 5eq/6eq) |
@@ -823,6 +824,14 @@ Accumulated fine-level fluxes are applied back to the coarse level (reflux corre
 after each coarse step.
 `amr_subcycle` is incompatible with `cfl_dt` (variable time step) and requires `amr = T`.
 
+**Patch slots.**
+`amr_max_patches` (default 4) sets the number of fixed refined-patch slots preallocated
+for the run. Each slot is sized to the maximum patch extent, so `N` slots require roughly
+`N` times the device memory of a single patch; the goal is the compute win of refining
+separated features independently, and memory efficiency (compact per-patch pools) is a
+follow-up. The current release populates a single slot; multi-patch clustering is
+forthcoming.
+
 **Restart.**
 Each save step writes a fine-level AMR restart file alongside the level-0 restart data
 (whose format is unchanged): the current — possibly regridded — patch box and the fine
@@ -847,6 +856,7 @@ visualization output is future work.
 | `amr_tag_eps`           | Real    | Normalized density-gradient threshold for refinement tagging; must be > 0 when `amr_regrid_int > 0` (default 0.1) |
 | `amr_buf`               | Integer | Coarse-cell padding around tagged cells; must be >= 1 when `amr_regrid_int > 0` (default 3) |
 | `amr_subcycle`          | Logical | Advance fine level at dt/2 (two substeps per coarse step) with Berger–Colella refluxing |
+| `amr_max_patches`       | Integer | Number of fixed refined-patch slots preallocated (each max-patch sized; ~N x device memory); must be >= 1 (default 4) |
 
 ### 8. Acoustic Source {#sec-acoustic-source}
 

src/simulation/m_amr.fpp

@@ -289,6 +289,11 @@ module m_global_parameters
     !! level covers exactly this intersection: local fine index 0 = patch-global fine index 2*(amr_isect_lo - amr_region_lo).
     integer :: amr_isect_lo(3) = 0, amr_isect_hi(3) = 0
 
+    !> Number of currently-active AMR fine-patch slots (T1: forced to 1) and the working slot index selecting which slot the
+    !! per-patch machinery (advance/reflux/restrict/regrid/IO) operates on. Read by m_amr and m_amr_registers (mirrors, no
+    !! use-cycle).
+    integer :: amr_num_patches = 1, amr_cur = 1
+
 contains
 
     !> Assigns default values to the user inputs before reading them in. This enables for an easier consistency check of these
@@ -462,6 +467,7 @@ contains
         amr_tag_eps = 0.1_wp
         amr_buf = 3
         amr_subcycle = .false.
+        amr_max_patches = 4
         hybrid_smooth_flux = 2
         partition_tile_size = 8
         many_ib_patch_parallelism = .false.

src/simulation/m_amr_registers.fpp

@@ -499,12 +499,19 @@ contains
             end if
 
             ! AMR fine-level stage advance (interleaved, non-subcycled): q_cons_ts(1)%vf still holds
-            ! the coarse stage-entry state here (the stage-1 backup and RK update below have not run yet)
-            if (amr .and. .not. amr_subcycle) call s_advance_amr_fine_stage(s, rk_coef(s,:), q_cons_ts(1)%vf, bc_type, q_T_sf, &
-                & pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
-            ! freg slices of rank-boundary patch faces move to the outside rank (ALL ranks call; no-op at np=1)
-            if (amr .and. .not. amr_subcycle) call s_mpi_sendrecv_amr_reflux_faces()
-            if (amr .and. .not. amr_subcycle) call s_amr_apply_reflux(rhs_vf)  ! coarse update sees the fine flux at c/f faces
+            ! the coarse stage-entry state here (the stage-1 backup and RK update below have not run yet).
+            ! Each active patch slot is advanced + refluxed in turn (T1: one slot); amr_cur is reset to 1
+            ! afterwards so the next stage's coarse RHS captures creg into slot 1.
+            if (amr .and. .not. amr_subcycle) then
+                do amr_cur = 1, amr_num_patches
+                    call s_advance_amr_fine_stage(s, rk_coef(s,:), q_cons_ts(1)%vf, bc_type, q_T_sf, pb_ts(1)%sf, rhs_pb, &
+                                                  & mv_ts(1)%sf, rhs_mv, t_step, time_avg)
+                    ! freg slices of rank-boundary patch faces move to the outside rank (ALL ranks call; no-op at np=1)
+                    call s_mpi_sendrecv_amr_reflux_faces()
+                    call s_amr_apply_reflux(rhs_vf)  ! coarse update sees the fine flux at c/f faces
+                end do
+                amr_cur = 1
+            end if
 
             if (bubbles_lagrange .and. .not. adap_dt) call s_update_lagrange_tdv_rk(stage=s)
             if (ab_active) then
@@ -591,15 +598,19 @@ contains
         ! Berger-Colella state reflux on the first coarse cells outside the patch
         if (amr) then
             ! ghost lerp sources, restriction target, and state-reflux target are all device-resident:
-            ! the substep/restriction/reflux machinery runs as device kernels (M2)
-            if (amr_subcycle) then
-                call s_advance_amr_fine_substeps(q_cons_ts(stor)%vf, q_cons_ts(1)%vf, rk_coef, bc_type, q_T_sf, pb_ts(1)%sf, &
-                                                 & rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
-            end if
-            call s_restrict_fine_to_coarse(q_cons_ts(1)%vf)
-            ! freg slices of rank-boundary patch faces move to the outside rank (ALL ranks call; no-op at np=1)
-            if (amr_subcycle) call s_mpi_sendrecv_amr_reflux_faces()
-            if (amr_subcycle) call s_amr_apply_reflux_state(q_cons_ts(1)%vf)
+            ! the substep/restriction/reflux machinery runs as device kernels (M2). Each active patch slot
+            ! (T1: one) is subcycled, restricted, and state-refluxed in turn; amr_cur resets to 1 afterwards.
+            do amr_cur = 1, amr_num_patches
+                if (amr_subcycle) then
+                    call s_advance_amr_fine_substeps(q_cons_ts(stor)%vf, q_cons_ts(1)%vf, rk_coef, bc_type, q_T_sf, pb_ts(1)%sf, &
+                                                     & rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
+                end if
+                call s_restrict_fine_to_coarse(q_cons_ts(1)%vf)
+                ! freg slices of rank-boundary patch faces move to the outside rank (ALL ranks call; no-op at np=1)
+                if (amr_subcycle) call s_mpi_sendrecv_amr_reflux_faces()
+                if (amr_subcycle) call s_amr_apply_reflux_state(q_cons_ts(1)%vf)
+            end do
+            amr_cur = 1
         end if
 
 #ifdef MFC_DEBUG

src/simulation/m_global_parameters.fpp

@@ -1370,7 +1370,9 @@ def check_amr(self):
         acoustic_source = self.get("acoustic_source", "F") == "T"
         amr_tag_eps = self.get("amr_tag_eps")
         amr_buf = self.get("amr_buf")
+        amr_max_patches = self.get("amr_max_patches")
 
+        self.prohibit(amr_max_patches is not None and amr_max_patches < 1, "amr_max_patches must be >= 1")
         self.prohibit(recon_type is not None and recon_type != 1, "amr requires WENO reconstruction (recon_type = 1)")
         self.prohibit(time_stepper is not None and time_stepper != 3, "amr requires time_stepper = 3 (SSP-RK3)")
         self.prohibit(model_eqns is not None and model_eqns != 2, "amr requires model_eqns = 2 (5-equation)")

src/simulation/m_time_steppers.fpp

@@ -678,6 +678,7 @@ def _load():
     _r("amr_tag_eps", REAL)
     _r("amr_buf", INT)
     _r("amr_subcycle", LOG)
+    _r("amr_max_patches", INT)
     _r("hybrid_weno_eps", REAL, {"output"})
     _r("hybrid_smooth_flux", INT, {"output"})
     _r("partition_tile_size", INT, {"output"})
@@ -1363,6 +1364,7 @@ def _nv(targets: set, *names: str) -> None:
     "amr_tag_eps",
     "amr_buf",
     "amr_subcycle",
+    "amr_max_patches",
     "alf_factor",
     "num_igr_iters",
     "num_igr_warm_start_iters",

toolchain/mfc/case_validator.py

@@ -123,6 +123,7 @@
     "amr_tag_eps": "Relative density-gradient threshold for AMR refinement tagging",
     "amr_buf": "Coarse-cell padding around tagged cells when regridding",
     "amr_subcycle": "Advance the coarse level at the case dt and the fine level at dt/2 (two substeps; Berger-Colella refluxing)",
+    "amr_max_patches": "Number of fixed refined-patch slots preallocated for multi-patch AMR (each sized max-patch; N slots ~ N x device memory)",
     "hybrid_weno": "Use linear-optimal reconstruction in smooth cells, full WENO only at flagged discontinuities",
     "hybrid_weno_eps": "Smoothness threshold for hybrid WENO shock flagging (must be > 0)",
     "hybrid_riemann": "Use a cheap central/Rusanov flux in smooth cells, full HLLC only at flagged discontinuities (requires HLLC, 5eq/6eq)",