Make common global ownership stage-independent

Author: sbryngelson

src/common/m_global_parameters_common.fpp

@@ -27,20 +27,6 @@ module m_global_parameters_common
     ! num_dims, num_vels, weno_polyn, muscl_polyn, weno_num_stencils, wenojs, igr, etc.
     #:include 'generated_case_opt_decls.fpp'
 
-    ! For pre_process and post_process: num_dims and num_vels are declared manually here
-    ! (sim gets them from generated_case_opt_decls.fpp above)
-#ifndef MFC_SIMULATION
-    integer :: num_dims  !< Number of spatial dimensions
-    integer :: num_vels  !< Number of velocity components (different from num_dims for mhd)
-#endif
-
-    ! For pre_process: weno_polyn and muscl_polyn are declared manually here
-    ! (sim gets them from generated_case_opt_decls.fpp; post does not use them)
-#ifdef MFC_PRE_PROCESS
-    integer :: weno_polyn   !< Degree of the WENO polynomials
-    integer :: muscl_polyn  !< Degree of the MUSCL polynomials
-#endif
-
     !> @name Annotations of the structure of the state and flux vectors in terms of the size and configuration of the system of
     !! equations
     !> @{
@@ -90,49 +76,9 @@ module m_global_parameters_common
     !> Minimum cell widths. These are distinct from the per-cell width arrays named dx, dy, and dz in simulation and post-process.
     real(wp) :: dx_min, dy_min, dz_min
 
-#ifdef MFC_SIMULATION
     $:GPU_DECLARE(create='[sys_size, eqn_idx, b_size, tensor_size]')
     $:GPU_DECLARE(create='[shear_num, shear_indices, shear_BC_flip_num, shear_BC_flip_indices]')
-    ! Device residency for namelist/case-opt state declared above via the generated
-    ! includes: declare directives must live in the declaring module (Cray ftn rejects
-    ! declare-target on use-associated names), so these moved here from simulation.
-    $:GPU_DECLARE(create='[cyl_coord]')
-    $:GPU_DECLARE(create='[dt, m, n, p]')
-    $:GPU_DECLARE(create='[cfl_target]')
-    $:GPU_DECLARE(create='[int_comp, ic_eps, ic_beta]')
-    $:GPU_DECLARE(create='[muscl_eps]')
-    $:GPU_DECLARE(create='[mpp_lim, model_eqns, mixture_err, alt_soundspeed]')
-    $:GPU_DECLARE(create='[avg_state, mp_weno, weno_eps, teno_CT, hypoelasticity]')
-    $:GPU_DECLARE(create='[hyperelasticity, elasticity, low_Mach]')
-    $:GPU_DECLARE(create='[cont_damage, hyper_cleaning]')
-    $:GPU_DECLARE(create='[relax, relax_model, palpha_eps, ptgalpha_eps]')
-    $:GPU_DECLARE(create='[down_sample]')
-    $:GPU_DECLARE(create='[fd_order]')
-    $:GPU_DECLARE(create='[rhoref, pref]')
-    $:GPU_DECLARE(create='[ib, num_ibs]')
-    $:GPU_DECLARE(create='[ib_coefficient_of_friction]')
-    $:GPU_DECLARE(create='[Ca, Web, Re_inv]')
-    $:GPU_DECLARE(create='[bubbles_euler, polytropic, polydisperse]')
-    $:GPU_DECLARE(create='[adv_n, adap_dt, adap_dt_tol, adap_dt_max_iters]')
-    $:GPU_DECLARE(create='[bubble_model, thermal]')
-    $:GPU_DECLARE(create='[poly_sigma]')
-    $:GPU_DECLARE(create='[qbmm, pi_fac]')
-    $:GPU_DECLARE(create='[R0ref]')
-    $:GPU_DECLARE(create='[acoustic_source, num_source]')
-    $:GPU_DECLARE(create='[sigma, surface_tension]')
-    $:GPU_DECLARE(create='[bubbles_lagrange]')
-    $:GPU_DECLARE(create='[Bx0]')
-    $:GPU_DECLARE(create='[tau_star, cont_damage_s, alpha_bar]')
-    $:GPU_DECLARE(create='[hyper_cleaning_speed, hyper_cleaning_tau]')
-    $:GPU_DECLARE(create='[synthetic_turbulence, num_turbulent_sources, synth_U_inf]')
-    #:if not MFC_CASE_OPTIMIZATION
-        $:GPU_DECLARE(create='[num_dims, num_vels, weno_polyn, weno_order]')
-        $:GPU_DECLARE(create='[weno_num_stencils, num_fluids, wenojs]')
-        $:GPU_DECLARE(create='[mapped_weno, wenoz, teno, wenoz_q, mhd, relativity]')
-        $:GPU_DECLARE(create='[igr_iter_solver, igr_order, viscous, igr_pres_lim, igr]')
-        $:GPU_DECLARE(create='[recon_type, muscl_order, muscl_polyn, muscl_lim]')
-    #:endif
-#endif
+    $:GPU_DECLARE(create='[elasticity]')
 
     !> @name Processor coordinates and parallel-IO addressing (identical declaration across all three targets)
     !> @{
@@ -163,10 +109,11 @@ contains
     !!   - qbmm_idx allocations and fills (diverge between pre vs sim/post)
     !!   - sim-only: gam = bub_pp%gam_g, nmomsp/nmomtot, Re_idx allocation, GPU_UPDATE calls
     !!   - post-only: beta_idx increment (bubbles_lagrange path), offset/grid allocations
-    impure subroutine s_initialize_eqn_idx(nmom_in, nb_in)
+    impure subroutine s_initialize_eqn_idx(nmom_in, nb_in, six_eqn_alf_is_advected)
 
         integer, intent(in) :: nmom_in
         integer, intent(in) :: nb_in
+        logical, intent(in) :: six_eqn_alf_is_advected
 
         ! Gamma/Pi_inf Model
 
@@ -252,9 +199,7 @@ contains
             eqn_idx%E = eqn_idx%mom%end + 1
             eqn_idx%adv%beg = eqn_idx%E + 1
             eqn_idx%adv%end = eqn_idx%E + num_fluids
-#ifdef MFC_SIMULATION
-            eqn_idx%alf = eqn_idx%adv%end
-#endif
+            if (six_eqn_alf_is_advected) eqn_idx%alf = eqn_idx%adv%end
             eqn_idx%int_en%beg = eqn_idx%adv%end + 1
             eqn_idx%int_en%end = eqn_idx%adv%end + num_fluids
             sys_size = eqn_idx%int_en%end
@@ -352,7 +297,6 @@ contains
         integer :: ierr  !< Generic flag used to identify and report MPI errors
 #endif
 
-#ifdef MFC_SIMULATION
         ! Under case-optimization, num_dims and num_vels are compile-time parameters; skip assignment.
         #:if not MFC_CASE_OPTIMIZATION
             num_dims = 1 + min(1, n) + min(1, p)
@@ -363,15 +307,6 @@ contains
                 num_vels = num_dims
             end if
         #:endif
-#else
-        num_dims = 1 + min(1, n) + min(1, p)
-
-        if (mhd) then
-            num_vels = 3
-        else
-            num_vels = num_dims
-        end if
-#endif
 
         allocate (proc_coords(1:num_dims))
 

src/post_process/m_global_parameters.fpp

@@ -326,7 +326,7 @@ contains
         if (model_eqns == model_eqns_5eq .and. qbmm) nmom = 6
 
         ! Populate eqn_idx, sys_size, b_size, tensor_size, elasticity, shear_* (shared logic)
-        call s_initialize_eqn_idx(nmom, nb)
+        call s_initialize_eqn_idx(nmom, nb, six_eqn_alf_is_advected=.false.)
 
         ! post-only: 6eq alf is a dummy (no void fraction in 6eq)
         if (model_eqns == model_eqns_6eq) eqn_idx%alf = 1

src/pre_process/m_global_parameters.fpp

@@ -447,7 +447,7 @@ contains
         if (model_eqns == model_eqns_5eq .and. bubbles_euler .and. qbmm .and. nnode == 4) nmom = 6
 
         ! Populate eqn_idx, sys_size, b_size, tensor_size, elasticity, shear_* (shared logic)
-        call s_initialize_eqn_idx(nmom, nb)
+        call s_initialize_eqn_idx(nmom, nb, six_eqn_alf_is_advected=.false.)
 
         ! Per-target (pre_process): qbmm_idx allocations and fills
         if (model_eqns == model_eqns_5eq .and. bubbles_euler) then

src/simulation/m_global_parameters.fpp

@@ -708,7 +708,7 @@ contains
         Re_size_max = 0
 
         ! Populate eqn_idx, sys_size, b_size, tensor_size, elasticity, shear_* (shared logic)
-        call s_initialize_eqn_idx(nmom, nb)
+        call s_initialize_eqn_idx(nmom, nb, six_eqn_alf_is_advected=.true.)
 
         ! sim-only: GPU update for shear state after s_initialize_eqn_idx populated it
         if (model_eqns == model_eqns_5eq .or. model_eqns == model_eqns_6eq) then

toolchain/mfc/params/generators/fortran_gen.py

@@ -28,6 +28,97 @@
 
 _VALID_TARGETS = ("pre", "sim", "post")
 
+# Scalar simulation parameters whose generated declarations own device storage.
+# Case-optimization parameters are emitted by generated_case_opt_decls.fpp; all
+# other names are emitted by generated_decls.fpp.
+SIM_GPU_DECL_VARS = {
+    "Bx0",
+    "Ca",
+    "R0ref",
+    "Re_inv",
+    "Web",
+    "acoustic_source",
+    "adap_dt",
+    "adap_dt_max_iters",
+    "adap_dt_tol",
+    "adv_n",
+    "alpha_bar",
+    "alt_soundspeed",
+    "avg_state",
+    "bubble_model",
+    "bubbles_euler",
+    "bubbles_lagrange",
+    "cfl_target",
+    "cont_damage",
+    "cont_damage_s",
+    "cyl_coord",
+    "down_sample",
+    "dt",
+    "fd_order",
+    "hyper_cleaning",
+    "hyper_cleaning_speed",
+    "hyper_cleaning_tau",
+    "hyperelasticity",
+    "hypoelasticity",
+    "ib",
+    "ib_coefficient_of_friction",
+    "ic_beta",
+    "ic_eps",
+    "igr",
+    "igr_iter_solver",
+    "igr_order",
+    "igr_pres_lim",
+    "int_comp",
+    "low_Mach",
+    "m",
+    "mapped_weno",
+    "mixture_err",
+    "model_eqns",
+    "mp_weno",
+    "mpp_lim",
+    "muscl_eps",
+    "muscl_lim",
+    "muscl_order",
+    "muscl_polyn",
+    "n",
+    "num_dims",
+    "num_fluids",
+    "num_ibs",
+    "num_source",
+    "num_turbulent_sources",
+    "num_vels",
+    "p",
+    "palpha_eps",
+    "pi_fac",
+    "poly_sigma",
+    "polydisperse",
+    "polytropic",
+    "pref",
+    "ptgalpha_eps",
+    "qbmm",
+    "recon_type",
+    "relativity",
+    "relax",
+    "relax_model",
+    "rhoref",
+    "sigma",
+    "surface_tension",
+    "synth_U_inf",
+    "synthetic_turbulence",
+    "tau_star",
+    "teno",
+    "teno_CT",
+    "thermal",
+    "viscous",
+    "weno_eps",
+    "weno_num_stencils",
+    "weno_order",
+    "weno_polyn",
+    "wenojs",
+    "wenoz",
+    "wenoz_q",
+}
+
 
 def _check_target(target: str) -> None:
     if target not in _VALID_TARGETS:
@@ -117,6 +208,7 @@ def generate_decls_fpp(target: str) -> str:
     """Return Fortran declarations (scalars + known arrays) for a target."""
     _check_target(target)
     lines = [_HEADER.rstrip()]
+    declared_names = set()
     for name in _decl_vars_for_target(target):
         if not _is_simple_scalar(name):
             continue
@@ -132,13 +224,15 @@ def generate_decls_fpp(target: str) -> str:
             ftype = fortran_type_decl(member)
             dim = FORTRAN_ARRAY_DIMS[name]
             lines.append(f"{(ftype + ', dimension(' + dim + ')').ljust(_ARRAY_DECL_COL)}:: {name}")
+            declared_names.add(name)
             continue
         param = REGISTRY.all_params.get(name)
         if param is None:
             continue
         if any(k.startswith(f"{name}(") for k in REGISTRY.all_params):
             raise ValueError(f"{name!r} has indexed variants (e.g. {name}(1)) but is missing from " "FORTRAN_ARRAY_DIMS. Add it there with its Fortran dimension expression.")
         lines.append(f"{fortran_type_decl(param).ljust(_DECL_COL)}:: {name}")
+        declared_names.add(name)
     for name, (ftype, dim, gpu, desc) in TYPED_DECLS.items():
         if name not in NAMELIST_VARS or target not in NAMELIST_VARS[name]:
             continue
@@ -148,8 +242,12 @@ def generate_decls_fpp(target: str) -> str:
             padded += " "
         doc = f" !< {desc}" if desc else ""
         lines.append(f"{padded}:: {name}{doc}")
+        declared_names.add(name)
         if gpu and target == "sim":
             lines.append(f"$:GPU_DECLARE(create='[{name}]')")
+    if target == "sim":
+        for name in sorted(SIM_GPU_DECL_VARS & declared_names):
+            lines.append(f"$:GPU_DECLARE(create='[{name}]')")
     return "\n".join(lines) + "\n"
 
 
@@ -179,6 +277,7 @@ def generate_constants_fpp() -> str:
     ("weno_num_stencils", "integer", "Number of stencils for WENO reconstruction"),
     ("wenojs", "logical", "WENO-JS (default)"),
 ]
+COMMON_CASE_OPT_EXTRA_NAMES = {"num_dims", "num_vels", "weno_polyn", "muscl_polyn"}
 
 _CASE_OPT_DECL_COL = 24  # '::' alignment for case-opt declarations
 
@@ -227,6 +326,10 @@ def generate_case_opt_decls_fpp() -> str:
             if_lines.append(f"    {ftype}, parameter :: {name} = ${{{name}}}$  !< {desc}")
             else_lines.append(f"    {ftype.ljust(_CASE_OPT_DECL_COL)}:: {name}")
 
+    declared_names = {name for name, _, _ in CASE_OPT_EXTRA_LINES} | set(params_to_emit)
+    for name in sorted(SIM_GPU_DECL_VARS & declared_names):
+        else_lines.append(f"    $:GPU_DECLARE(create='[{name}]')")
+
     parts = [_HEADER.rstrip(), "#:if MFC_CASE_OPTIMIZATION"]
     parts.extend(if_lines)
     parts.append("#:else")
@@ -235,6 +338,15 @@ def generate_case_opt_decls_fpp() -> str:
     return "\n".join(parts) + "\n"
 
 
+def generate_common_extra_decls_fpp() -> str:
+    """Return computed-scalar declarations needed by common pre/post code."""
+    lines = [_HEADER.rstrip()]
+    for name, ftype, _ in CASE_OPT_EXTRA_LINES:
+        if name in COMMON_CASE_OPT_EXTRA_NAMES:
+            lines.append(f"{ftype.ljust(_CASE_OPT_DECL_COL)}:: {name}")
+    return "\n".join(lines) + "\n"
+
+
 # Struct roots in NAMELIST_VARS whose member-level broadcasts are irregular
 # (per-target member subsets, grouped array members, nested loops, etc.).
 # These are kept in the manual residue of m_mpi_proxy.fpp.
@@ -572,9 +684,8 @@ def get_generated_files(build_dir: Path) -> List[Tuple[Path, str]]:
 
     Paths match the cmake include directory structure:
       build_dir/include/{full_target}/generated_{namelist,decls,constants,case_opt_decls,bcast}.fpp
-    Every target gets generated_case_opt_decls.fpp: real content for simulation,
-    a header-only stub for the others (Fypp resolves #:include at parse time, so
-    the file must exist for every target even inside a dead conditional).
+    Every target gets generated_case_opt_decls.fpp: the full case-optimization
+    block for simulation and common computed-scalar declarations for pre/post.
     Every target gets generated_bcast.fpp with its MPI broadcast statements.
     """
     result = []
@@ -585,7 +696,7 @@ def get_generated_files(build_dir: Path) -> List[Tuple[Path, str]]:
         result.append((inc / "generated_constants.fpp", generate_constants_fpp()))
     for short, full in TARGETS:
         inc = build_dir / "include" / full
-        content = generate_case_opt_decls_fpp() if short == "sim" else _HEADER + "! (no case-optimization declarations for this target)\n"
+        content = generate_case_opt_decls_fpp() if short == "sim" else generate_common_extra_decls_fpp()
         result.append((inc / "generated_case_opt_decls.fpp", content))
     for short, full in TARGETS:
         inc = build_dir / "include" / full

toolchain/mfc/params_tests/test_fortran_gen.py

@@ -323,6 +323,25 @@ def test_generate_case_opt_decls_fpp():
     assert "AUTO-GENERATED" in out
 
 
+def test_common_computed_scalars_are_declared_for_every_target():
+    from pathlib import Path
+
+    from mfc.params.generators.fortran_gen import get_generated_files
+
+    files = {path.parent.name: content for path, content in get_generated_files(Path("/build")) if path.name == "generated_case_opt_decls.fpp"}
+    for target in ("pre_process", "simulation", "post_process"):
+        for name in ("num_dims", "num_vels", "weno_polyn", "muscl_polyn"):
+            assert f":: {name}" in files[target]
+
+
+def test_simulation_generated_scalars_own_gpu_declarations():
+    from mfc.params.generators.fortran_gen import SIM_GPU_DECL_VARS, generate_case_opt_decls_fpp, generate_decls_fpp
+
+    generated = generate_decls_fpp("sim") + generate_case_opt_decls_fpp()
+    for name in SIM_GPU_DECL_VARS:
+        assert generated.count(f"$:GPU_DECLARE(create='[{name}]')") == 1
+
+
 # ── generate_bcast_fpp tests ──────────────────────────────────────────────────