Merge master into hypo_hlld (resolve muscl_eps f_is_default)

ChrisZYJ · ChrisZYJ · commit 930659203355 · 2026-05-05T20:20:48.000-07:00
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -149,7 +149,7 @@ jobs:
         continue-on-error: true
 
   github:
-    name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, intel={3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel) }}
+    name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, {3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel && 'intel' || 'GNU') }}
     needs: [lint-gate, file-changes, rebuild-cache]
     if: >-
       !cancelled() &&
diff --git a/docs/documentation/visualization.md b/docs/documentation/visualization.md
@@ -205,7 +205,7 @@ Post-processed data in Silo-HDF5 format (`format=1`) can be opened directly in P
 Paraview 5.11.0 has been confirmed to work with the MFC databases for some parallel environments.
 Nevertheless, the installation and configuration of Paraview can be environment-dependent and are left to the user.
 
-The user can launch Paraview and open the index files under `/silo_hdf5/root`.
+The user can launch Paraview and open the series file `/silo_hdf5/collection.silo.series`.
 Once the database is loaded, flow field variables contained in the database can be added to the render view.
 Further information on Paraview can be found in its [documentation](https://docs.paraview.org/en/latest/).
 The figure below shows the iso-contour of the liquid void fraction (`alpha1`) in the database generated by the example case `3D_sphbubcollapse`.
diff --git a/docs/index.html b/docs/index.html
@@ -58,6 +58,7 @@
             { name: "Mach 0.3 flow over a corgi (2M STL)", image: "res/simulations/u.png", computer: "HiPerGator", computerUrl: "https://www.rc.ufl.edu/about/hipergator/",                                     accelerators: "2 GPUs",    walltime: "80s",  source: "https://www.youtube.com/watch?v=O8dSRqHLp_o" },
             // Shock-droplet
             { name: "Shedding water droplet",              image: "res/simulations/a.png", computer: "Summit",     computerUrl: "https://www.olcf.ornl.gov/summit/",                                            accelerators: "960 V100s", walltime: "4h",   source: "https://www.youtube.com/watch?v=Gjj-qZkXcrg" },
+            { name: "1K resolved particles",               image: "res/simulations/x.jpg", computer: "Phoenix",    computerUrl: "https://www.pace.gatech.edu/",                                                 accelerators: "4 A100s",   walltime: "~1.5h", source: "https://www.youtube.com/watch?v=ufisHG0KkOU" },
             // Biomedical & acoustics
             { name: "Burstwave lithotripsy",               image: "res/simulations/k.png", computer: "Delta",      computerUrl: "https://www.ncsa.illinois.edu/research/project-highlights/delta/",              accelerators: "128 A100s", walltime: "30m",  source: "https://www.youtube.com/watch?v=XWsUTaJXGF8" },
             { name: "Cavitation fragments kidney stone",   image: "res/simulations/d.png", computer: "Summit",     computerUrl: "https://www.olcf.ornl.gov/summit/",                                            accelerators: "576 V100s", walltime: "30m",  source: "https://doi.org/10.48550/arXiv.2305.09163" },
@@ -74,6 +75,7 @@
             { name: "Breakup of vibrated interface",       image: "res/simulations/f.png", computer: "Summit",     computerUrl: "https://www.olcf.ornl.gov/summit/",                                            accelerators: "128 V100s", walltime: "4h",   source: "https://www.youtube.com/watch?v=XQ3g1oSg8mc" },
             { name: "Viscous Taylor-Green vortex",         image: "res/simulations/h.png", computer: "Delta",      computerUrl: "https://www.ncsa.illinois.edu/research/project-highlights/delta/",              accelerators: "128 A100s", walltime: "17h",  source: "https://www.youtube.com/watch?v=7i2h08dlDQw" },
             { name: "Mach 1.5 shock-helium bubble",        image: "res/simulations/t.png", computer: "Phoenix",    computerUrl: "https://www.pace.gatech.edu/",                                                 accelerators: "1 A100",    walltime: "1h",   source: "https://www.youtube.com/watch?v=zDJoe0NYZsQ" },
+            { name: "Sphere collision and wall rebound",   image: "res/simulations/w.jpg", computer: "Phoenix",    computerUrl: "https://www.pace.gatech.edu/",                                                 accelerators: "4 A100s",   walltime: "~2h",   source: "https://www.youtube.com/watch?v=5ugLuDtV0C0" },
         ];
 
         const scalings = [
diff --git a/docs/res/simulations/w.jpg b/docs/res/simulations/w.jpg
diff --git a/docs/res/simulations/x.jpg b/docs/res/simulations/x.jpg
diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90
@@ -211,13 +211,17 @@ impure subroutine s_read_serial_data_files(t_step)
             call s_assign_default_bc_type(bc_type)
         end if
 
-        call s_read_grid_data_direction(t_step_dir, 'x', x_cb, dx, x_cc, m)
+        ! Pass explicit slices so the dummy `dimension(-1:)` / `dimension(0:)` arguments map to the correct interior indices of the
+        ! actual arrays. Without slicing, when offset_x%beg or buff_size > 0 (i.e. format=1 parallel 3D ranks), Fortran's
+        ! assumed-shape re-mapping shifts the read by that many slots and leaves the last interior cells uninitialized - corrupting
+        ! downstream ghost-cell extrapolation.
+        call s_read_grid_data_direction(t_step_dir, 'x', x_cb(-1:m), dx(0:m), x_cc(0:m), m)
 
         if (n > 0) then
-            call s_read_grid_data_direction(t_step_dir, 'y', y_cb, dy, y_cc, n)
+            call s_read_grid_data_direction(t_step_dir, 'y', y_cb(-1:n), dy(0:n), y_cc(0:n), n)
 
             if (p > 0) then
-                call s_read_grid_data_direction(t_step_dir, 'z', z_cb, dz, z_cc, p)
+                call s_read_grid_data_direction(t_step_dir, 'z', z_cb(-1:p), dz(0:p), z_cc(0:p), p)
             end if
         end if
 
diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp
@@ -4,8 +4,7 @@
 
 #:include 'macros.fpp'
 
-!> @brief Shared bubble-dynamics procedures (radial acceleration, wall pressure, sound speed) for ensemble- and volume-averaged
-!! models
+!> @brief Bubble-dynamics procedures for ensemble- and volume-averaged models
 module m_bubbles
 
     use m_derived_types
diff --git a/src/simulation/m_global_parameters.fpp b/src/simulation/m_global_parameters.fpp
@@ -883,7 +883,7 @@ contains
         #:endif
 
         ! muscl_eps: use per-limiter defaults when user did not set it
-        if (muscl_eps == dflt_real) then
+        if (f_is_default(muscl_eps)) then
             if (muscl_lim <= 2) then
                 muscl_eps = 1e-9_wp  ! minmod, MC
             else
diff --git a/src/simulation/m_pressure_relaxation.fpp b/src/simulation/m_pressure_relaxation.fpp
@@ -217,7 +217,7 @@ contains
         type(scalar_field), dimension(sys_size), intent(inout) :: q_cons_vf
         integer, intent(in)                                    :: j, k, l
         #:if not MFC_CASE_OPTIMIZATION and USING_AMD
-            real(wp), dimension(2) :: alpha_rho, alpha
+            real(wp), dimension(3) :: alpha_rho, alpha
         #:else
             real(wp), dimension(num_fluids) :: alpha_rho, alpha
         #:endif
diff --git a/src/simulation/m_rhs.fpp b/src/simulation/m_rhs.fpp
@@ -68,7 +68,7 @@ module m_rhs
     $:GPU_DECLARE(create='[tau_Re_vf]')
 
     !> @name The cell-boundary values of the fluxes (src - source, gsrc - geometrical source). These are computed by applying the
-    !! chosen Riemann problem solver .on the left and right cell-boundary values of the primitive variables
+    !! chosen Riemann problem solver on the left and right cell-boundary values of the primitive variables
     !> @{
     type(vector_field), allocatable, dimension(:) :: flux_n
     type(vector_field), allocatable, dimension(:) :: flux_src_n
diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp
@@ -818,7 +818,7 @@ contains
         end if
 
         ! Write IB kinematic state for restart
-        if (ib) call s_write_ib_state_file(t_step)
+        if (ib) call s_write_ib_state_file(save_count)
 
         call nvtxEndRange
         call cpu_time(finish)
@@ -914,9 +914,13 @@ contains
 
         if (model_eqns == 3) call s_initialize_internal_energy_equations(q_cons_ts(1)%vf)
         if (ib) then
-            if (t_step_start > 0) call s_read_ib_restart_data(t_step_start)
+            if (cfl_dt .and. n_start > 0) then
+                call s_read_ib_restart_data(n_start)
+            else if (t_step_start > 0) then
+                call s_read_ib_restart_data(t_step_start)
+            end if
             call s_ibm_setup()
-            if (t_step_start == 0) then
+            if (t_step_start == 0 .or. (cfl_dt .and. n_start == 0)) then
                 call s_write_ib_data_file(0)
                 call s_write_ib_state_file(0)
             end if
diff --git a/toolchain/bootstrap/modules.sh b/toolchain/bootstrap/modules.sh
@@ -119,6 +119,13 @@ if [ $(echo "$VARIABLES" | grep = | wc -c) -gt 0 ]; then
     export $(eval "echo $VARIABLES") > /dev/null
 fi
 
+UNLOAD_MODULES="$(__extract "$u_c-all-unload") $(__extract "$u_c-$cg-unload")"
+UNLOAD_MODULES=$(echo "$UNLOAD_MODULES" | xargs)
+if [ -n "$UNLOAD_MODULES" ]; then
+    log " $ module unload $UNLOAD_MODULES"
+    module unload $UNLOAD_MODULES
+fi
+
 # Don't check for Cray paths on Carpenter, otherwise do check if they exist
 if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ] &&  [ "$u_c" '!=' 'famd' ] ; then
     ok "Found $M\$CRAY_LD_LIBRARY_PATH$CR. Prepending to $M\$LD_LIBRARY_PATH$CR."
diff --git a/toolchain/bootstrap/python.sh b/toolchain/bootstrap/python.sh
@@ -180,6 +180,14 @@ if ! cmp "$(pwd)/toolchain/pyproject.toml" "$(pwd)/build/pyproject.toml" > /dev/
     if [ "$USE_UV" = "1" ]; then
         # UV_LINK_MODE=copy avoids slow hardlink failures on cross-filesystem installs (common on HPC)
         export UV_LINK_MODE=copy
+        # On GitHub Actions self-hosted runners, the default uv cache (~/.cache/uv)
+        # often lives on a shared NFS $HOME (e.g. OLCF /ccs/home), where uv's
+        # file-lock implementation hits "os error 524" when concurrent runners on
+        # different nodes contend for the same .lock. Redirect to node-local
+        # storage in CI only, so non-CI users keep their normal reusable cache.
+        if [ "${GITHUB_ACTIONS:-}" = "true" ] && [ -w "${TMPDIR:-/tmp}" ]; then
+            export UV_CACHE_DIR="${TMPDIR:-/tmp}/uv-cache-${USER:-$(id -un)}"
+        fi
         log "(venv) Using$MAGENTA uv$COLOR_RESET for fast installation..."
 
         if [ "$verbose" = "1" ]; then
diff --git a/toolchain/mfc/case_validator.py b/toolchain/mfc/case_validator.py
@@ -328,11 +328,15 @@ def check_igr(self):
     def check_weno(self):
         """Checks constraints regarding WENO order"""
         recon_type = self.get("recon_type", 1)
+        self.prohibit(recon_type not in [1, 2], "recon_type must be 1 (WENO) or 2 (MUSCL)")
 
         # WENO_TYPE = 1
         if recon_type != 1:
             return
 
+        for param in ["muscl_order", "muscl_lim"]:
+            self.prohibit(self.is_set(param), f"recon_type = 1 (WENO) is not compatible with {param}")
+
         weno_order = self.get("weno_order")
         m = self.get("m", 0)
         n = self.get("n", 0)
@@ -349,6 +353,8 @@ def check_weno(self):
     def check_muscl(self):
         """Check constraints regarding MUSCL order"""
         recon_type = self.get("recon_type", 1)
+        self.prohibit(recon_type not in [1, 2], "recon_type must be 1 (WENO) or 2 (MUSCL)")
+
         int_comp = self.get("int_comp", "F") == "T"
 
         self.prohibit(int_comp and recon_type != 2, "int_comp (THINC interface compression) requires recon_type = 2 (MUSCL)")
@@ -357,6 +363,17 @@ def check_muscl(self):
         if recon_type != 2:
             return
 
+        weno_log_params = ["mapped_weno", "wenoz", "teno", "mp_weno", "weno_avg", "null_weights", "weno_Re_flux"]
+        for param in weno_log_params:
+            self.prohibit(self.get(param) == "T", f"recon_type = 2 (MUSCL) is not compatible with {param} = T")
+
+        weno_numeric_params = ["wenoz_q", "teno_CT", "weno_eps"]
+        for param in weno_numeric_params:
+            self.prohibit(self.is_set(param), f"recon_type = 2 (MUSCL) is not compatible with {param}")
+
+        weno_order = self.get("weno_order")
+        self.prohibit(weno_order is not None and weno_order != 0, f"recon_type = 2 (MUSCL) requires weno_order unset or 0, but got {weno_order}")
+
         muscl_order = self.get("muscl_order")
         m = self.get("m", 0)
         n = self.get("n", 0)
@@ -725,6 +742,13 @@ def check_finite_difference(self):
 
     def check_weno_simulation(self):
         """Checks WENO-specific constraints for simulation"""
+        recon_type = self.get("recon_type", 1)
+        self.prohibit(recon_type not in [1, 2], "recon_type must be 1 (WENO) or 2 (MUSCL)")
+
+        # WENO_TYPE = 1
+        if recon_type != 1:
+            return
+
         weno_order = self.get("weno_order")
         weno_eps = self.get("weno_eps")
         wenoz = self.get("wenoz", "F") == "T"
@@ -759,6 +783,13 @@ def check_weno_simulation(self):
 
     def check_muscl_simulation(self):
         """Checks MUSCL-specific constraints for simulation"""
+        recon_type = self.get("recon_type", 1)
+        self.prohibit(recon_type not in [1, 2], "recon_type must be 1 (WENO) or 2 (MUSCL)")
+
+        # MUSCL_TYPE = 2
+        if recon_type != 2:
+            return
+
         muscl_order = self.get("muscl_order")
         muscl_lim = self.get("muscl_lim")
         muscl_eps = self.get("muscl_eps")
diff --git a/toolchain/mfc/params/registry.py b/toolchain/mfc/params/registry.py
@@ -233,7 +233,11 @@ def register(self, param: ParamDef) -> None:
         if param.name in self._params:
             existing = self._params[param.name]
             if existing.param_type != param.param_type:
-                raise ValueError(f"Type mismatch for '{param.name}'")
+                raise ValueError(
+                    f"Type mismatch for '{param.name}': "
+                    f"existing type is {existing.param_type!r}, "
+                    f"new type is {param.param_type!r}"
+                )
             existing.tags.update(param.tags)
             for tag in param.tags:
                 self._by_tag[tag].add(param.name)
diff --git a/toolchain/mfc/params_tests/test_registry.py b/toolchain/mfc/params_tests/test_registry.py
@@ -38,7 +38,8 @@ def test_register_type_mismatch_raises(self):
         with self.assertRaises(ValueError) as ctx:
             reg.register(ParamDef(name="test", param_type=ParamType.REAL))
 
-        self.assertIn("Type mismatch", str(ctx.exception))
+        self.assertIn("existing type is", str(ctx.exception))
+        self.assertIn("new type is", str(ctx.exception))
 
     def test_get_params_by_tag(self):
         """get_params_by_tag should return params with that tag."""
diff --git a/toolchain/mfc/test/case.py b/toolchain/mfc/test/case.py
@@ -161,7 +161,9 @@ def __init__(self, trace: str, mods: dict, ppn: int = None, override_tol: float
         self.ppn = ppn or 1
         self.override_tol = override_tol
         self.restart_check = restart_check
-        super().__init__({**BASE_CFG.copy(), **mods})
+        merge = {**BASE_CFG.copy(), **mods}
+        merge = {key: val for key, val in merge.items() if val is not None}
+        super().__init__(merge)
 
     def run(self, targets: List[Union[str, MFCTarget]], gpus: Set[int]) -> subprocess.CompletedProcess:
         if gpus is not None and len(gpus) != 0:
diff --git a/toolchain/mfc/test/cases.py b/toolchain/mfc/test/cases.py
@@ -300,7 +300,7 @@ def alter_igr():
 
     def alter_muscl():
         for muscl_order in [1, 2]:
-            stack.push(f"muscl_order={muscl_order}", {"muscl_order": muscl_order, "recon_type": 2, "weno_order": 0})
+            stack.push(f"muscl_order={muscl_order}", {"muscl_order": muscl_order, "recon_type": 2, "weno_order": 0, "weno_eps": None, "wenoz_q": None, "teno_CT": None})
 
             if muscl_order == 1:
                 for int_comp in ["T", "F"]:
@@ -1784,6 +1784,10 @@ def modify_example_case(case: dict):
                     case["t_step_stop"] = 50
                     case["t_step_save"] = 50
 
+                if case.get("recon_type") == 2:
+                    for k in ("weno_order", "weno_eps", "wenoz_q", "teno_CT"):
+                        case[k] = None
+
                 caseSize = case["m"] * max(case["n"], 1) * max(case["p"], 1)
                 if caseSize > 25 * 25:
                     if case["n"] == 0 and case["p"] == 0:
diff --git a/toolchain/modules b/toolchain/modules
@@ -39,6 +39,7 @@ p-all python/3.12.5
 p-cpu gcc/12.3.0 openmpi/4.1.5
 p-gpu python/3.12.5 nvhpc/24.5 hpcx/2.19-cuda cuda/12.1.1
 p-gpu MFC_CUDA_CC=70,75,80,89,90 NVHPC_CUDA_HOME=$CUDA_HOME CC=nvc CXX=nvc++ FC=nvfortran
+p-gpu-unload xalt
 
 f     OLCF Frontier
 f-all cpe/25.03 rocm/6.3.1
@@ -113,4 +114,4 @@ h-gpu NVCOMPILER_COMM_LIBS_HOME=/apps/compilers/nvhpc/25.9/Linux_x86_64/25.9/com
 
 t     WPI Turing
 t-all slurm
-t-cpu gcc/12.1.0/i6yk33f openmpi/4.1.3/ebae7zc python/3.13.5/6anz4qy
+t-cpu gcc/12.1.0/i6yk33f openmpi/4.1.3/ebae7zc python/3.13.5/6anz4qy
diff --git a/toolchain/templates/include/generate_silo_series.py b/toolchain/templates/include/generate_silo_series.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Generate SILO series file for time-series visualization in ParaView/VisIt."""
+
+import json
+import sys
+from pathlib import Path
+
+
+def generate_silo_series(case_dir):
+    """Generate collection.silo.series file from existing collection files."""
+    root_dir = Path(case_dir) / "silo_hdf5" / "root"
+    series_file = Path(case_dir) / "silo_hdf5" / "collection.silo.series"
+
+    if not root_dir.exists():
+        return
+
+    collection_files = sorted(
+        root_dir.glob("collection_*.silo"),
+        key=lambda p: int(p.stem.split("_")[1]),
+    )
+
+    if not collection_files:
+        return
+
+    files = [{"name": f"root/{f.name}", "time": i} for i, f in enumerate(collection_files)]
+
+    series_data = {"file-series-version": "1.0", "files": files}
+
+    with open(series_file, "w") as f:
+        json.dump(series_data, f, indent=2)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: generate_silo_series.py <case_dir>")
+        sys.exit(1)
+
+    case_dir = sys.argv[1]
+    generate_silo_series(case_dir)
diff --git a/toolchain/templates/include/helpers.mako b/toolchain/templates/include/helpers.mako
@@ -99,6 +99,10 @@ END
         exit 1
     fi
 
+% if target.name == 'post_process':
+    python3 "${MFC_ROOT_DIR}/toolchain/templates/include/generate_silo_series.py" '${os.path.dirname(input)}'
+% endif
+
     % if output_summary:
 
         cd '${MFC_ROOT_DIR}'
