Merge branch 'master' into fix/inactive-patch-validation

Author: SVS87

.github/scripts/prebuild-case-optimization.sh

@@ -22,12 +22,44 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
+# Optional sharding (format "i/N", e.g. "1/2"), set by submit-slurm-job.sh's
+# [shard] argument via $job_shard: shard i builds every Nth case of the sorted
+# case list. Unset = build all cases in one job (default; other clusters).
+shard="${job_shard:-}"
+if [ -n "$shard" ]; then
+    # Validate full shape: must be exactly "digits/digits" — one slash with
+    # non-empty, purely numeric, non-leading-zero parts on both sides.
+    # Split first, then validate each part independently so that inputs like
+    # "1/" "/2" "//" "1/2/3" "a/b" "12" are all caught before any arithmetic.
+    shard_idx="${shard%%/*}"
+    shard_count="${shard##*/}"
+    # Reject if no slash (idx and count are equal and equal to the whole string)
+    case "$shard_idx" in
+        ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;;
+    esac
+    case "$shard_count" in
+        ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;;
+    esac
+    # Confirm the string is exactly "idx/count" — catches "12" (no slash) and
+    # "1/2/3" (extra slash, where idx=1 and count=2/3 would have failed above,
+    # but this is an extra safety net).
+    if [ "$shard" != "$shard_idx/$shard_count" ]; then
+        echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1
+    fi
+    if [ "$shard_idx" -lt 1 ] || [ "$shard_idx" -gt "$shard_count" ]; then
+        echo "ERROR: bad shard '$shard' (expected i/N with 1 <= i <= N)"; exit 1
+    fi
+fi
+
 # Phoenix starts fresh (no prior dep build); other clusters pre-build deps via
 # build.sh first, so we must preserve them and only clean MFC target staging.
+# Sharded jobs share one workspace and run concurrently, so the workflow
+# cleans once before submitting them — cleaning here would wipe a sibling
+# shard's in-progress build.
 if [ "$cluster" = "phoenix" ]; then
     source .github/scripts/clean-build.sh
     clean_build
-else
+elif [ -z "$shard" ]; then
     find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
     find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
 fi
@@ -40,7 +72,49 @@ case "$job_interface" in
     *)   echo "ERROR: prebuild requires gpu interface (acc or omp)"; exit 1 ;;
 esac
 
+# Case-optimized simulation builds land in per-case hash-named staging dirs,
+# but syscheck/pre_process/post_process hash identically across these cases.
+# Concurrent shards must not build those shared staging dirs simultaneously:
+# shard 1 builds them first and drops a done marker; other shards wait for it,
+# after which their builds no-op in the shared dirs.
+if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then
+    shared_marker_done="build/.prebuild-shared-targets-done"
+    shared_marker_failed="build/.prebuild-shared-targets-failed"
+    set -- benchmarks/*/case.py
+    first_case="$1"
+    if [ "$shard_idx" -eq 1 ]; then
+        # Remove both markers at the start so reruns and manual invocations
+        # never observe stale state from a prior run.
+        rm -f "$shared_marker_done" "$shared_marker_failed"
+        echo "=== Shard 1/$shard_count: building shared targets ==="
+        # Write the failure marker if the build exits non-zero so other shards
+        # can detect the failure immediately instead of waiting 90 minutes.
+        trap 'touch "$shared_marker_failed"' ERR
+        ./mfc.sh build -i "$first_case" -t syscheck pre_process post_process --case-optimization $gpu_opts -j 8
+        trap - ERR
+        touch "$shared_marker_done"
+    else
+        echo "=== Shard $shard_idx/$shard_count: waiting for shard 1 to build shared targets ==="
+        waited=0
+        until [ -f "$shared_marker_done" ]; do
+            if [ -f "$shared_marker_failed" ]; then
+                echo "ERROR: shard 1 failed to build shared targets; see shard 1 log"; exit 1
+            fi
+            if [ "$waited" -ge 5400 ]; then
+                echo "ERROR: timed out waiting for $shared_marker_done"; exit 1
+            fi
+            sleep 30
+            waited=$((waited + 30))
+        done
+    fi
+fi
+
+idx=0
 for case in benchmarks/*/case.py; do
+    idx=$((idx + 1))
+    if [ -n "$shard" ] && [ $(((idx - 1) % shard_count)) -ne $((shard_idx - 1)) ]; then
+        continue
+    fi
     echo "=== Pre-building: $case ==="
     ./mfc.sh run "$case" --case-optimization $gpu_opts -j 8 --dry-run
 done

.github/scripts/run_parallel_benchmarks.sh

@@ -62,6 +62,12 @@ bash "${SCRIPT_DIR}/run_monitored_slurm_job.sh" "$pr_job_id" "pr/${job_slug}.out
 if [ "$pr_exit" -ne 0 ]; then
     echo "PR job exited with code: $pr_exit"
     tail -n 50 "pr/${job_slug}.out" 2>/dev/null || echo "  Could not read PR log"
+    # The PR benchmark run genuinely failed (cases crashed/hung/SIGTERM'd, not a
+    # monitor false-positive -- run_monitored_slurm_job.sh re-checks sacct). Fail
+    # the job instead of falling through to the YAML-exists check, which would let
+    # a broken PR pass green as long as a partial YAML was written. Scoped to PR
+    # only: a master/baseline infra flake stays a warning and does not red-cross.
+    exit 1
 else
     echo "PR job completed successfully"
 fi

.github/workflows/test.yml

@@ -402,6 +402,13 @@ jobs:
             cluster_name: 'Oak Ridge | Frontier (AMD)'
             device: 'cpu'
             interface: 'none'
+            shard: '1/2'
+          - runner:       'frontier'
+            cluster:      'frontier_amd'
+            cluster_name: 'Oak Ridge | Frontier (AMD)'
+            device: 'cpu'
+            interface: 'none'
+            shard: '2/2'
     runs-on:
       group:  phoenix
       labels: ${{ matrix.runner }}
@@ -420,7 +427,7 @@ jobs:
 
       - name: Fetch Dependencies
         if:   matrix.cluster != 'phoenix'
-        timeout-minutes: 60
+        timeout-minutes: 120
         run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Build
@@ -523,7 +530,22 @@ jobs:
 
       - name: Pre-Build (SLURM)
         if:   matrix.cluster == 'frontier_amd'
-        run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }}
+        # AMD flang is slow enough that one serial pre-build job exceeds its
+        # walltime, so split the case list across two concurrent SLURM jobs.
+        # The shards share this workspace and skip their in-job staging clean,
+        # so clean once here on the login node before submitting.
+        run:  |
+          find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+          find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+          rm -f build/.prebuild-shared-targets-done
+          bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} 1/2 &
+          pid1=$!
+          bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} 2/2 &
+          pid2=$!
+          rc=0
+          wait "$pid1" || rc=1
+          wait "$pid2" || rc=1
+          exit $rc
 
       - name: Build & Run Case-Optimization Tests
         if:   matrix.cluster != 'phoenix' && matrix.cluster != 'frontier_amd'
@@ -546,6 +568,8 @@ jobs:
         if:   always()
         run:  |
           for f in prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out \
+                   prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}-1-of-2.out \
+                   prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}-2-of-2.out \
                    run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out; do
             [ -f "$f" ] && echo "=== $f ===" && cat "$f"
           done
@@ -556,5 +580,5 @@ jobs:
         with:
           name: case-opt-${{ strategy.job-index }}-${{ matrix.cluster }}-${{ matrix.interface }}
           path: |
-            prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out
+            prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}*.out
             run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out

benchmarks/igr/case.py

@@ -102,10 +102,11 @@
             "patch_icpp(1)%length_x": 2 * math.pi * L,
             "patch_icpp(1)%length_y": 2 * math.pi * L,
             "patch_icpp(1)%length_z": 2 * math.pi * L,
-            "patch_icpp(1)%vel(1)": f"{V0}*sin(x/{L})*cos(y/{L})*sin(z/{L})",
-            "patch_icpp(1)%vel(2)": f"-{V0}*cos(x/{L})*sin(y/{L})*sin(z/{L})",
+            "patch_icpp(1)%vel(1)": 0.0,
+            "patch_icpp(1)%vel(2)": 0.0,
             "patch_icpp(1)%vel(3)": 0,
-            "patch_icpp(1)%pres": f"{P0} + ({rho0}*{V0}**2/16)*(cos(2*x/{L}) + cos(2*y/{L}))*(cos(2*z/{L}) + 2)",
+            "patch_icpp(1)%pres": 0.0,
+            "patch_icpp(1)%hcid": 380,
             "patch_icpp(1)%alpha_rho(1)": 1,
             "patch_icpp(1)%alpha(1)": 1,
             # Fluids Physical Parameters

docs/documentation/case.md

@@ -320,7 +320,8 @@ This is enabled by adding ``'elliptic_smoothing': "T",`` and ``'elliptic_smoothi
 | `num_ibs`            | Integer | Number of immersed boundary patches |
 | `num_stl_models`     | Integer | Number of STL/OBJ model entries in the `stl_models` array |
 | `num_particle_clouds` | Integer | Number of particle bed specifications to generate immersed boundary patches from |
-| `ib_neighborhood_radius` | Integer | Parameter that controls the neighborhood size for IB detection. |
+| `ib_neighborhood_radius`    | Integer | Parameter that controls the neighborhood size for IB detection. |
+| `many_ib_patch_parallelism` | Logical | Parallelize over IB patches instead of grid cells (better for many small patches). |
 | `geometry`           | Integer | Geometry configuration of the patch.|
 | `x[y,z]_centroid`    | Real    | Centroid of the applied geometry in the [x,y,z]-direction. |
 | `length_x[y,z]`      | Real    | Length, if applicable, in the [x,y,z]-direction. |
@@ -368,7 +369,7 @@ Additional details on this specification can be found in [NACA airfoil](https://
 
 - For STL/OBJ geometry (geometry 5 or 12), set `model_id` to index into the `stl_models` array and specify `model_filepath`, `model_scale`, `model_translate`, and `model_threshold` on that entry.
 
-- `moving_ibm` sets the method by which movement will be applied to the immersed boundary. Using 0 will result in no movement. Using 1 will result 1-way coupling where the boundary moves at a constant rate and applied forces to the fluid based upon it's own motion. In 1-way coupling, the fluid does not apply forces back onto the IB. Using 2 will result in 2-way coupling, where the boundary pushes on the fluid and the fluid pushes back on the boundary via pressure and viscous forces. If external forces are applied, the boundary will also experience those forces.
+- `moving_ibm` sets the method by which movement will be applied to the immersed boundary. Using 0 will result in no movement. Using 1 will result 1-way coupling where the boundary moves at a constant rate and applied forces to the fluid based upon its own motion. In 1-way coupling, the fluid does not apply forces back onto the IB. Using 2 will result in 2-way coupling, where the boundary pushes on the fluid and the fluid pushes back on the boundary via pressure and viscous forces. If external forces are applied, the boundary will also experience those forces.
 
 - `vel(i)` is the initial linear velocity of the IB in the x, y, z direction for i=1, 2, 3. When `moving_ibm` equals 2, this velocity is just the starting speed of the object, which will then accelerate due to external forces. If `moving_ibm` equals 1, then this is constant if it is a number, or can be described analytically with an expression.
 
@@ -378,15 +379,35 @@ Additional details on this specification can be found in [NACA airfoil](https://
   Available variables: `x` (`x_cc(i)`), `y` (`y_cc(j)`), `z` (`z_cc(k)`), `t` (current simulation time), and `r` (the IB patch radius).
   The same intrinsic functions and `pi` constant apply; bare `e` is not available.
 
-- `coefficient_of_restitution` is a number from 0 (exclusive) to 1 (inclusive) describing how elastic IB collisions are. 0 is for perfectly inellastic collisions while 1 is for perfectly ellastic collisions.
+- `coefficient_of_restitution` is a number from 0 (exclusive) to 1 (inclusive) describing how elastic IB collisions are. 0 is for perfectly inelastic collisions while 1 is for perfectly elastic collisions.
 
-- `collision_model` is an integer to select the collision model being used for IB collisions. Using 0 disables collisions and collisiono checking. 1 enables the soft-sphere collision model, where all IBs must be circles or sphere and those IBs can collide with each other as well as walls.
+- `collision_model` is an integer to select the collision model being used for IB collisions. Using 0 disables collisions and collision checking. 1 enables the soft-sphere collision model, where all IBs must be circles or sphere and those IBs can collide with each other as well as walls.
 
-- `collision_time` is approximately the amount of simulation time used to resolve collisions. This is handled by modifying the spring gonstant used to apply collision forces.
+- `collision_time` is approximately the amount of simulation time used to resolve collisions. This is handled by modifying the spring constant used to apply collision forces.
 
 - `ib_coefficient_of_friction` is the coefficient of friction used in IB collisions.
 
-- `ib_neighborhood_radius` controls the size of the neighborhood size. This value defaults to 1, which indicates that any given rank is aware of IB's up to 1 ranks away. This parameter is required to strong-scale a case when IB's eventually grow to be larger than one full processor domain wide.
+- `ib_neighborhood_radius` controls the size of the neighborhood size. This value defaults to 1, which indicates that any given rank is aware of IBs up to 1 ranks away. This parameter is required to strong-scale a case when IBs eventually grow to be larger than one full processor domain wide.
+
+#### Particle Clouds
+
+A particle cloud is a compact specification of a bed of identical circular (2D) or spherical (3D) immersed boundaries; each cloud is expanded into individual `patch_ib` particles at startup. Set `num_particle_clouds` to the number of beds and prepend the parameters below with `particle_cloud(j)%` where $j$ is the cloud index.
+
+| Parameter         | Type    | Description |
+| ---:              | :----:  | :---        |
+| `x[y,z]_centroid` | Real    | Centre of the cloud region in the [x,y,z]-direction. |
+| `length_x[y,z]`   | Real    | Extent of the cloud region in the [x,y,z]-direction. |
+| `num_particles`   | Integer | Number of particles to place in the region. |
+| `radius`          | Real    | Radius of every particle in the cloud. |
+| `mass`            | Real    | Mass of every particle in the cloud. |
+| `min_spacing`     | Real    | Minimum surface-to-surface gap between particles (centres are `2*radius + min_spacing` apart). |
+| `moving_ibm`      | Integer | Motion flag applied to every particle (see `patch_ib(j)%%moving_ibm`). |
+| `seed`            | Integer | Random seed for reproducible placement (used by `packing_method = 1`). |
+| `packing_method`  | Integer | Algorithm used to place the particles. |
+
+- `packing_method` selects how the `num_particles` are positioned within the cloud region:
+  - `1` (rejection sampling) draws random positions and rejects any that violate `min_spacing`, producing a disordered bed. `seed` makes the placement reproducible.
+  - `2` (lattice) places the particles on the optimally dense lattice for the geometry — a triangular lattice in 2D and a face-centered cubic lattice in 3D. The lattice spacing is derived from the particle density (`num_particles` over the region area/volume); if that spacing is below the required `2*radius + min_spacing`, the region is too dense and the run aborts.
 
 ### 5. Fluid Material's {#sec-fluid-materials}
 
@@ -1248,7 +1269,7 @@ Boundary is at polar angle \f$\theta = \mathrm{atan2}(y - y_{\mathrm{centroid}},
 | 3    | 2D Rectangle       | 2      |
 | 4    | 2D Airfoil         | 2      |
 | 8    | 3D Sphere          | 3      |
-| 10   | 3D Cylinder        | 3      |
+| 10   | 3D Cylinder        | 3      | `length_x` sets the axial length of the cylinder. |
 | 11   | 3D Airfoil         | 3      |
 
 ### Acoustic Supports {#acoustic-supports}

docs/index.html

@@ -63,6 +63,7 @@
             { name: "Burstwave lithotripsy",               image: "res/simulations/k.png", computer: "Delta",      computerUrl: "https://www.ncsa.illinois.edu/research/project-highlights/delta/",              accelerators: "128 A100s", walltime: "30m",  source: "https://www.youtube.com/watch?v=XWsUTaJXGF8" },
             { name: "Cavitation fragments kidney stone",   image: "res/simulations/d.png", computer: "Summit",     computerUrl: "https://www.olcf.ornl.gov/summit/",                                            accelerators: "576 V100s", walltime: "30m",  source: "https://doi.org/10.48550/arXiv.2305.09163" },
             { name: "Focused ultrasound on kidney stone",  image: "res/simulations/v.png", computer: "Bridges2",   computerUrl: "https://www.psc.edu/resources/bridges-2/",                                     accelerators: "10 V100s",  walltime: "2h",   source: "https://www.youtube.com/watch?v=z8j3NH-Y6i0" },
+            { name: "Microbubble-enhanced HIFU in a liver", image: "res/simulations/y.png", computer: "Delta",      computerUrl: "https://www.ncsa.illinois.edu/research/project-highlights/delta/",              accelerators: "128 CPU cores", walltime: "26m",  source: "https://www.youtube.com/watch?v=TbR0MEdG8OU" },
             { name: "Kidney stone stress waves",           image: "res/simulations/l.png", computer: "Bridges2",   computerUrl: "https://www.psc.edu/resources/bridges-2/",                                     accelerators: "8 V100s",   walltime: "20m",  source: "https://www.youtube.com/watch?v=Q2L0J68qnRw" },
             { name: "Whale bubble net feeding",            image: "res/simulations/p.png", computer: "Delta",      computerUrl: "https://www.ncsa.illinois.edu/research/project-highlights/delta/",              accelerators: "128 A100s", walltime: "30m",  source: "https://www.youtube.com/watch?v=6EpP6tdCZSA" },
             { name: "Earplug acoustics (kinetic energy)",  image: "res/simulations/q.png", computer: "Delta",      computerUrl: "https://www.ncsa.illinois.edu/research/project-highlights/delta/",              accelerators: "8 A100s",   walltime: "5h",   source: "https://www.youtube.com/watch?v=xSW5wZkdbrc" },

docs/module_categories.json

@@ -81,7 +81,8 @@
             "m_checker",
             "m_checker_common",
             "m_sim_helpers",
-            "m_derived_variables"
+            "m_derived_variables",
+            "m_patch_geometries"
         ]
     },
     {

docs/res/simulations/y.png

@@ -79,6 +79,7 @@
             "ib": "T",
             "num_ibs": 0,
             "viscous": "T",
+            "many_ib_patch_parallelism": "T",
             # Collision model (soft-sphere, from 3D_mibm_sphere_head_on_collision)
             "collision_model": 1,
             "coefficient_of_restitution": 0.9,