Merge branch 'master' into bala_force_changes

Author: sbryngelson

.github/file-filter.yml

@@ -25,6 +25,7 @@ yml: &yml
   - '.github/workflows/phoenix/**'
   - '.github/workflows/frontier/**'
   - '.github/workflows/frontier_amd/**'
+  - '.github/scripts/**'
   - '.github/workflows/bench.yml'
   - '.github/workflows/test.yml'
   - '.github/workflows/formatting.yml'

.github/scripts/bench-preamble.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Shared preamble for benchmark scripts: detects GPUs, sets build/device opts.
+# Sets: $gpu_opts, $build_opts, $device_opts, $n_ranks, $ngpus, $gpu_ids
+# Usage: source .github/scripts/bench-preamble.sh
+
+source .github/scripts/detect-gpus.sh
+source .github/scripts/gpu-opts.sh
+
+n_ranks=12
+build_opts="$gpu_opts"
+device_opts=""
+if [ "$job_device" = "gpu" ]; then
+    n_ranks=$ngpus
+    device_opts="$gpu_opts -g $gpu_ids"
+fi

.github/scripts/check_case_optimization_output.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+"""Validate case-optimization output: check D/*.dat for NaN/Inf via the packer."""
+
+import math
+import sys
+import os
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} <case_directory>", file=sys.stderr)
+    sys.exit(1)
+
+# Allow importing from the repo root
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+
+from toolchain.mfc.packer.pack import compile as pack_compile
+
+case_dir = sys.argv[1]
+if os.path.isfile(case_dir):
+    case_dir = os.path.dirname(case_dir)
+
+pack, err = pack_compile(case_dir)
+if err is not None:
+    print(f"ERROR: {err}")
+    sys.exit(1)
+
+if not pack.entries:
+    print(f"ERROR: No data found in {case_dir}/D/")
+    sys.exit(1)
+
+if pack.has_bad_values():
+    print("ERROR: NaN or Inf detected in output:")
+    for name, entry in pack.entries.items():
+        for i, val in enumerate(entry.doubles):
+            if math.isnan(val) or math.isinf(val):
+                label = 'NaN' if math.isnan(val) else 'Inf'
+                print(f"  {label} at index {i} in {name}")
+                break
+    sys.exit(1)
+
+total = sum(len(e.doubles) for e in pack.entries.values())
+print(f"OK: {len(pack.entries)} files, {total} values — no NaN/Inf found")

.github/scripts/detect-gpus.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Detects GPUs (NVIDIA or AMD), sets $ngpus and $gpu_ids.
+# Usage: source .github/scripts/detect-gpus.sh
+
+ngpus=0
+gpu_ids=""
+if command -v nvidia-smi &>/dev/null; then
+    ngpus=$(nvidia-smi -L | wc -l)
+    gpu_ids=$(seq -s ' ' 0 $((ngpus - 1)))
+elif command -v rocm-smi &>/dev/null; then
+    gpu_ids=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
+    ngpus=$(echo "$gpu_ids" | wc -w)
+fi

.github/scripts/gpu-opts.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Sets $gpu_opts from $job_device and $job_interface.
+# Usage: source .github/scripts/gpu-opts.sh
+
+gpu_opts=""
+if [ "$job_device" = "gpu" ]; then
+    gpu_opts="--gpu"
+    if [ "$job_interface" = "omp" ]; then
+        gpu_opts+=" mp"
+    elif [ "$job_interface" = "acc" ]; then
+        gpu_opts+=" acc"
+    fi
+fi

.github/scripts/monitor_slurm_job.sh

@@ -9,11 +9,17 @@ cleanup() {
   if [ -n "${tail_pid:-}" ]; then
     kill "${tail_pid}" 2>/dev/null || true
   fi
-  # Cancel the SLURM job if the monitor is exiting due to an error
-  # (e.g., the CI runner is being killed). Don't cancel on success.
+  # Cancel the SLURM job only if it is still active in the scheduler.
+  # If the job already left the queue (squeue returns empty), it has finished
+  # and run_monitored_slurm_job.sh will recover via sacct — don't cancel it.
   if [ "${monitor_success:-0}" -ne 1 ] && [ -n "${job_id:-}" ]; then
-    echo "Monitor exiting abnormally — cancelling SLURM job $job_id"
-    scancel "$job_id" 2>/dev/null || true
+    active_state=$(squeue -j "$job_id" -h -o '%T' 2>/dev/null | head -n1 | tr -d ' ' || echo "")
+    if [ -n "$active_state" ]; then
+      echo "Monitor exiting abnormally — cancelling SLURM job $job_id (state: $active_state)"
+      scancel "$job_id" 2>/dev/null || true
+    else
+      echo "Monitor exiting abnormally — SLURM job $job_id already left queue, not cancelling"
+    fi
   fi
 }
 trap cleanup EXIT
@@ -56,9 +62,11 @@ get_job_state() {
 }
 
 # Check if a state is terminal (job is done, for better or worse)
+# PREEMPTED is intentionally excluded: with --requeue the job restarts under
+# the same job ID and we must keep monitoring rather than exiting early.
 is_terminal_state() {
   case "$1" in
-    COMPLETED|FAILED|CANCELLED|CANCELLED+|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|BOOT_FAIL|DEADLINE|PREEMPTED|REVOKED)
+    COMPLETED|FAILED|CANCELLED|CANCELLED+|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|BOOT_FAIL|DEADLINE|REVOKED)
       return 0 ;;
     *)
       return 1 ;;
@@ -74,7 +82,7 @@ while [ ! -f "$output_file" ]; do
   state=$(get_job_state "$job_id")
 
   case "$state" in
-    PENDING|CONFIGURING)
+    PENDING|CONFIGURING|PREEMPTED)
       unknown_count=0
       sleep 5
       ;;

.github/scripts/prebuild-case-optimization.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Pre-builds all benchmark cases with --case-optimization.
+# Can run in two modes:
+#   1. Direct (Frontier login nodes): pass cluster/device/interface as args
+#   2. Inside SLURM (Phoenix): uses $job_device/$job_interface from submit.sh
+# Usage: bash prebuild-case-optimization.sh [<cluster> <device> <interface>]
+
+set -e
+
+# Support both positional args (direct invocation) and env vars (SLURM via submit.sh)
+cluster="${1:-${job_cluster:-phoenix}}"
+job_device="${2:-$job_device}"
+job_interface="${3:-$job_interface}"
+
+# Derive module flag from cluster name
+case "$cluster" in
+    phoenix)      flag="p" ;;
+    frontier)     flag="f" ;;
+    frontier_amd) flag="famd" ;;
+    *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
+esac
+
+rm -rf build
+
+. ./mfc.sh load -c "$flag" -m g
+source .github/scripts/gpu-opts.sh
+
+for case in benchmarks/*/case.py; do
+    echo "=== Pre-building: $case ==="
+    ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8
+done

.github/scripts/retry-build.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Provides retry_build(): 2-attempt loop.
+# On failure of attempt 1, nukes the entire build directory before attempt 2.
+# Set RETRY_VALIDATE_CMD to run a post-build validation; failure triggers a retry.
+# Usage: source .github/scripts/retry-build.sh
+#        retry_build ./mfc.sh build -j 8 --gpu acc
+
+retry_build() {
+    local validate_cmd="${RETRY_VALIDATE_CMD:-}"
+    local max_attempts=2
+    local attempt=1
+    while [ $attempt -le $max_attempts ]; do
+        echo "Build attempt $attempt of $max_attempts..."
+        if "$@"; then
+            if [ -n "$validate_cmd" ]; then
+                if ! eval "$validate_cmd"; then
+                    echo "Post-build validation failed on attempt $attempt."
+                    if [ $attempt -lt $max_attempts ]; then
+                        echo "  Nuking build directory before retry..."
+                        rm -rf build 2>/dev/null || true
+                        sleep 5
+                        attempt=$((attempt + 1))
+                        continue
+                    else
+                        echo "Validation still failing after $max_attempts attempts."
+                        return 1
+                    fi
+                fi
+            fi
+            echo "Build succeeded on attempt $attempt."
+            return 0
+        fi
+        if [ $attempt -lt $max_attempts ]; then
+            echo "  Build failed — nuking build directory before retry..."
+            rm -rf build 2>/dev/null || true
+            sleep 30
+        else
+            echo "Build failed after $max_attempts attempts."
+            return 1
+        fi
+        attempt=$((attempt + 1))
+    done
+}

.github/scripts/run-tests-with-retry.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Runs ./mfc.sh test with all provided arguments, then retries a small number
+# of sporadic failures (up to 5). Exits non-zero on real failures.
+# Usage: bash .github/scripts/run-tests-with-retry.sh [mfc test args...]
+
+# Extract flags that should carry over to retries (retries build their own
+# argument list with --only, so we capture passthrough flags here).
+PASSTHROUGH=""
+for arg in "$@"; do
+    case "$arg" in
+        --test-all) PASSTHROUGH="$PASSTHROUGH --test-all" ;;
+    esac
+done
+
+rm -f tests/failed_uuids.txt
+TEST_EXIT=0
+/bin/bash mfc.sh test "$@" || TEST_EXIT=$?
+
+# Retry only if a small number of tests failed (sporadic failures)
+if [ -s tests/failed_uuids.txt ]; then
+    NUM_FAILED=$(wc -l < tests/failed_uuids.txt)
+    if [ "$NUM_FAILED" -le 5 ]; then
+        FAILED=$(tr '\n' ' ' < tests/failed_uuids.txt)
+        echo ""
+        echo "=== Retrying $NUM_FAILED failed test(s): $FAILED ==="
+        echo ""
+        /bin/bash mfc.sh test -v --max-attempts 3 -j "$(nproc)" --only $FAILED $PASSTHROUGH || exit $?
+    else
+        echo "Too many failures ($NUM_FAILED) to retry — likely a real issue."
+        exit 1
+    fi
+elif [ "$TEST_EXIT" -ne 0 ]; then
+    exit $TEST_EXIT
+fi

.github/scripts/run_case_optimization.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Case-optimization CI test script.
+# Runs inside a SLURM job — expects $job_device and $job_interface from submit.sh.
+
+set -e
+
+source .github/scripts/detect-gpus.sh
+source .github/scripts/gpu-opts.sh
+
+# Default to 1 GPU if detection found none but we're in GPU mode
+if [ "$job_device" = "gpu" ] && [ "$ngpus" -eq 0 ]; then
+    ngpus=1
+fi
+
+# Verify the venv Python interpreter exists (created by ./mfc.sh build)
+if [ ! -x build/venv/bin/python3 ]; then
+    echo "ERROR: build/venv/bin/python3 not found."
+    echo "The MFC build venv may not have been created. Was the pre-build step successful?"
+    exit 1
+fi
+
+benchmarks=(
+    benchmarks/5eq_rk3_weno3_hllc/case.py
+    benchmarks/viscous_weno5_sgb_acoustic/case.py
+    benchmarks/hypo_hll/case.py
+    benchmarks/ibm/case.py
+    benchmarks/igr/case.py
+)
+
+passed=0
+failed=0
+failed_cases=""
+
+for case in "${benchmarks[@]}"; do
+    case_dir="$(dirname "$case")"
+    case_name="$(basename "$case_dir")"
+    echo ""
+    echo "========================================"
+    echo "Case-optimization test: $case_name"
+    echo "========================================"
+
+    # Clean any previous output
+    rm -rf "$case_dir/D" "$case_dir/p_all" "$case_dir/restart_data"
+
+    # Build + run with --case-optimization, small grid, 10 timesteps
+    if ./mfc.sh run "$case" --case-optimization $gpu_opts -n "$ngpus" -j "$(nproc)" -- --gbpp 1 --steps 10; then
+        # Validate output
+        if build/venv/bin/python3 .github/scripts/check_case_optimization_output.py "$case_dir"; then
+            echo "PASS: $case_name"
+            passed=$((passed + 1))
+        else
+            echo "FAIL: $case_name (validation error)"
+            failed=$((failed + 1))
+            failed_cases="$failed_cases $case_name"
+        fi
+    else
+        echo "FAIL: $case_name (build or run error)"
+        failed=$((failed + 1))
+        failed_cases="$failed_cases $case_name"
+    fi
+
+    # Clean up output between cases
+    rm -rf "$case_dir/D" "$case_dir/p_all" "$case_dir/restart_data"
+done
+
+echo ""
+echo "========================================"
+echo "Case-optimization summary: $passed passed, $failed failed"
+if [ $failed -gt 0 ]; then
+    echo "Failed cases:$failed_cases"
+fi
+echo "========================================"
+
+[ $failed -eq 0 ] && exit 0 || exit 1