Merge pull request #6 from MFlowCode/master

Merge MFC changes into ibdata-output

Author: mrvandenboom

.github/scripts/monitor_slurm_job.sh

@@ -9,11 +9,17 @@ cleanup() {
   if [ -n "${tail_pid:-}" ]; then
     kill "${tail_pid}" 2>/dev/null || true
   fi
-  # Cancel the SLURM job if the monitor is exiting due to an error
-  # (e.g., the CI runner is being killed). Don't cancel on success.
+  # Cancel the SLURM job only if it is still active in the scheduler.
+  # If the job already left the queue (squeue returns empty), it has finished
+  # and run_monitored_slurm_job.sh will recover via sacct — don't cancel it.
   if [ "${monitor_success:-0}" -ne 1 ] && [ -n "${job_id:-}" ]; then
-    echo "Monitor exiting abnormally — cancelling SLURM job $job_id"
-    scancel "$job_id" 2>/dev/null || true
+    active_state=$(squeue -j "$job_id" -h -o '%T' 2>/dev/null | head -n1 | tr -d ' ' || echo "")
+    if [ -n "$active_state" ]; then
+      echo "Monitor exiting abnormally — cancelling SLURM job $job_id (state: $active_state)"
+      scancel "$job_id" 2>/dev/null || true
+    else
+      echo "Monitor exiting abnormally — SLURM job $job_id already left queue, not cancelling"
+    fi
   fi
 }
 trap cleanup EXIT
@@ -56,9 +62,11 @@ get_job_state() {
 }
 
 # Check if a state is terminal (job is done, for better or worse)
+# PREEMPTED is intentionally excluded: with --requeue the job restarts under
+# the same job ID and we must keep monitoring rather than exiting early.
 is_terminal_state() {
   case "$1" in
-    COMPLETED|FAILED|CANCELLED|CANCELLED+|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|BOOT_FAIL|DEADLINE|PREEMPTED|REVOKED)
+    COMPLETED|FAILED|CANCELLED|CANCELLED+|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|BOOT_FAIL|DEADLINE|REVOKED)
       return 0 ;;
     *)
       return 1 ;;
@@ -74,7 +82,7 @@ while [ ! -f "$output_file" ]; do
   state=$(get_job_state "$job_id")
 
   case "$state" in
-    PENDING|CONFIGURING)
+    PENDING|CONFIGURING|PREEMPTED)
       unknown_count=0
       sleep 5
       ;;

.github/scripts/prebuild-case-optimization.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
 # Pre-builds all benchmark cases with --case-optimization.
+# No GPU hardware needed — compilation only.
 # Can run in two modes:
 #   1. Direct (Frontier login nodes): pass cluster/device/interface as args
-#   2. Inside SLURM (Phoenix): uses $job_device/$job_interface from submit.sh
+#   2. Inside SLURM (Phoenix): uses $job_device/$job_interface from submit-slurm-job.sh
 # Usage: bash prebuild-case-optimization.sh [<cluster> <device> <interface>]
 
 set -e
 
-# Support both positional args (direct invocation) and env vars (SLURM via submit.sh)
+# Support both positional args (direct invocation) and env vars (SLURM)
 cluster="${1:-${job_cluster:-phoenix}}"
 job_device="${2:-$job_device}"
 job_interface="${3:-$job_interface}"
@@ -21,8 +22,18 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
+rm -rf build
+
 . ./mfc.sh load -c "$flag" -m g
-source .github/scripts/gpu-opts.sh
+
+# Set GPU build flags from interface — this is always a GPU build.
+# Don't use gpu-opts.sh since $job_device may be "cpu" when submitted
+# to a CPU SLURM partition (no GPU hardware needed for compilation).
+case "$job_interface" in
+    acc) gpu_opts="--gpu acc" ;;
+    omp) gpu_opts="--gpu mp" ;;
+    *)   echo "ERROR: prebuild requires gpu interface (acc or omp)"; exit 1 ;;
+esac
 
 for case in benchmarks/*/case.py; do
     echo "=== Pre-building: $case ==="

.github/scripts/retry-build.sh

@@ -1,30 +1,16 @@
 #!/bin/bash
-# Provides retry_build(): 3-attempt loop with configurable cleanup.
-# Set RETRY_CLEAN_CMD to override cleanup (default: rm -rf build/staging build/install build/lock.yaml).
-# Set RETRY_VALIDATE_CMD to run a post-build validation; failure triggers a retry.
+# Provides retry_build(): 2-attempt loop.
+# On failure of attempt 1, nukes the entire build directory before attempt 2.
+# If RETRY_VALIDATE_CMD is set, runs it after a successful build; a non-zero
+# exit triggers the same nuke-and-retry, catching e.g. SIGILL from binaries
+# compiled on a different CPU architecture.
 # Usage: source .github/scripts/retry-build.sh
 #        retry_build ./mfc.sh build -j 8 --gpu acc
-
-# Try normal cleanup; if it fails, escalate to cache nuke.
-_retry_clean() {
-    local clean_cmd="$1"
-    if eval "$clean_cmd" 2>/dev/null; then
-        return 0
-    fi
-    echo "  Normal cleanup failed."
-    if type _cache_nuke > /dev/null 2>&1; then
-        echo "  Escalating to NFS cache nuke..."
-        _cache_nuke
-    else
-        echo "  _cache_nuke not available, best-effort rm."
-        rm -rf build/staging build/install build/lock.yaml 2>/dev/null || true
-    fi
-}
+#        RETRY_VALIDATE_CMD='./syscheck' retry_build ./mfc.sh build -j 8
 
 retry_build() {
-    local clean_cmd="${RETRY_CLEAN_CMD:-rm -rf build/staging build/install build/lock.yaml}"
+    local max_attempts=2
     local validate_cmd="${RETRY_VALIDATE_CMD:-}"
-    local max_attempts=3
     local attempt=1
     while [ $attempt -le $max_attempts ]; do
         echo "Build attempt $attempt of $max_attempts..."
@@ -33,8 +19,8 @@ retry_build() {
                 if ! eval "$validate_cmd"; then
                     echo "Post-build validation failed on attempt $attempt."
                     if [ $attempt -lt $max_attempts ]; then
-                        echo "Cleaning and retrying in 5s..."
-                        _retry_clean "$clean_cmd"
+                        echo "  Nuking build directory before retry..."
+                        rm -rf build 2>/dev/null || true
                         sleep 5
                         attempt=$((attempt + 1))
                         continue
@@ -48,8 +34,8 @@ retry_build() {
             return 0
         fi
         if [ $attempt -lt $max_attempts ]; then
-            echo "Build failed on attempt $attempt. Retrying in 30s..."
-            _retry_clean "$clean_cmd"
+            echo "  Build failed — nuking build directory before retry..."
+            rm -rf build 2>/dev/null || true
             sleep 30
         else
             echo "Build failed after $max_attempts attempts."

.github/scripts/run-tests-with-retry.sh

@@ -44,7 +44,7 @@ for case in "${benchmarks[@]}"; do
     rm -rf "$case_dir/D" "$case_dir/p_all" "$case_dir/restart_data"
 
     # Build + run with --case-optimization, small grid, 10 timesteps
-    if ./mfc.sh run "$case" --case-optimization $gpu_opts -n "$ngpus" -j "$(nproc)" -- --gbpp 1 --steps 10; then
+    if ./mfc.sh run "$case" --case-optimization $gpu_opts -n "$ngpus" -j 8 -- --gbpp 1 --steps 10; then
         # Validate output
         if build/venv/bin/python3 .github/scripts/check_case_optimization_output.py "$case_dir"; then
             echo "PASS: $case_name"

.github/scripts/run_case_optimization.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Run monitor_slurm_job.sh and recover if the monitor is killed (e.g. SIGKILL
+# from the runner OS) before the SLURM job completes.  When the monitor exits
+# non-zero, sacct is used to verify the job's actual final state; if the SLURM
+# job succeeded we exit 0 so the CI step is not falsely marked as failed.
+#
+# Usage: run_monitored_slurm_job.sh <job_id> <output_file>
+
+set -euo pipefail
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 <job_id> <output_file>"
+    exit 1
+fi
+
+job_id="$1"
+output_file="$2"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+monitor_exit=0
+bash "$SCRIPT_DIR/monitor_slurm_job.sh" "$job_id" "$output_file" || monitor_exit=$?
+
+if [ "$monitor_exit" -ne 0 ]; then
+    echo "Monitor exited with code $monitor_exit; re-checking SLURM job $job_id final state..."
+    # Give the SLURM epilog time to finalize if the job just finished
+    sleep 30
+    final_state=$(sacct -j "$job_id" -n -X -P -o State 2>/dev/null | head -n1 | cut -d'|' -f1 | tr -d ' ' || true)
+    final_state="${final_state:-UNKNOWN}"
+    final_exit=$(sacct -j "$job_id" -X --format=ExitCode --noheader --parsable2 2>/dev/null | head -n1 | tr -d ' ' || true)
+    final_exit="${final_exit:-}"
+    echo "Final SLURM state=$final_state exit=$final_exit"
+    if [ "$final_state" = "COMPLETED" ] && [ "$final_exit" = "0:0" ]; then
+        echo "SLURM job $job_id completed successfully despite monitor failure — continuing."
+    else
+        echo "ERROR: SLURM job $job_id did not complete successfully (state=$final_state exit=$final_exit)"
+        exit 1
+    fi
+fi

.github/scripts/run_monitored_slurm_job.sh

@@ -20,6 +20,16 @@ echo "=========================================="
 echo "Starting parallel benchmark jobs..."
 echo "=========================================="
 
+# For Phoenix GPU benchmarks, select a consistent GPU partition before launching
+# both parallel jobs so PR and master always land on the same GPU type.
+if [ "$device" = "gpu" ] && [ "$cluster" = "phoenix" ]; then
+    echo "Selecting Phoenix GPU partition for benchmark consistency..."
+    # Require 2 nodes so both PR and master jobs can run concurrently.
+    GPU_PARTITION_MIN_NODES=2 source "${SCRIPT_DIR}/select-gpu-partition.sh"
+    BENCH_GPU_PARTITION="$SELECTED_GPU_PARTITION"
+    export BENCH_GPU_PARTITION
+fi
+
 # Run both jobs with monitoring using dedicated script from PR
 # Use stdbuf for line-buffered output and prefix each line for clarity
 (set -o pipefail; stdbuf -oL -eL bash "${SCRIPT_DIR}/submit_and_monitor_bench.sh" pr "$device" "$interface" "$cluster" 2>&1 | while IFS= read -r line; do echo "[PR] $line"; done) &
@@ -32,22 +42,26 @@ echo "Master job started in background (PID: $master_pid)"
 
 echo "Waiting for both jobs to complete..."
 
-# Wait and capture exit codes reliably
+# Wait and capture exit codes reliably.
+# Use `wait ... || exit=$?` to avoid set -e aborting on the first failure
+# (which would orphan the second job).
 pr_exit=0
 master_exit=0
 
-wait "$pr_pid"
-pr_exit=$?
+wait "$pr_pid" || pr_exit=$?
 if [ "$pr_exit" -ne 0 ]; then
   echo "PR job exited with code: $pr_exit"
+  echo "Last 50 lines of PR job log:"
+  tail -n 50 "pr/bench-${device}-${interface}.out" 2>/dev/null || echo "  Could not read PR log"
 else
   echo "PR job completed successfully"
 fi
 
-wait "$master_pid"
-master_exit=$?
+wait "$master_pid" || master_exit=$?
 if [ "$master_exit" -ne 0 ]; then
   echo "Master job exited with code: $master_exit"
+  echo "Last 50 lines of master job log:"
+  tail -n 50 "master/bench-${device}-${interface}.out" 2>/dev/null || echo "  Could not read master log"
 else
   echo "Master job completed successfully"
 fi

.github/scripts/run_parallel_benchmarks.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Select the best available Phoenix GPU partition using sinfo.
+# Sources into caller: exports SELECTED_GPU_PARTITION.
+#
+# Priority order prefers partitions most likely to have availability.
+# V100 is last due to slower performance near the test time limit.
+# Falls back to gpu-l40s if no partition meets the idle node threshold.
+# RTX 6000 nodes are excluded (too slow for the test suite time limit).
+#
+# Optional: set GPU_PARTITION_MIN_NODES before sourcing to require a minimum
+# number of idle/mix nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs).
+#
+# Usage: source .github/scripts/select-gpu-partition.sh
+
+_GPU_PARTITION_PRIORITY="gpu-l40s gpu-h200 gpu-h100 gpu-a100 gpu-v100"
+_GPU_PARTITION_FALLBACK="gpu-l40s"
+_GPU_PARTITION_MIN_NODES="${GPU_PARTITION_MIN_NODES:-1}"
+
+SELECTED_GPU_PARTITION=""
+for _part in $_GPU_PARTITION_PRIORITY; do
+    _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^(idle|mix)" || true)
+    if [ "${_idle:-0}" -ge "$_GPU_PARTITION_MIN_NODES" ]; then
+        SELECTED_GPU_PARTITION="$_part"
+        echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle/mix nodes)"
+        break
+    fi
+done
+
+if [ -z "$SELECTED_GPU_PARTITION" ]; then
+    echo "WARNING: No idle GPU partition found; falling back to $_GPU_PARTITION_FALLBACK (may queue)"
+    SELECTED_GPU_PARTITION="$_GPU_PARTITION_FALLBACK"
+fi
+
+export SELECTED_GPU_PARTITION
+unset _GPU_PARTITION_PRIORITY _GPU_PARTITION_FALLBACK _GPU_PARTITION_MIN_NODES _part _idle