Merge branch 'master' into fix/viscous-3d-gpu-private

Author: sbryngelson

.github/scripts/clean-build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Provides clean_build(): renames build/ aside and deletes it in the background.
+# mv is a metadata-only operation that succeeds even with stale NFS file handles,
+# unlike rm -rf which fails on ESTALE. The background delete is best-effort and
+# scoped to this job's PID to avoid races with concurrent matrix jobs.
+#
+# Usage: source .github/scripts/clean-build.sh
+#        clean_build
+
+clean_build() {
+    # Clean up leftover stale directories from previous runs before adding a new one.
+    rm -rf build.stale.* 2>/dev/null || true
+    mv build "build.stale.$$" 2>/dev/null || true
+    rm -rf "build.stale.$$" 2>/dev/null & disown
+}

.github/scripts/monitor_slurm_job.sh

@@ -52,6 +52,15 @@ get_job_state() {
   # Fallback to sacct (works for completed/historical jobs)
   if command -v sacct >/dev/null 2>&1; then
     state=$(sacct -j "$jid" -n -X -P -o State 2>/dev/null | head -n1 | cut -d'|' -f1 || true)
+    # When a job is preempted+requeued, sacct -X reports PREEMPTED for the
+    # original attempt while the requeued run may have completed.  Check all
+    # records (without -X) for a terminal state that supersedes PREEMPTED.
+    if [ "$state" = "PREEMPTED" ]; then
+      requeue_state=$(sacct -j "$jid" -n -P -o State 2>/dev/null | grep -v PREEMPTED | head -n1 | cut -d'|' -f1 || true)
+      if [ -n "$requeue_state" ]; then
+        state="$requeue_state"
+      fi
+    fi
     if [ -n "$state" ]; then
       echo "$state"
       return

.github/scripts/prebuild-case-optimization.sh

@@ -22,7 +22,8 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
-rm -rf build
+source .github/scripts/clean-build.sh
+clean_build
 
 . ./mfc.sh load -c "$flag" -m g
 

.github/scripts/retry-sbatch.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Provides retry_sbatch(): submits a job script string via sbatch with retries.
+# Only retries on known transient SLURM/infrastructure errors (socket timeouts,
+# connection failures). Hard failures (bad account, invalid partition, QOS
+# violations) are not retried.
+#
+# Usage: source .github/scripts/retry-sbatch.sh
+#        job_id=$(retry_sbatch "$script_contents")
+
+retry_sbatch() {
+    local script_contents="$1"
+    local max_attempts=3
+    local attempt=1
+    local submit_output job_id last_output=""
+
+    while [ $attempt -le $max_attempts ]; do
+        echo "sbatch attempt $attempt of $max_attempts..." >&2
+        submit_output=$(printf '%s\n' "$script_contents" | sbatch 2>&1) || true
+        job_id=$(echo "$submit_output" | grep -oE 'Submitted batch job ([0-9]+)' | grep -oE '[0-9]+$')
+        if [ -n "$job_id" ]; then
+            echo "$job_id"
+            return 0
+        fi
+        last_output="$submit_output"
+        echo "sbatch failed: $submit_output" >&2
+        if ! echo "$submit_output" | grep -qiE "timed out|connection refused|connection reset|temporarily unavailable"; then
+            echo "Non-transient sbatch failure — not retrying." >&2
+            return 1
+        fi
+        if [ $attempt -lt $max_attempts ]; then
+            echo "Transient error — retrying in 30s..." >&2
+            sleep 30
+        fi
+        attempt=$((attempt + 1))
+    done
+
+    echo "sbatch failed after $max_attempts attempts. Last error: $last_output" >&2
+    return 1
+}
+

.github/scripts/run_parallel_benchmarks.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
-# Run PR and master benchmarks in parallel and verify outputs
+# Run PR and master benchmarks and verify outputs.
+# Both SLURM jobs are submitted up front so they run concurrently on
+# compute nodes (fair comparison under the same cluster load), but
+# monitoring happens sequentially to stay within the per-user cgroup
+# memory limit on login nodes (4 GB on Phoenix shared by 7 runners).
 # Usage: run_parallel_benchmarks.sh <device> <interface> <cluster>
 
 set -euo pipefail
@@ -17,88 +21,104 @@ cluster="$3"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 echo "=========================================="
-echo "Starting parallel benchmark jobs..."
+echo "Starting benchmark jobs..."
 echo "=========================================="
 
-# For Phoenix GPU benchmarks, select a consistent GPU partition before launching
-# both parallel jobs so PR and master always land on the same GPU type.
+# For Phoenix GPU benchmarks, select a consistent GPU partition so PR and
+# master always land on the same GPU type.
 if [ "$device" = "gpu" ] && [ "$cluster" = "phoenix" ]; then
     echo "Selecting Phoenix GPU partition for benchmark consistency..."
-    # Require 2 nodes so both PR and master jobs can run concurrently.
+    # Require 2 nodes so both jobs can run concurrently on compute.
     GPU_PARTITION_MIN_NODES=2 source "${SCRIPT_DIR}/select-gpu-partition.sh"
     BENCH_GPU_PARTITION="$SELECTED_GPU_PARTITION"
     export BENCH_GPU_PARTITION
 fi
 
-# Run both jobs with monitoring using dedicated script from PR
-# Use stdbuf for line-buffered output and prefix each line for clarity
-(set -o pipefail; stdbuf -oL -eL bash "${SCRIPT_DIR}/submit_and_monitor_bench.sh" pr "$device" "$interface" "$cluster" 2>&1 | while IFS= read -r line; do echo "[PR] $line"; done) &
-pr_pid=$!
-echo "PR job started in background (PID: $pr_pid)"
-
-(set -o pipefail; stdbuf -oL -eL bash "${SCRIPT_DIR}/submit_and_monitor_bench.sh" master "$device" "$interface" "$cluster" 2>&1 | while IFS= read -r line; do echo "[MASTER] $line"; done) &
-master_pid=$!
-echo "Master job started in background (PID: $master_pid)"
-
-echo "Waiting for both jobs to complete..."
-
-# Wait and capture exit codes reliably.
-# Use `wait ... || exit=$?` to avoid set -e aborting on the first failure
-# (which would orphan the second job).
+# The bench script must come from the PR tree (master may not have it).
+PR_BENCH_SCRIPT="$(cd "${SCRIPT_DIR}/../workflows/common" && pwd)/bench.sh"
+# Must match the slug computed by submit-slurm-job.sh:
+#   basename("bench.sh") → "bench" → "bench-${device}-${interface}"
+job_slug="bench-${device}-${interface}"
+
+# --- Phase 1: Submit both SLURM jobs (no monitoring yet) ---
+echo "Submitting PR benchmark..."
+(cd pr && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster")
+pr_job_id=$(cat "pr/${job_slug}.slurm_job_id")
+echo "PR job submitted: $pr_job_id"
+
+echo "Submitting master benchmark..."
+(cd master && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster")
+master_job_id=$(cat "master/${job_slug}.slurm_job_id")
+echo "Master job submitted: $master_job_id"
+
+echo "Both SLURM jobs submitted — running concurrently on compute nodes."
+echo "Monitoring sequentially to conserve login node memory."
+
+# --- Phase 2: Monitor sequentially (one at a time on login node) ---
+echo ""
+echo "=== Monitoring PR job $pr_job_id ==="
 pr_exit=0
-master_exit=0
-
-wait "$pr_pid" || pr_exit=$?
+bash "${SCRIPT_DIR}/run_monitored_slurm_job.sh" "$pr_job_id" "pr/${job_slug}.out" || pr_exit=$?
 if [ "$pr_exit" -ne 0 ]; then
-  echo "PR job exited with code: $pr_exit"
-  echo "Last 50 lines of PR job log:"
-  tail -n 50 "pr/bench-${device}-${interface}.out" 2>/dev/null || echo "  Could not read PR log"
+    echo "PR job exited with code: $pr_exit"
+    tail -n 50 "pr/${job_slug}.out" 2>/dev/null || echo "  Could not read PR log"
 else
-  echo "PR job completed successfully"
+    echo "PR job completed successfully"
 fi
 
-wait "$master_pid" || master_exit=$?
+echo ""
+echo "=== Monitoring master job $master_job_id ==="
+master_exit=0
+bash "${SCRIPT_DIR}/run_monitored_slurm_job.sh" "$master_job_id" "master/${job_slug}.out" || master_exit=$?
 if [ "$master_exit" -ne 0 ]; then
-  echo "Master job exited with code: $master_exit"
-  echo "Last 50 lines of master job log:"
-  tail -n 50 "master/bench-${device}-${interface}.out" 2>/dev/null || echo "  Could not read master log"
+    echo "Master job exited with code: $master_exit"
+    tail -n 50 "master/${job_slug}.out" 2>/dev/null || echo "  Could not read master log"
 else
-  echo "Master job completed successfully"
+    echo "Master job completed successfully"
 fi
 
-# Warn if either job failed (partial results may still be usable)
+# --- Phase 3: Verify outputs ---
 if [ "${pr_exit}" -ne 0 ] || [ "${master_exit}" -ne 0 ]; then
-  echo "WARNING: Benchmark jobs had failures: pr=${pr_exit}, master=${master_exit}"
-  echo "Checking for partial results..."
+    echo "WARNING: Benchmark jobs had failures: pr=${pr_exit}, master=${master_exit}"
+    echo "Checking for partial results..."
 else
-  echo "=========================================="
-  echo "Both benchmark jobs completed successfully!"
-  echo "=========================================="
+    echo "=========================================="
+    echo "Both benchmark jobs completed successfully!"
+    echo "=========================================="
 fi
 
-# Final verification that output files exist before proceeding
-pr_yaml="pr/bench-${device}-${interface}.yaml"
-master_yaml="master/bench-${device}-${interface}.yaml"
+pr_yaml="pr/${job_slug}.yaml"
+master_yaml="master/${job_slug}.yaml"
+
+# Wait briefly for YAML files to appear on NFS.  When monitoring starts
+# after a job has already completed (common for the second job), the
+# recovery path in run_monitored_slurm_job.sh sleeps 30s, but NFS
+# propagation can take longer under load.
+for yaml in "$pr_yaml" "$master_yaml"; do
+    attempts=0
+    while [ ! -f "$yaml" ] && [ $attempts -lt 6 ]; do
+        echo "Waiting for $yaml to appear (NFS propagation)..."
+        sleep 5
+        attempts=$((attempts + 1))
+    done
+done
 
 if [ ! -f "$pr_yaml" ]; then
-  echo "ERROR: PR benchmark output not found: $pr_yaml"
-  ls -la pr/ || true
-  echo ""
-  echo "Last 100 lines of PR log:"
-  tail -n 100 "pr/bench-${device}-${interface}.out" 2>/dev/null || echo "  Could not read PR log"
-  exit 1
+    echo "ERROR: PR benchmark output not found: $pr_yaml"
+    ls -la pr/ || true
+    echo ""
+    tail -n 100 "pr/${job_slug}.out" 2>/dev/null || echo "  Could not read PR log"
+    exit 1
 fi
 
 if [ ! -f "$master_yaml" ]; then
-  echo "ERROR: Master benchmark output not found: $master_yaml"
-  ls -la master/ || true
-  echo ""
-  echo "Last 100 lines of master log:"
-  tail -n 100 "master/bench-${device}-${interface}.out" 2>/dev/null || echo "  Could not read master log"
-  exit 1
+    echo "ERROR: Master benchmark output not found: $master_yaml"
+    ls -la master/ || true
+    echo ""
+    tail -n 100 "master/${job_slug}.out" 2>/dev/null || echo "  Could not read master log"
+    exit 1
 fi
 
 echo "Verified both YAML files exist:"
 echo "  - $pr_yaml"
 echo "  - $master_yaml"
-

.github/scripts/submit-slurm-job.sh

@@ -85,9 +85,9 @@ if [ "$device" = "cpu" ]; then
     case "$cluster" in
         phoenix)
             sbatch_device_opts="\
-#SBATCH -p cpu-small
-#SBATCH --ntasks-per-node=24
-#SBATCH --mem-per-cpu=2G"
+#SBATCH -p cpu-small,cpu-medium,cpu-large
+#SBATCH --ntasks-per-node=12
+#SBATCH --mem-per-cpu=8G"
             ;;
         frontier|frontier_amd)
             sbatch_device_opts="\
@@ -161,8 +161,9 @@ rm -f "$output_file"
 # --- Module load mode (short form) ---
 module_mode=$([ "$device" = "gpu" ] && echo "g" || echo "c")
 
-# --- Submit ---
-submit_output=$(sbatch <<EOT
+# --- Submit (with retries for transient SLURM errors) ---
+source "${SCRIPT_DIR}/retry-sbatch.sh"
+_sbatch_script=$(cat <<EOT
 #!/bin/bash
 #SBATCH -J ${job_prefix}-${job_slug}
 #SBATCH --account=${account}
@@ -192,16 +193,16 @@ $sbatch_script_contents
 EOT
 )
 
-job_id=$(echo "$submit_output" | grep -oE '[0-9]+')
-if [ -z "$job_id" ]; then
-    echo "ERROR: Failed to submit job. sbatch output:"
-    echo "$submit_output"
-    exit 1
-fi
+job_id=$(retry_sbatch "$_sbatch_script")
+unset _sbatch_script
 
 echo "Submitted batch job $job_id"
 echo "$job_id" > "$id_file"
 echo "Job ID written to $id_file"
 
-# --- Monitor ---
-bash "$SCRIPT_DIR/run_monitored_slurm_job.sh" "$job_id" "$output_file"
+# --- Monitor (skip if SUBMIT_ONLY=1, e.g. for parallel submission) ---
+if [ "${SUBMIT_ONLY:-0}" = "1" ]; then
+    echo "SUBMIT_ONLY mode: skipping monitor (job_id=$job_id output=$output_file)"
+else
+    bash "$SCRIPT_DIR/run_monitored_slurm_job.sh" "$job_id" "$output_file"
+fi

.github/workflows/common/bench.sh

@@ -25,7 +25,8 @@ fi
 # Phoenix builds inside SLURM; Frontier pre-builds via build.sh on the login node.
 # Phoenix: always nuke stale builds (heterogeneous compute nodes → ISA mismatch risk).
 if [ "$job_cluster" = "phoenix" ]; then
-    rm -rf build
+    source .github/scripts/clean-build.sh
+    clean_build
 fi
 
 if [ ! -d "build" ]; then

.github/workflows/common/test.sh

@@ -8,12 +8,26 @@ set -euo pipefail
 source .github/scripts/gpu-opts.sh
 build_opts="$gpu_opts"
 
+# --- Phoenix TMPDIR setup ---
+# Phoenix compute nodes have a small /tmp. With 8 parallel test threads each
+# spawning MPI processes, it fills up and ORTE session dir creation fails.
+# Redirect TMPDIR to project storage, same as bench.sh.
+if [ "$job_cluster" = "phoenix" ]; then
+    tmpbuild=/storage/project/r-sbryngelson3-0/sbryngelson3/mytmp_build
+    currentdir=$tmpbuild/run-$(( RANDOM % 9000 ))
+    mkdir -p $tmpbuild
+    mkdir -p $currentdir
+    export TMPDIR=$currentdir
+    trap 'rm -rf "$currentdir" || true' EXIT
+fi
+
 # --- Build (if not pre-built on login node) ---
 # Phoenix builds inside SLURM; Frontier pre-builds via build.sh on the login node.
 # Phoenix builds inside SLURM on heterogeneous compute nodes — always start fresh
 # to avoid SIGILL from stale binaries compiled on a different microarchitecture.
 if [ "$job_cluster" = "phoenix" ]; then
-    rm -rf build
+    source .github/scripts/clean-build.sh
+    clean_build
 fi
 
 if [ ! -d "build" ]; then

.github/workflows/frontier/build.sh

@@ -20,7 +20,8 @@ build_opts="$gpu_opts"
 
 . ./mfc.sh load -c $compiler_flag -m $([ "$job_device" = "gpu" ] && echo "g" || echo "c")
 
-rm -rf build
+source .github/scripts/clean-build.sh
+clean_build
 
 source .github/scripts/retry-build.sh
 if [ "$run_bench" == "bench" ]; then

.github/workflows/test.yml

@@ -234,6 +234,9 @@ jobs:
           # submit-slurm-job.sh can detect and cancel stale SLURM jobs on retry.
           clean: false
 
+      - name: Clean stale output files
+        run:  rm -f *.out
+
       - name: Build (login node)
         if:   matrix.cluster != 'phoenix'
         timeout-minutes: 60
@@ -317,6 +320,9 @@ jobs:
         with:
           clean: false
 
+      - name: Clean stale output files
+        run:  rm -f *.out
+
       - name: Pre-Build (SLURM)
         if:   matrix.cluster == 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh cpu ${{ matrix.interface }} ${{ matrix.cluster }}