Auto-requeue SLURM jobs on preemption

Add --requeue to Phoenix sbatch scripts so preempted embers-QOS jobs
are automatically rescheduled. Remove PREEMPTED from the monitor's
terminal state list so it keeps waiting through the requeue cycle.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/scripts/monitor_slurm_job.sh

@@ -58,7 +58,7 @@ get_job_state() {
 # Check if a state is terminal (job is done, for better or worse)
 is_terminal_state() {
   case "$1" in
-    COMPLETED|FAILED|CANCELLED|CANCELLED+|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|PREEMPTED|BOOT_FAIL|DEADLINE)
+    COMPLETED|FAILED|CANCELLED|CANCELLED+|TIMEOUT|OUT_OF_MEMORY|NODE_FAIL|BOOT_FAIL|DEADLINE)
       return 0 ;;
     *)
       return 1 ;;

.github/workflows/phoenix/submit-bench.sh

@@ -44,6 +44,7 @@ sbatch <<EOT
 $sbatch_device_opts
 #SBATCH -t 04:00:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -q embers                  # QOS Name
+#SBATCH --requeue                  # Auto-requeue on preemption
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 
 set -e

.github/workflows/phoenix/submit.sh

@@ -48,6 +48,7 @@ submit_output=$(sbatch <<EOT
 $sbatch_device_opts
 #SBATCH -t 03:00:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -q embers                  # QOS Name
+#SBATCH --requeue                  # Auto-requeue on preemption
 #SBATCH -o$output_file             # Combined output and error messages file
 
 set -e