ci: add explanatory comments, fix backtick in submit.sh

- submit.sh: replace backtick with $() for job_slug; add comment that
  the sed pipeline must stay in sync with submit-job.sh
- test.yml: explain clean: false (preserves .slurm_job_id for stale job
  detection) and continue-on-error on Frontier (CCE compiler instability)
- run_parallel_benchmarks.sh: explain GPU partition priority order
  (prefer smaller/older partitions to leave large nodes for production)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/scripts/run_parallel_benchmarks.sh

@@ -24,6 +24,9 @@ echo "=========================================="
 # both parallel jobs so PR and master always land on the same GPU type.
 if [ "$device" = "gpu" ] && [ "$cluster" = "phoenix" ]; then
     echo "Selecting Phoenix GPU partition for benchmark consistency..."
+    # Prefer older/smaller partitions first (rtx6000, l40s, v100) to leave
+    # large modern nodes (h200, h100, a100) free for production workloads.
+    # rtx6000 has the most nodes and gives the most consistent baselines.
     BENCH_GPU_PARTITION=""
     for part in gpu-rtx6000 gpu-l40s gpu-v100 gpu-h200 gpu-h100 gpu-a100; do
         # || true: grep -c exits 1 on zero matches (or when sinfo returns no output

.github/workflows/phoenix/submit.sh

@@ -23,8 +23,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Submit (idempotent — skips resubmission if a live job already exists)
 bash "$SCRIPT_DIR/submit-job.sh" "$@"
 
-# Derive the same job slug and file paths as submit-job.sh
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
+# Derive the same job slug and file paths as submit-job.sh.
+# NOTE: this sed pipeline must stay identical to the one in submit-job.sh —
+# if they diverge the id-file will not be found and the monitor will fail.
+job_slug="$(basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g')-$2-$3"
 output_file="$job_slug.out"
 id_file="${job_slug}.slurm_job_id"
 

.github/workflows/test.yml

@@ -155,6 +155,9 @@ jobs:
     name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})"
     if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && github.event.pull_request.draft != true
     needs: [lint-gate, file-changes]
+    # Frontier CCE compiler is periodically broken by toolchain updates (e.g.
+    # cpe/25.03 introduced an IPA SIGSEGV in CCE 19.0.0). Allow Frontier to
+    # fail without blocking PR merges; Phoenix remains a hard gate.
     continue-on-error: ${{ matrix.runner == 'frontier' }}
     timeout-minutes: 480
     strategy:
@@ -233,6 +236,8 @@ jobs:
       - name: Clone
         uses: actions/checkout@v4
         with:
+          # clean: false preserves .slurm_job_id files across reruns so
+          # submit-job.sh can detect and cancel stale SLURM jobs on retry.
           clean: false
 
       - name: Build
@@ -294,6 +299,7 @@ jobs:
     name: "Case Opt | ${{ matrix.cluster_name }} (${{ matrix.device }}-${{ matrix.interface }})"
     if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && github.event.pull_request.draft != true
     needs: [lint-gate, file-changes]
+    # Frontier is non-blocking for the same reason as the self job above.
     continue-on-error: ${{ matrix.runner == 'frontier' }}
     timeout-minutes: 480
     strategy: