Dan ss slurm array2

vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub

@@ -0,0 +1,59 @@
+#!/bin/bash --login
+#SBATCH --job-name=advanced
+#SBATCH --nodes=1
+#SBATCH --ntasks=4
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH -o %x.stdout
+#SBATCH -e %x.stderr
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00
+
+log_file="${SLURM_JOB_NAME}.log"
+total_jobs=10  # total number of jobs to run
+timeout_duration=15s  # maximum allowed runtime for each job
+max_concurrent_jobs=$SLURM_NTASKS  # number of jobs allowed at once
+
+# clear the log file at the start
+echo "job execution log" > $log_file
+echo "------------------" >> $log_file
+
+declare -A job_pid_map  # associative array to store job index -> pid mapping
+job_pids=()  # list to track pids
+
+for i in $(seq 0 $((total_jobs - 1))); do
+    # log job start
+    echo "job $i started at $(date)" >> $log_file
+    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput &  # run each script in parallel
+    pid=$!  # capture the job pid
+    job_pids+=($pid)  # store the pid
+    job_pid_map[$pid]=$i  # map job index to pid
+
+    # wait for a batch of ntasks before launching new ones
+    if (( (i + 1) % max_concurrent_jobs == 0 || i == total_jobs - 1 )); then
+        echo "waiting for jobs to finish at $(date)" >> $log_file
+
+        for pid in "${job_pids[@]}"; do
+            wait $pid
+            exit_code=$?  # capture the exit code of the job
+            job_index=${job_pid_map[$pid]}  # retrieve original job index
+            echo "job $job_index with pid $pid finished with exit code $exit_code at $(date)" >> $log_file
+        done
+
+        # clear job pids for the next batch
+        job_pids=()
+        job_pid_map=()
+    fi
+done
+
+echo "starting the last job at $(date)" >> $log_file
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # run last script with a timeout
+last_pid=$!
+wait $last_pid  # explicitly wait for last.sh to finish
+exit_code=$?  # capture the exit code of the job
+echo "job 'last' with pid $last_pid finished with exit code $exit_code at $(date)" >> $log_file
+echo "the final job finished at $(date)" >> $log_file
+
+echo "all jobs completed at $(date)" >> $log_file
+echo "end script" >> $log_file
+

vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub

@@ -0,0 +1,44 @@
+#!/bin/bash --login
+#SBATCH --job-name=SimID_483072744_0_   # this matches the input example we're going to use
+#SBATCH --nodes=1
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00     # terminates any running job that failed to stop after timeout_duration below
+#SBATCH -o %x_%a.out        # uses the job name (%x) and array task ID (%a)
+#SBATCH --array=1-7%3       # running 7 jobs, limit concurrency to 3 jobs (numbers must be explicit)
+
+log_file="job_status.log"
+timeout_duration=5s  # Maximum allowed runtime for each job
+
+# clean log file BEFORE any job starts
+echo "Job Execution Log" > $log_file
+echo "------------------" >> $log_file
+sync  # ensure file write completes before jobs begin
+
+# job name is $SLURM_JOB_NAME (defined in SBATCH --job-name=SimID_483072744_0_)
+# can be combined with task id, like: echo "Job $SLURM_JOB_NAME ($SLURM_ARRAY_TASK_ID) started at $(date)" >> $log_file
+
+echo "Job $SLURM_ARRAY_TASK_ID started at $(date)" >> $log_file
+
+# execute the corresponding subscript with timeout
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$SLURM_ARRAY_TASK_ID.sh
+
+exit_code=$?
+
+# Log results and job completion status
+if [ $exit_code -eq 0 ]; then
+    echo "Job $SLURM_ARRAY_TASK_ID finished successfully at $(date)" >> $log_file
+elif [ $exit_code -eq 124 ]; then
+    echo "Job $SLURM_ARRAY_TASK_ID timed out at $(date)" >> $log_file
+elif [ $exit_code -gt 128 ]; then
+    signal=$((exit_code - 128))
+    echo "Job $SLURM_ARRAY_TASK_ID terminated by signal $signal at $(date)" >> $log_file
+else
+    echo "Job $SLURM_ARRAY_TASK_ID failed with exit code $exit_code at $(date)" >> $log_file
+fi
+
+# output results
+result=$(cat subscript_$SLURM_ARRAY_TASK_ID.output 2>/dev/null || echo "No output (job may have failed)")
+echo "Result of job $SLURM_ARRAY_TASK_ID is $result"
+echo "Result of job $SLURM_ARRAY_TASK_ID: $result" >> $log_file

vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput

@@ -0,0 +1,10 @@
+0 7 0
+1 9 99
+2 11 0
+3 44 0
+4 7 0
+5 8 130
+6 9 0
+7 8 0
+8 7 0
+9 6 0

vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub

@@ -0,0 +1,71 @@
+#!/bin/bash --login
+#SBATCH --job-name=dynamic
+#SBATCH --nodes=1
+#SBATCH --ntasks=4     # use 3 for fast testing
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH -o %x.stdout
+#SBATCH -e %x.stderr
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00
+
+log_file="${SLURM_JOB_NAME}.log"
+total_jobs=10  # total number of jobs to run (use 7 for fast testing)
+timeout_duration=15s  # maximum allowed runtime for each job
+max_concurrent_jobs=$SLURM_NTASKS  # number of jobs allowed at once
+
+# Clear the log file at the start
+echo "Job Execution Log" > $log_file
+echo "------------------" >> $log_file
+
+declare -A job_pid_map  # associative array to store job index -> PID mapping
+job_pids=()  # list to track PIDs
+running_jobs=0
+
+for i in $(seq 0 $((total_jobs - 1))); do
+    # log job start
+    echo "Job $i started at $(date)" >> $log_file
+    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput & # run each script in parallel
+    pid=$!  # capture the job PID
+    job_pids+=($pid)  # store the PID
+    job_pid_map[$pid]=$i  # map job index to PID
+    ((running_jobs++))  # increment running job count
+
+    # wait for a finished job before launching a new one if we hit the concurrency limit
+    while (( running_jobs >= max_concurrent_jobs )); do
+        for idx in "${!job_pids[@]}"; do
+            pid="${job_pids[$idx]}"
+            if ! kill -0 "$pid" 2>/dev/null; then  # check if process is still running
+                wait "$pid"  # ensure exit status is collected
+                exit_code=$?
+                job_index=${job_pid_map[$pid]}  # retrieve original job index
+                echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+                unset "job_pids[$idx]"  # remove PID from list
+                unset "job_pid_map[$pid]"  # remove mapping
+                ((running_jobs--))  # decrement count
+                break  # break once we free up a slot
+            fi
+        done
+        sleep 1  # allow brief pause before rechecking
+    done
+done
+
+# final wait for any remaining jobs
+for pid in "${job_pids[@]}"; do
+    wait $pid
+    exit_code=$?
+    job_index=${job_pid_map[$pid]}  # retrieve original job index
+    echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+done
+echo "Batch jobs completed at $(date)" >> $log_file
+
+echo "Starting the last job at $(date)" >> $log_file
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # run last script with a timeout
+last_pid=$!
+wait $last_pid  # explicitly wait for last.sh to finish
+exit_code=$?    # capture the exit code of the job
+echo "Job 'Last' with PID $last_pid finished with exit code $exit_code at $(date)" >> $log_file
+echo "The final job finished at $(date)" >> $log_file
+
+echo "All jobs completed at $(date)" >> $log_file
+echo "End Script" >> $log_file

vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "LAST job start at $(date)" > last.output
+sleep 2
+echo "LAST job finished at $(date)" >> last.output
+exit 0
+

vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# ensure the script receives the required argument
+if [ -z "$1" ]; then
+    echo "Error: No job index provided."
+    exit 1
+fi
+
+job_index=$1
+input_file="dynamic.langevinInput"  # file containing job details
+output_file="subscript_${job_index}.output"
+
+# locate the row with the matching job index
+read duration exit_code < <(awk -v idx="$job_index" '$1 == idx {print $2, $3}' "$input_file")
+
+# ensure the job index is found
+if [ -z "$duration" ] || [ -z "$exit_code" ]; then
+    echo "Error: No matching job entry for index $job_index"
+    exit 1
+fi
+
+# log job start
+echo "Job $job_index started at $(date)" > "$output_file"
+
+# simulate workload based on parsed duration
+sleep "$duration"
+
+# log job completion
+echo "Job $job_index finished at $(date) with exit code $exit_code" >> "$output_file"
+
+# exit with the specified exit code
+exit "$exit_code"