diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub new file mode 100644 index 0000000000..0e0fa8e7e3 --- /dev/null +++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub @@ -0,0 +1,59 @@ +#!/bin/bash --login +#SBATCH --job-name=advanced +#SBATCH --nodes=1 +#SBATCH --ntasks=4 +#SBATCH --qos=vcell +#SBATCH --partition=vcell +#SBATCH -o %x.stdout +#SBATCH -e %x.stderr +#SBATCH --cpus-per-task=1 +#SBATCH --time=01:00:00 + +log_file="${SLURM_JOB_NAME}.log" +total_jobs=10 # total number of jobs to run +timeout_duration=15s # maximum allowed runtime for each job +max_concurrent_jobs=$SLURM_NTASKS # number of jobs allowed at once + +# clear the log file at the start +echo "job execution log" > $log_file +echo "------------------" >> $log_file + +declare -A job_pid_map # associative array to store job index -> pid mapping +job_pids=() # list to track pids + +for i in $(seq 0 $((total_jobs - 1))); do + # log job start + echo "job $i started at $(date)" >> $log_file + timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput & # run each script in parallel + pid=$! # capture the job pid + job_pids+=($pid) # store the pid + job_pid_map[$pid]=$i # map job index to pid + + # wait for a batch of ntasks before launching new ones + if (( (i + 1) % max_concurrent_jobs == 0 || i == total_jobs - 1 )); then + echo "waiting for jobs to finish at $(date)" >> $log_file + + for pid in "${job_pids[@]}"; do + wait $pid + exit_code=$? # capture the exit code of the job + job_index=${job_pid_map[$pid]} # retrieve original job index + echo "job $job_index with pid $pid finished with exit code $exit_code at $(date)" >> $log_file + done + + # clear job pids for the next batch + job_pids=() + job_pid_map=() + fi +done + +echo "starting the last job at $(date)" >> $log_file +timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh & # run last script with a timeout +last_pid=$! +wait $last_pid # explicitly wait for last.sh to finish +exit_code=$? # capture the exit code of the job +echo "job 'last' with pid $last_pid finished with exit code $exit_code at $(date)" >> $log_file +echo "the final job finished at $(date)" >> $log_file + +echo "all jobs completed at $(date)" >> $log_file +echo "end script" >> $log_file + diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub new file mode 100644 index 0000000000..62d4a86783 --- /dev/null +++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub @@ -0,0 +1,44 @@ +#!/bin/bash --login +#SBATCH --job-name=SimID_483072744_0_ # this matches the input example we're going to use +#SBATCH --nodes=1 +#SBATCH --qos=vcell +#SBATCH --partition=vcell +#SBATCH --cpus-per-task=1 +#SBATCH --time=01:00:00 # terminates any running job that failed to stop after timeout_duration below +#SBATCH -o %x_%a.out # uses the job name (%x) and array task ID (%a) +#SBATCH --array=1-7%3 # running 7 jobs, limit concurrency to 3 jobs (numbers must be explicit) + +log_file="job_status.log" +timeout_duration=5s # Maximum allowed runtime for each job + +# clean log file BEFORE any job starts +echo "Job Execution Log" > $log_file +echo "------------------" >> $log_file +sync # ensure file write completes before jobs begin + +# job name is $SLURM_JOB_NAME (defined in SBATCH --job-name=SimID_483072744_0_) +# can be combined with task id, like: echo "Job $SLURM_JOB_NAME ($SLURM_ARRAY_TASK_ID) started at $(date)" >> $log_file + +echo "Job $SLURM_ARRAY_TASK_ID started at $(date)" >> $log_file + +# execute the corresponding subscript with timeout +timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$SLURM_ARRAY_TASK_ID.sh + +exit_code=$? + +# Log results and job completion status +if [ $exit_code -eq 0 ]; then + echo "Job $SLURM_ARRAY_TASK_ID finished successfully at $(date)" >> $log_file +elif [ $exit_code -eq 124 ]; then + echo "Job $SLURM_ARRAY_TASK_ID timed out at $(date)" >> $log_file +elif [ $exit_code -gt 128 ]; then + signal=$((exit_code - 128)) + echo "Job $SLURM_ARRAY_TASK_ID terminated by signal $signal at $(date)" >> $log_file +else + echo "Job $SLURM_ARRAY_TASK_ID failed with exit code $exit_code at $(date)" >> $log_file +fi + +# output results +result=$(cat subscript_$SLURM_ARRAY_TASK_ID.output 2>/dev/null || echo "No output (job may have failed)") +echo "Result of job $SLURM_ARRAY_TASK_ID is $result" +echo "Result of job $SLURM_ARRAY_TASK_ID: $result" >> $log_file diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput new file mode 100644 index 0000000000..e15927c866 --- /dev/null +++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput @@ -0,0 +1,10 @@ +0 7 0 +1 9 99 +2 11 0 +3 44 0 +4 7 0 +5 8 130 +6 9 0 +7 8 0 +8 7 0 +9 6 0 diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub new file mode 100644 index 0000000000..c61083833d --- /dev/null +++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub @@ -0,0 +1,71 @@ +#!/bin/bash --login +#SBATCH --job-name=dynamic +#SBATCH --nodes=1 +#SBATCH --ntasks=4 # use 3 for fast testing +#SBATCH --qos=vcell +#SBATCH --partition=vcell +#SBATCH -o %x.stdout +#SBATCH -e %x.stderr +#SBATCH --cpus-per-task=1 +#SBATCH --time=01:00:00 + +log_file="${SLURM_JOB_NAME}.log" +total_jobs=10 # total number of jobs to run (use 7 for fast testing) +timeout_duration=15s # maximum allowed runtime for each job +max_concurrent_jobs=$SLURM_NTASKS # number of jobs allowed at once + +# Clear the log file at the start +echo "Job Execution Log" > $log_file +echo "------------------" >> $log_file + +declare -A job_pid_map # associative array to store job index -> PID mapping +job_pids=() # list to track PIDs +running_jobs=0 + +for i in $(seq 0 $((total_jobs - 1))); do + # log job start + echo "Job $i started at $(date)" >> $log_file + timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput & # run each script in parallel + pid=$! # capture the job PID + job_pids+=($pid) # store the PID + job_pid_map[$pid]=$i # map job index to PID + ((running_jobs++)) # increment running job count + + # wait for a finished job before launching a new one if we hit the concurrency limit + while (( running_jobs >= max_concurrent_jobs )); do + for idx in "${!job_pids[@]}"; do + pid="${job_pids[$idx]}" + if ! kill -0 "$pid" 2>/dev/null; then # check if process is still running + wait "$pid" # ensure exit status is collected + exit_code=$? + job_index=${job_pid_map[$pid]} # retrieve original job index + echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file + unset "job_pids[$idx]" # remove PID from list + unset "job_pid_map[$pid]" # remove mapping + ((running_jobs--)) # decrement count + break # break once we free up a slot + fi + done + sleep 1 # allow brief pause before rechecking + done +done + +# final wait for any remaining jobs +for pid in "${job_pids[@]}"; do + wait $pid + exit_code=$? + job_index=${job_pid_map[$pid]} # retrieve original job index + echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file +done +echo "Batch jobs completed at $(date)" >> $log_file + +echo "Starting the last job at $(date)" >> $log_file +timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh & # run last script with a timeout +last_pid=$! +wait $last_pid # explicitly wait for last.sh to finish +exit_code=$? # capture the exit code of the job +echo "Job 'Last' with PID $last_pid finished with exit code $exit_code at $(date)" >> $log_file +echo "The final job finished at $(date)" >> $log_file + +echo "All jobs completed at $(date)" >> $log_file +echo "End Script" >> $log_file diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh new file mode 100644 index 0000000000..86135025c5 --- /dev/null +++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "LAST job start at $(date)" > last.output +sleep 2 +echo "LAST job finished at $(date)" >> last.output +exit 0 + diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh new file mode 100644 index 0000000000..56af792544 --- /dev/null +++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ensure the script receives the required argument +if [ -z "$1" ]; then + echo "Error: No job index provided." + exit 1 +fi + +job_index=$1 +input_file="dynamic.langevinInput" # file containing job details +output_file="subscript_${job_index}.output" + +# locate the row with the matching job index +read duration exit_code < <(awk -v idx="$job_index" '$1 == idx {print $2, $3}' "$input_file") + +# ensure the job index is found +if [ -z "$duration" ] || [ -z "$exit_code" ]; then + echo "Error: No matching job entry for index $job_index" + exit 1 +fi + +# log job start +echo "Job $job_index started at $(date)" > "$output_file" + +# simulate workload based on parsed duration +sleep "$duration" + +# log job completion +echo "Job $job_index finished at $(date) with exit code $exit_code" >> "$output_file" + +# exit with the specified exit code +exit "$exit_code"