From d368781c998fe6930a02f2540ff8a84e934985c4 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Fri, 23 May 2025 13:42:24 -0400
Subject: [PATCH 01/12] WIP: slurm batch

---
 .../resources/slurm_fixtures/langevin/subscript_generic.sh  | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/subscript_generic.sh

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/subscript_generic.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/subscript_generic.sh
new file mode 100644
index 0000000000..3b742f5073
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/subscript_generic.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+echo "job7 start at $(date)" > subscript_7.output
+sleep 2
+echo "job7 finished at $(date)" >> subscript_7.output
+exit 0

From 69cfef5f59e95b4a3fde30de7ecd11b379d56737 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Fri, 23 May 2025 13:57:35 -0400
Subject: [PATCH 02/12] WIP: slurm batch

---
 .../langevin/slurm_array_poc/advanced.sub     | 64 +++++++++++++++++++
 .../langevin/slurm_array_poc/complex.sub      | 44 +++++++++++++
 .../langevin/slurm_array_poc/complex2.sub     | 52 +++++++++++++++
 .../langevin/slurm_array_poc/subscript_1.sh   |  7 ++
 .../langevin/slurm_array_poc/subscript_2.sh   |  7 ++
 .../langevin/slurm_array_poc/subscript_3.sh   |  7 ++
 .../langevin/slurm_array_poc/subscript_4.sh   |  7 ++
 .../langevin/slurm_array_poc/subscript_5.sh   |  7 ++
 .../langevin/slurm_array_poc/subscript_6.sh   |  7 ++
 .../subscript_7.sh}                           |  1 +
 10 files changed, 203 insertions(+)
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh
 rename vcell-server/src/test/resources/slurm_fixtures/langevin/{subscript_generic.sh => slurm_array_poc/subscript_7.sh} (99%)

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
new file mode 100644
index 0000000000..ca0b640bc5
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
@@ -0,0 +1,64 @@
+#!/bin/bash --login
+#SBATCH --job-name=advancedJob
+#SBATCH --nodes=1
+#SBATCH --ntasks=3
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH -o %x.stdout	# was /home/FCAM/vasilescu/advancedJob.stdout
+#SBATCH -e %x.stderr
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00
+
+log_file="job_status.log"
+max_jobs=3  # Number of cores available, must mirror --ntasks
+total_jobs=7  # Total number of jobs to run
+timeout_duration=5s  # Maximum allowed runtime for each job
+
+# Clear the log file at the start
+echo "Job Execution Log" > $log_file
+echo "------------------" >> $log_file
+
+job_pids=()  # Array to track job PIDs
+
+for i in $(seq 1 $total_jobs); do
+    # Log job start
+    echo "Job $i started at $(date)" >> $log_file
+    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script with a timeout in parallel
+    job_pids+=($!)  # Store the PID of the background job
+
+    # Manage batches of max_jobs
+    if (( i % max_jobs == 0 || i == total_jobs )); then
+        echo "Waiting for jobs to finish at $(date)" >> $log_file
+        
+        # Wait for each job in the current batch
+        for pid in "${job_pids[@]}"; do
+            wait $pid
+            exit_code=$?  # Capture the exit code of the job
+            
+            if [ $exit_code -eq 0 ]; then
+                echo "Job with PID $pid finished successfully at $(date)" >> $log_file
+            elif [ $exit_code -eq 124 ]; then
+                echo "Job with PID $pid timed out at $(date)" >> $log_file
+            elif [ $exit_code -gt 128 ]; then
+                signal=$((exit_code - 128))
+                echo "Job with PID $pid terminated by signal $signal at $(date)" >> $log_file
+            else
+                echo "Job with PID $pid failed with exit code $exit_code at $(date)" >> $log_file
+            fi
+        done
+        
+        # Clear job PIDs for the next batch
+        job_pids=()
+    fi
+done
+
+# Output results of all jobs and log them
+for i in $(seq 1 $total_jobs); do
+    result=$(cat subscript_$i.output 2>/dev/null || echo "No output (job may have failed)")
+    echo "Result of job $i is $result"
+    echo "Result of job $i: $result" >> $log_file
+done
+
+# Log the end of the job process
+echo "All jobs completed at $(date)" >> $log_file
+echo "End Job"
\ No newline at end of file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub
new file mode 100644
index 0000000000..c38ae02951
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub
@@ -0,0 +1,44 @@
+#!/bin/bash --login
+#SBATCH --job-name=jobStepEx1
+#SBATCH --nodes=1
+#SBATCH --ntasks=2
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH -o /home/FCAM/vasilescu/jobStepEx1.stdout
+#SBATCH -e /home/FCAM/vasilescu/jobStepEx1.stderr
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00
+
+# Run with  sbatch complex.sub
+log_file="job_status.log"
+max_jobs=2  # Number of cores available
+total_jobs=5  # Total number of jobs to run
+
+# Clear the log file at the start
+echo "Job Execution Log" > $log_file
+echo "------------------" >> $log_file
+
+for i in $(seq 1 $total_jobs); do
+    # Log job start
+    echo "Job $i started at $(date)" >> $log_file
+    srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script in parallel
+
+    # Log waiting when maximum parallel jobs are reached
+    if (( i % max_jobs == 0 )); then
+        echo "Waiting for jobs to finish at $(date)" >> $log_file
+        wait
+    fi
+done
+
+# Wait for any remaining jobs to complete
+wait
+echo "All jobs completed at $(date)" >> $log_file
+
+# Output results of all jobs and log them
+for i in $(seq 1 $total_jobs); do
+    result=$(cat subscript_$i.output)
+    echo "Result of job $i: $result" >> $log_file
+done
+
+# Log the end of the job process
+echo "End Job at $(date)" >> $log_file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub
new file mode 100644
index 0000000000..1f63528ff1
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub
@@ -0,0 +1,52 @@
+#!/bin/bash --login
+#SBATCH --job-name=jobStepEx1
+#SBATCH --nodes=1
+#SBATCH --ntasks=2
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH -o /home/FCAM/vasilescu/jobStepEx1.stdout
+#SBATCH -e /home/FCAM/vasilescu/jobStepEx1.stderr
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00
+
+log_file="job_status.log"
+max_jobs=2  # Number of cores available
+total_jobs=5  # Total number of jobs to run
+
+# Clear the log file at the start
+echo "Job Execution Log" > $log_file
+echo "------------------" >> $log_file
+
+job_pids=()  # Array to track job PIDs
+
+for i in $(seq 1 $total_jobs); do
+    # Log job start
+    echo "Job $i started at $(date)" >> $log_file
+    srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script in parallel
+    job_pids+=($!)  # Store the PID of the background job
+
+    # Manage batches of max_jobs
+    if (( i % max_jobs == 0 || i == total_jobs )); then
+        echo "Waiting for jobs to finish at $(date)" >> $log_file
+        
+        # Wait for each job in the current batch
+        for pid in "${job_pids[@]}"; do
+            wait $pid  # Wait for the specific job
+            echo "Job with PID $pid finished at $(date)" >> $log_file
+        done
+        
+        # Clear job PIDs for the next batch
+        job_pids=()
+    fi
+done
+
+# Output results of all jobs and log them
+for i in $(seq 1 $total_jobs); do
+    result=$(cat subscript_$i.output)
+    echo "Result of job $i is $result"
+    echo "Result of job $i: $result" >> $log_file
+done
+
+# Log the end of the job process
+echo "All jobs completed at $(date)" >> $log_file
+echo "End Job"
\ No newline at end of file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh
new file mode 100644
index 0000000000..3ded5de1f4
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "job1 start at $(date)" > subscript_1.output
+sleep 2
+echo "job1 finished at $(date)" >> subscript_1.output
+exit 0
+
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh
new file mode 100644
index 0000000000..066fbd60e5
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "job2 start at $(date)" > subscript_2.output
+sleep 2
+echo "job2 finished at $(date)" >> subscript_2.output
+exit 99
+
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
new file mode 100644
index 0000000000..19948b3b43
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "job3 start at $(date)" > subscript_3.output
+sleep 3
+echo "job3 finished at $(date)" >> subscript_3.output
+exit 0
+
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
new file mode 100644
index 0000000000..c8f9247916
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "job4 start at $(date)" > subscript_4.output
+sleep 11
+echo "job4 finished at $(date)" >> subscript_4.output
+exit 0
+
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh
new file mode 100644
index 0000000000..64a899bb49
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "job5 start at $(date)" > subscript_5.output
+sleep 2
+echo "job5 finished at $(date)" >> subscript_5.output
+exit 0
+
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh
new file mode 100644
index 0000000000..91988d3961
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "job6 start at $(date)" > subscript_6.output
+sleep 2
+echo "job6 finished at $(date)" >> subscript_6.output
+exit 130
+
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/subscript_generic.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_7.sh
similarity index 99%
rename from vcell-server/src/test/resources/slurm_fixtures/langevin/subscript_generic.sh
rename to vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_7.sh
index 3b742f5073..d79eadd9db 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/subscript_generic.sh
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_7.sh
@@ -4,3 +4,4 @@ echo "job7 start at $(date)" > subscript_7.output
 sleep 2
 echo "job7 finished at $(date)" >> subscript_7.output
 exit 0
+

From 81161b171652844093932970db457f5656b235d9 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Fri, 23 May 2025 17:34:17 -0400
Subject: [PATCH 03/12] running on slurm as an array of jobs

---
 .../langevin/slurm_array_poc/advanced.sub     |  3 +-
 .../langevin/slurm_array_poc/array.sub        | 44 +++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
index ca0b640bc5..430548ce08 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
@@ -10,7 +10,6 @@
 #SBATCH --time=01:00:00
 
 log_file="job_status.log"
-max_jobs=3  # Number of cores available, must mirror --ntasks
 total_jobs=7  # Total number of jobs to run
 timeout_duration=5s  # Maximum allowed runtime for each job
 
@@ -27,7 +26,7 @@ for i in $(seq 1 $total_jobs); do
     job_pids+=($!)  # Store the PID of the background job
 
     # Manage batches of max_jobs
-    if (( i % max_jobs == 0 || i == total_jobs )); then
+    if (( i % SLURM_NTASKS == 0 || i == total_jobs )); then
         echo "Waiting for jobs to finish at $(date)" >> $log_file
         
         # Wait for each job in the current batch
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub
new file mode 100644
index 0000000000..62d4a86783
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/array.sub
@@ -0,0 +1,44 @@
+#!/bin/bash --login
+#SBATCH --job-name=SimID_483072744_0_   # this matches the input example we're going to use
+#SBATCH --nodes=1
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00     # terminates any running job that failed to stop after timeout_duration below
+#SBATCH -o %x_%a.out        # uses the job name (%x) and array task ID (%a)
+#SBATCH --array=1-7%3       # running 7 jobs, limit concurrency to 3 jobs (numbers must be explicit)
+
+log_file="job_status.log"
+timeout_duration=5s  # Maximum allowed runtime for each job
+
+# clean log file BEFORE any job starts
+echo "Job Execution Log" > $log_file
+echo "------------------" >> $log_file
+sync  # ensure file write completes before jobs begin
+
+# job name is $SLURM_JOB_NAME (defined in SBATCH --job-name=SimID_483072744_0_)
+# can be combined with task id, like: echo "Job $SLURM_JOB_NAME ($SLURM_ARRAY_TASK_ID) started at $(date)" >> $log_file
+
+echo "Job $SLURM_ARRAY_TASK_ID started at $(date)" >> $log_file
+
+# execute the corresponding subscript with timeout
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$SLURM_ARRAY_TASK_ID.sh
+
+exit_code=$?
+
+# Log results and job completion status
+if [ $exit_code -eq 0 ]; then
+    echo "Job $SLURM_ARRAY_TASK_ID finished successfully at $(date)" >> $log_file
+elif [ $exit_code -eq 124 ]; then
+    echo "Job $SLURM_ARRAY_TASK_ID timed out at $(date)" >> $log_file
+elif [ $exit_code -gt 128 ]; then
+    signal=$((exit_code - 128))
+    echo "Job $SLURM_ARRAY_TASK_ID terminated by signal $signal at $(date)" >> $log_file
+else
+    echo "Job $SLURM_ARRAY_TASK_ID failed with exit code $exit_code at $(date)" >> $log_file
+fi
+
+# output results
+result=$(cat subscript_$SLURM_ARRAY_TASK_ID.output 2>/dev/null || echo "No output (job may have failed)")
+echo "Result of job $SLURM_ARRAY_TASK_ID is $result"
+echo "Result of job $SLURM_ARRAY_TASK_ID: $result" >> $log_file

From 39cad5b6c03dc79cbc869604b86772f139f6ec27 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Thu, 29 May 2025 14:25:45 -0400
Subject: [PATCH 04/12] WIP - abandoning arrays as unsuitable, going back to
 looping, as in advanced.sub

---
 .../langevin/slurm_array_poc/advanced.sub     | 27 ++++++++++++++++---
 .../langevin/slurm_array_poc/last.sh          |  7 +++++
 2 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
index 430548ce08..e9ba145b6a 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
@@ -54,10 +54,31 @@ done
 # Output results of all jobs and log them
 for i in $(seq 1 $total_jobs); do
     result=$(cat subscript_$i.output 2>/dev/null || echo "No output (job may have failed)")
-    echo "Result of job $i is $result"
-    echo "Result of job $i: $result" >> $log_file
+    echo "Result of job $i is $result"              # simple echo goes to stdout
+    echo "Result of job $i: $result" >> $log_file   # redirected echo goes to log_file
 done
 
+echo "Starting the last job at $(date)"
+echo "Starting the last job at $(date)" >> $log_file
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # Run last script with a timeout
+last_pid=$!
+wait $last_pid  # Explicitly wait for last.sh to finish
+exit_code=$?  # Capture the exit code of the job
+if [ $exit_code -eq 0 ]; then
+    echo "Job with PID $last_pid finished successfully at $(date)" >> $log_file
+elif [ $exit_code -eq 124 ]; then
+    echo "Job with PID $last_pid timed out at $(date)" >> $log_file
+elif [ $exit_code -gt 128 ]; then
+    signal=$((exit_code - 128))
+    echo "Job with PID $last_pid terminated by signal $signal at $(date)" >> $log_file
+else
+    echo "Job with PID $last_pid failed with exit code $exit_code at $(date)" >> $log_file
+fi
+echo "Last job finished at $(date)"
+echo "Last job finished at $(date)" >> $log_file
+
 # Log the end of the job process
+echo "All jobs completed at $(date)"
 echo "All jobs completed at $(date)" >> $log_file
-echo "End Job"
\ No newline at end of file
+echo "End Script"
+echo "End Script" >> $log_file
\ No newline at end of file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh
new file mode 100644
index 0000000000..86135025c5
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/last.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "LAST job start at $(date)" > last.output
+sleep 2
+echo "LAST job finished at $(date)" >> last.output
+exit 0
+

From 4807487ce9246b280bb1af041f9d3e6ed13bff08 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Thu, 29 May 2025 15:46:29 -0400
Subject: [PATCH 05/12] dynamic allocation of running slots, once a job
 finishes another one starts

---
 .../langevin/slurm_array_poc/advanced.sub     |  2 +-
 .../langevin/slurm_array_poc/dynamic.sub      | 55 +++++++++++++++++++
 .../langevin/slurm_array_poc/subscript_3.sh   |  2 +-
 .../langevin/slurm_array_poc/subscript_4.sh   |  2 +-
 4 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
index e9ba145b6a..d12fd0e76c 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
@@ -11,7 +11,7 @@
 
 log_file="job_status.log"
 total_jobs=7  # Total number of jobs to run
-timeout_duration=5s  # Maximum allowed runtime for each job
+timeout_duration=10s  # Maximum allowed runtime for each job
 
 # Clear the log file at the start
 echo "Job Execution Log" > $log_file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
new file mode 100644
index 0000000000..318e8b8870
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
@@ -0,0 +1,55 @@
+#!/bin/bash --login
+#SBATCH --job-name=advancedJob
+#SBATCH --nodes=1
+#SBATCH --ntasks=3
+#SBATCH --qos=vcell
+#SBATCH --partition=vcell
+#SBATCH -o %x.stdout
+#SBATCH -e %x.stderr
+#SBATCH --cpus-per-task=1
+#SBATCH --time=01:00:00
+
+log_file="job_status.log"
+total_jobs=7  # Total number of jobs to run
+timeout_duration=10s  # Maximum allowed runtime for each job
+max_concurrent_jobs=$SLURM_NTASKS  # Number of jobs allowed at once
+
+# Clear the log file at the start
+echo "Job Execution Log" > $log_file
+echo "------------------" >> $log_file
+
+job_pids=()  # Array to track job PIDs
+running_jobs=0
+
+for i in $(seq 1 $total_jobs); do
+    # Log job start
+    echo "Job $i started at $(date)" >> $log_file
+    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script in parallel
+    job_pids+=($!)  # Store the PID
+    ((running_jobs++))  # Increment running job count
+
+    # Wait for a finished job before launching a new one if we hit the concurrency limit
+    while (( running_jobs >= max_concurrent_jobs )); do
+        for idx in "${!job_pids[@]}"; do
+            if ! kill -0 "${job_pids[$idx]}" 2>/dev/null; then  # Check if process is still running
+                wait "${job_pids[$idx]}"  # Ensure exit status is collected
+                exit_code=$?
+                echo "Job with PID ${job_pids[$idx]} finished with exit code $exit_code at $(date)" >> $log_file
+                unset "job_pids[$idx]"  # Remove from tracking
+                ((running_jobs--))  # Decrement count
+                break  # Break once we free up a slot
+            fi
+        done
+        sleep 1  # Allow brief pause before rechecking
+    done
+done
+
+# Final wait for any remaining jobs
+for pid in "${job_pids[@]}"; do
+    wait $pid
+    exit_code=$?
+    echo "Job with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+done
+
+echo "All jobs completed at $(date)" >> $log_file
+echo "End Script" >> $log_file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
index 19948b3b43..a4d0582914 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 echo "job3 start at $(date)" > subscript_3.output
-sleep 3
+sleep 7
 echo "job3 finished at $(date)" >> subscript_3.output
 exit 0
 
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
index c8f9247916..67940ff5f0 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 echo "job4 start at $(date)" > subscript_4.output
-sleep 11
+sleep 15
 echo "job4 finished at $(date)" >> subscript_4.output
 exit 0
 

From 13b9e809ff3620127b15fe7658cf1c43c489a513 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Thu, 29 May 2025 17:10:42 -0400
Subject: [PATCH 06/12] dynamic allocation of running slots, once a job
 finishes another one starts, complete version

---
 .../langevin/slurm_array_poc/dynamic.sub      | 30 ++++++++++++++-----
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
index 318e8b8870..99fae73b93 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
@@ -18,24 +18,30 @@ max_concurrent_jobs=$SLURM_NTASKS  # Number of jobs allowed at once
 echo "Job Execution Log" > $log_file
 echo "------------------" >> $log_file
 
-job_pids=()  # Array to track job PIDs
+declare -A job_pid_map  # Associative array to store job index -> PID mapping
+job_pids=()  # List to track PIDs
 running_jobs=0
 
 for i in $(seq 1 $total_jobs); do
     # Log job start
     echo "Job $i started at $(date)" >> $log_file
     timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script in parallel
-    job_pids+=($!)  # Store the PID
+    pid=$!  # Capture the job PID
+    job_pids+=($pid)  # Store the PID
+    job_pid_map[$pid]=$i  # Map job index to PID
     ((running_jobs++))  # Increment running job count
 
     # Wait for a finished job before launching a new one if we hit the concurrency limit
     while (( running_jobs >= max_concurrent_jobs )); do
         for idx in "${!job_pids[@]}"; do
-            if ! kill -0 "${job_pids[$idx]}" 2>/dev/null; then  # Check if process is still running
-                wait "${job_pids[$idx]}"  # Ensure exit status is collected
+            pid="${job_pids[$idx]}"
+            if ! kill -0 "$pid" 2>/dev/null; then  # Check if process is still running
+                wait "$pid"  # Ensure exit status is collected
                 exit_code=$?
-                echo "Job with PID ${job_pids[$idx]} finished with exit code $exit_code at $(date)" >> $log_file
-                unset "job_pids[$idx]"  # Remove from tracking
+                job_index=${job_pid_map[$pid]}  # Retrieve original job index
+                echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+                unset "job_pids[$idx]"  # Remove PID from list
+                unset "job_pid_map[$pid]"  # Remove mapping
                 ((running_jobs--))  # Decrement count
                 break  # Break once we free up a slot
             fi
@@ -48,8 +54,18 @@ done
 for pid in "${job_pids[@]}"; do
     wait $pid
     exit_code=$?
-    echo "Job with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+    job_index=${job_pid_map[$pid]}  # Retrieve original job index
+    echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
 done
+echo "Batch jobs completed at $(date)" >> $log_file
+
+echo "Starting the last job at $(date)" >> $log_file
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # Run last script with a timeout
+last_pid=$!
+wait $last_pid  # Explicitly wait for last.sh to finish
+exit_code=$?  # Capture the exit code of the job
+echo "Job 'Last' with PID $last_pid finished with exit code $exit_code at $(date)" >> $log_file
+echo "Last job finished at $(date)" >> $log_file
 
 echo "All jobs completed at $(date)" >> $log_file
 echo "End Script" >> $log_file

From 385e5f9075a11dcb351527c0946bf23232c77a61 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Mon, 2 Jun 2025 11:52:02 -0400
Subject: [PATCH 07/12] dynamic allocation of jobs, use argument as job index

---
 .../langevin/slurm_array_poc/dynamic.sub      |  2 +-
 .../langevin/slurm_array_poc/subscript.sh     | 22 +++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
index 99fae73b93..d3e6bf3278 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
@@ -25,7 +25,7 @@ running_jobs=0
 for i in $(seq 1 $total_jobs); do
     # Log job start
     echo "Job $i started at $(date)" >> $log_file
-    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script in parallel
+    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i &   # run each script in parallel
     pid=$!  # Capture the job PID
     job_pids+=($pid)  # Store the PID
     job_pid_map[$pid]=$i  # Map job index to PID
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
new file mode 100644
index 0000000000..06c38aa30a
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# ensure the script receives the required argument
+if [ -z "$1" ]; then
+    echo "Error: No job index provided."
+    exit 1
+fi
+
+job_index=$1
+output_file="subscript_${job_index}.output"
+
+# log job start
+echo "Job $job_index start at $(date)" > "$output_file"
+
+# simulating workload (replace with actual job logic)
+sleep $(( job_index + 1 ))  # variable sleep time to simulate different workloads
+
+# log job completion
+echo "Job $job_index finished at $(date)" >> "$output_file"
+
+exit 0
+

From a210c8ce7b78ce676aaebe4c2a10cf5cf39d16e2 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Mon, 2 Jun 2025 12:33:46 -0400
Subject: [PATCH 08/12] dynamic allocation of jobs, provide file with job
 duration and exit code for testing

---
 .../langevin/slurm_array_poc/complex.sub      | 44 ----------------
 .../langevin/slurm_array_poc/complex2.sub     | 52 -------------------
 .../slurm_array_poc/dynamic.langevinInput     |  7 +++
 .../langevin/slurm_array_poc/dynamic.sub      |  4 +-
 .../langevin/slurm_array_poc/subscript.sh     | 26 +++++++---
 5 files changed, 27 insertions(+), 106 deletions(-)
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub
 create mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub
deleted file mode 100644
index c38ae02951..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex.sub
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash --login
-#SBATCH --job-name=jobStepEx1
-#SBATCH --nodes=1
-#SBATCH --ntasks=2
-#SBATCH --qos=vcell
-#SBATCH --partition=vcell
-#SBATCH -o /home/FCAM/vasilescu/jobStepEx1.stdout
-#SBATCH -e /home/FCAM/vasilescu/jobStepEx1.stderr
-#SBATCH --cpus-per-task=1
-#SBATCH --time=01:00:00
-
-# Run with  sbatch complex.sub
-log_file="job_status.log"
-max_jobs=2  # Number of cores available
-total_jobs=5  # Total number of jobs to run
-
-# Clear the log file at the start
-echo "Job Execution Log" > $log_file
-echo "------------------" >> $log_file
-
-for i in $(seq 1 $total_jobs); do
-    # Log job start
-    echo "Job $i started at $(date)" >> $log_file
-    srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script in parallel
-
-    # Log waiting when maximum parallel jobs are reached
-    if (( i % max_jobs == 0 )); then
-        echo "Waiting for jobs to finish at $(date)" >> $log_file
-        wait
-    fi
-done
-
-# Wait for any remaining jobs to complete
-wait
-echo "All jobs completed at $(date)" >> $log_file
-
-# Output results of all jobs and log them
-for i in $(seq 1 $total_jobs); do
-    result=$(cat subscript_$i.output)
-    echo "Result of job $i: $result" >> $log_file
-done
-
-# Log the end of the job process
-echo "End Job at $(date)" >> $log_file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub
deleted file mode 100644
index 1f63528ff1..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/complex2.sub
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash --login
-#SBATCH --job-name=jobStepEx1
-#SBATCH --nodes=1
-#SBATCH --ntasks=2
-#SBATCH --qos=vcell
-#SBATCH --partition=vcell
-#SBATCH -o /home/FCAM/vasilescu/jobStepEx1.stdout
-#SBATCH -e /home/FCAM/vasilescu/jobStepEx1.stderr
-#SBATCH --cpus-per-task=1
-#SBATCH --time=01:00:00
-
-log_file="job_status.log"
-max_jobs=2  # Number of cores available
-total_jobs=5  # Total number of jobs to run
-
-# Clear the log file at the start
-echo "Job Execution Log" > $log_file
-echo "------------------" >> $log_file
-
-job_pids=()  # Array to track job PIDs
-
-for i in $(seq 1 $total_jobs); do
-    # Log job start
-    echo "Job $i started at $(date)" >> $log_file
-    srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script in parallel
-    job_pids+=($!)  # Store the PID of the background job
-
-    # Manage batches of max_jobs
-    if (( i % max_jobs == 0 || i == total_jobs )); then
-        echo "Waiting for jobs to finish at $(date)" >> $log_file
-        
-        # Wait for each job in the current batch
-        for pid in "${job_pids[@]}"; do
-            wait $pid  # Wait for the specific job
-            echo "Job with PID $pid finished at $(date)" >> $log_file
-        done
-        
-        # Clear job PIDs for the next batch
-        job_pids=()
-    fi
-done
-
-# Output results of all jobs and log them
-for i in $(seq 1 $total_jobs); do
-    result=$(cat subscript_$i.output)
-    echo "Result of job $i is $result"
-    echo "Result of job $i: $result" >> $log_file
-done
-
-# Log the end of the job process
-echo "All jobs completed at $(date)" >> $log_file
-echo "End Job"
\ No newline at end of file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput
new file mode 100644
index 0000000000..1652f54226
--- /dev/null
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput
@@ -0,0 +1,7 @@
+1 5 0
+2 5 99
+3 4 0
+4 15 0
+5 6 0
+6 6 130
+7 5 0
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
index d3e6bf3278..465a464e30 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
@@ -1,5 +1,5 @@
 #!/bin/bash --login
-#SBATCH --job-name=advancedJob
+#SBATCH --job-name=dynamicJob
 #SBATCH --nodes=1
 #SBATCH --ntasks=3
 #SBATCH --qos=vcell
@@ -25,7 +25,7 @@ running_jobs=0
 for i in $(seq 1 $total_jobs); do
     # Log job start
     echo "Job $i started at $(date)" >> $log_file
-    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i &   # run each script in parallel
+    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput & # run each script in parallel
     pid=$!  # Capture the job PID
     job_pids+=($pid)  # Store the PID
     job_pid_map[$pid]=$i  # Map job index to PID
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
index 06c38aa30a..91bc193ac8 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
@@ -1,22 +1,32 @@
 #!/bin/bash
 
-# ensure the script receives the required argument
+# Ensure the script receives the required argument
 if [ -z "$1" ]; then
     echo "Error: No job index provided."
     exit 1
 fi
 
 job_index=$1
+input_file="dynamic.langevinInput"  # File containing job details
 output_file="subscript_${job_index}.output"
 
-# log job start
-echo "Job $job_index start at $(date)" > "$output_file"
+# Locate the row with the matching job index
+read duration exit_code < <(awk -v idx="$job_index" '$1 == idx {print $2, $3}' "$input_file")
 
-# simulating workload (replace with actual job logic)
-sleep $(( job_index + 1 ))  # variable sleep time to simulate different workloads
+# Ensure the job index is found
+if [ -z "$duration" ] || [ -z "$exit_code" ]; then
+    echo "Error: No matching job entry for index $job_index"
+    exit 1
+fi
+
+# Log job start
+echo "Job $job_index started at $(date)" > "$output_file"
 
-# log job completion
-echo "Job $job_index finished at $(date)" >> "$output_file"
+# Simulate workload based on parsed duration
+sleep "$duration"
 
-exit 0
+# Log job completion
+echo "Job $job_index finished at $(date) with exit code $exit_code" >> "$output_file"
 
+# Exit with the specified exit code
+exit "$exit_code"

From a2d9e73ad4bdd2e8acaa97ececc007fbf2db787b Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Tue, 3 Jun 2025 14:34:25 -0400
Subject: [PATCH 09/12] dynamic allocation of jobs, job index is 0-based

---
 .../slurm_array_poc/dynamic.langevinInput     | 17 +++--
 .../langevin/slurm_array_poc/dynamic.sub      | 70 +++++++++----------
 2 files changed, 45 insertions(+), 42 deletions(-)

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput
index 1652f54226..e15927c866 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.langevinInput
@@ -1,7 +1,10 @@
-1 5 0
-2 5 99
-3 4 0
-4 15 0
-5 6 0
-6 6 130
-7 5 0
+0 7 0
+1 9 99
+2 11 0
+3 44 0
+4 7 0
+5 8 130
+6 9 0
+7 8 0
+8 7 0
+9 6 0
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
index 465a464e30..d7757bfa53 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
@@ -1,7 +1,7 @@
 #!/bin/bash --login
 #SBATCH --job-name=dynamicJob
 #SBATCH --nodes=1
-#SBATCH --ntasks=3
+#SBATCH --ntasks=4     # use 3 for fast testing
 #SBATCH --qos=vcell
 #SBATCH --partition=vcell
 #SBATCH -o %x.stdout
@@ -10,62 +10,62 @@
 #SBATCH --time=01:00:00
 
 log_file="job_status.log"
-total_jobs=7  # Total number of jobs to run
-timeout_duration=10s  # Maximum allowed runtime for each job
-max_concurrent_jobs=$SLURM_NTASKS  # Number of jobs allowed at once
+total_jobs=10  # total number of jobs to run (use 7 for fast testing)
+timeout_duration=15s  # maximum allowed runtime for each job
+max_concurrent_jobs=$SLURM_NTASKS  # number of jobs allowed at once
 
 # Clear the log file at the start
 echo "Job Execution Log" > $log_file
 echo "------------------" >> $log_file
 
-declare -A job_pid_map  # Associative array to store job index -> PID mapping
-job_pids=()  # List to track PIDs
+declare -A job_pid_map  # associative array to store job index -> PID mapping
+job_pids=()  # list to track PIDs
 running_jobs=0
 
-for i in $(seq 1 $total_jobs); do
-    # Log job start
-    echo "Job $i started at $(date)" >> $log_file
+for i in $(seq 0 $((total_jobs - 1))); do
+    # log job start
+    echo "job $i started at $(date)" >> $log_file
     timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput & # run each script in parallel
-    pid=$!  # Capture the job PID
-    job_pids+=($pid)  # Store the PID
-    job_pid_map[$pid]=$i  # Map job index to PID
-    ((running_jobs++))  # Increment running job count
+    pid=$!  # capture the job PID
+    job_pids+=($pid)  # store the PID
+    job_pid_map[$pid]=$i  # map job index to PID
+    ((running_jobs++))  # increment running job count
 
-    # Wait for a finished job before launching a new one if we hit the concurrency limit
+    # wait for a finished job before launching a new one if we hit the concurrency limit
     while (( running_jobs >= max_concurrent_jobs )); do
         for idx in "${!job_pids[@]}"; do
             pid="${job_pids[$idx]}"
-            if ! kill -0 "$pid" 2>/dev/null; then  # Check if process is still running
-                wait "$pid"  # Ensure exit status is collected
+            if ! kill -0 "$pid" 2>/dev/null; then  # check if process is still running
+                wait "$pid"  # ensure exit status is collected
                 exit_code=$?
-                job_index=${job_pid_map[$pid]}  # Retrieve original job index
-                echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
-                unset "job_pids[$idx]"  # Remove PID from list
-                unset "job_pid_map[$pid]"  # Remove mapping
-                ((running_jobs--))  # Decrement count
-                break  # Break once we free up a slot
+                job_index=${job_pid_map[$pid]}  # retrieve original job index
+                echo "job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+                unset "job_pids[$idx]"  # remove PID from list
+                unset "job_pid_map[$pid]"  # remove mapping
+                ((running_jobs--))  # decrement count
+                break  # break once we free up a slot
             fi
         done
-        sleep 1  # Allow brief pause before rechecking
+        sleep 1  # allow brief pause before rechecking
     done
 done
 
-# Final wait for any remaining jobs
+# final wait for any remaining jobs
 for pid in "${job_pids[@]}"; do
     wait $pid
     exit_code=$?
-    job_index=${job_pid_map[$pid]}  # Retrieve original job index
-    echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+    job_index=${job_pid_map[$pid]}  # retrieve original job index
+    echo "job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
 done
-echo "Batch jobs completed at $(date)" >> $log_file
+echo "batch jobs completed at $(date)" >> $log_file
 
-echo "Starting the last job at $(date)" >> $log_file
-timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # Run last script with a timeout
+echo "starting the last job at $(date)" >> $log_file
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # run last script with a timeout
 last_pid=$!
-wait $last_pid  # Explicitly wait for last.sh to finish
-exit_code=$?  # Capture the exit code of the job
-echo "Job 'Last' with PID $last_pid finished with exit code $exit_code at $(date)" >> $log_file
-echo "Last job finished at $(date)" >> $log_file
+wait $last_pid  # explicitly wait for last.sh to finish
+exit_code=$?    # capture the exit code of the job
+echo "job 'Last' with PID $last_pid finished with exit code $exit_code at $(date)" >> $log_file
+echo "last job finished at $(date)" >> $log_file
 
-echo "All jobs completed at $(date)" >> $log_file
-echo "End Script" >> $log_file
+echo "all jobs completed at $(date)" >> $log_file
+echo "end Script" >> $log_file

From eb5645d64f7ed70bce642dfb3c206647e942a7d7 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Tue, 3 Jun 2025 14:43:54 -0400
Subject: [PATCH 10/12] dynamic allocation of jobs, cosmetic

---
 .../langevin/slurm_array_poc/dynamic.sub       | 18 +++++++++---------
 .../langevin/slurm_array_poc/subscript.sh      | 16 ++++++++--------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
index d7757bfa53..400689eb99 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
@@ -24,7 +24,7 @@ running_jobs=0
 
 for i in $(seq 0 $((total_jobs - 1))); do
     # log job start
-    echo "job $i started at $(date)" >> $log_file
+    echo "Job $i started at $(date)" >> $log_file
     timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput & # run each script in parallel
     pid=$!  # capture the job PID
     job_pids+=($pid)  # store the PID
@@ -39,7 +39,7 @@ for i in $(seq 0 $((total_jobs - 1))); do
                 wait "$pid"  # ensure exit status is collected
                 exit_code=$?
                 job_index=${job_pid_map[$pid]}  # retrieve original job index
-                echo "job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+                echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
                 unset "job_pids[$idx]"  # remove PID from list
                 unset "job_pid_map[$pid]"  # remove mapping
                 ((running_jobs--))  # decrement count
@@ -55,17 +55,17 @@ for pid in "${job_pids[@]}"; do
     wait $pid
     exit_code=$?
     job_index=${job_pid_map[$pid]}  # retrieve original job index
-    echo "job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
+    echo "Job $job_index with PID $pid finished with exit code $exit_code at $(date)" >> $log_file
 done
-echo "batch jobs completed at $(date)" >> $log_file
+echo "Batch jobs completed at $(date)" >> $log_file
 
-echo "starting the last job at $(date)" >> $log_file
+echo "Starting the last job at $(date)" >> $log_file
 timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # run last script with a timeout
 last_pid=$!
 wait $last_pid  # explicitly wait for last.sh to finish
 exit_code=$?    # capture the exit code of the job
-echo "job 'Last' with PID $last_pid finished with exit code $exit_code at $(date)" >> $log_file
-echo "last job finished at $(date)" >> $log_file
+echo "Job 'Last' with PID $last_pid finished with exit code $exit_code at $(date)" >> $log_file
+echo "The final job finished at $(date)" >> $log_file
 
-echo "all jobs completed at $(date)" >> $log_file
-echo "end Script" >> $log_file
+echo "All jobs completed at $(date)" >> $log_file
+echo "End Script" >> $log_file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
index 91bc193ac8..56af792544 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript.sh
@@ -1,32 +1,32 @@
 #!/bin/bash
 
-# Ensure the script receives the required argument
+# ensure the script receives the required argument
 if [ -z "$1" ]; then
     echo "Error: No job index provided."
     exit 1
 fi
 
 job_index=$1
-input_file="dynamic.langevinInput"  # File containing job details
+input_file="dynamic.langevinInput"  # file containing job details
 output_file="subscript_${job_index}.output"
 
-# Locate the row with the matching job index
+# locate the row with the matching job index
 read duration exit_code < <(awk -v idx="$job_index" '$1 == idx {print $2, $3}' "$input_file")
 
-# Ensure the job index is found
+# ensure the job index is found
 if [ -z "$duration" ] || [ -z "$exit_code" ]; then
     echo "Error: No matching job entry for index $job_index"
     exit 1
 fi
 
-# Log job start
+# log job start
 echo "Job $job_index started at $(date)" > "$output_file"
 
-# Simulate workload based on parsed duration
+# simulate workload based on parsed duration
 sleep "$duration"
 
-# Log job completion
+# log job completion
 echo "Job $job_index finished at $(date) with exit code $exit_code" >> "$output_file"
 
-# Exit with the specified exit code
+# exit with the specified exit code
 exit "$exit_code"

From 69d0efacaa4216a7762a272ce3185fee53b24bb6 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Tue, 3 Jun 2025 15:30:35 -0400
Subject: [PATCH 11/12] advanced.sub launches jobs in batches, and waits for a
 batch to finish before starting another batch

---
 .../langevin/slurm_array_poc/advanced.sub     | 95 +++++++------------
 .../langevin/slurm_array_poc/subscript_1.sh   |  7 --
 .../langevin/slurm_array_poc/subscript_2.sh   |  7 --
 .../langevin/slurm_array_poc/subscript_3.sh   |  7 --
 .../langevin/slurm_array_poc/subscript_4.sh   |  7 --
 .../langevin/slurm_array_poc/subscript_5.sh   |  7 --
 .../langevin/slurm_array_poc/subscript_6.sh   |  7 --
 .../langevin/slurm_array_poc/subscript_7.sh   |  7 --
 8 files changed, 35 insertions(+), 109 deletions(-)
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh
 delete mode 100644 vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_7.sh

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
index d12fd0e76c..1e1345e3f4 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
@@ -1,84 +1,59 @@
 #!/bin/bash --login
 #SBATCH --job-name=advancedJob
 #SBATCH --nodes=1
-#SBATCH --ntasks=3
+#SBATCH --ntasks=4
 #SBATCH --qos=vcell
 #SBATCH --partition=vcell
-#SBATCH -o %x.stdout	# was /home/FCAM/vasilescu/advancedJob.stdout
+#SBATCH -o %x.stdout
 #SBATCH -e %x.stderr
 #SBATCH --cpus-per-task=1
 #SBATCH --time=01:00:00
 
 log_file="job_status.log"
-total_jobs=7  # Total number of jobs to run
-timeout_duration=10s  # Maximum allowed runtime for each job
+total_jobs=10  # total number of jobs to run
+timeout_duration=15s  # maximum allowed runtime for each job
+max_concurrent_jobs=$SLURM_NTASKS  # number of jobs allowed at once
 
-# Clear the log file at the start
-echo "Job Execution Log" > $log_file
+# clear the log file at the start
+echo "job execution log" > $log_file
 echo "------------------" >> $log_file
 
-job_pids=()  # Array to track job PIDs
+declare -A job_pid_map  # associative array to store job index -> pid mapping
+job_pids=()  # list to track pids
 
-for i in $(seq 1 $total_jobs); do
-    # Log job start
-    echo "Job $i started at $(date)" >> $log_file
-    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript_$i.sh &  # Run each script with a timeout in parallel
-    job_pids+=($!)  # Store the PID of the background job
+for i in $(seq 0 $((total_jobs - 1))); do
+    # log job start
+    echo "job $i started at $(date)" >> $log_file
+    timeout $timeout_duration srun -N 1 -n 1 -c 1 ./subscript.sh $i dynamic.langevinInput &  # run each script in parallel
+    pid=$!  # capture the job pid
+    job_pids+=($pid)  # store the pid
+    job_pid_map[$pid]=$i  # map job index to pid
+
+    # wait for a batch of ntasks before launching new ones
+    if (( (i + 1) % max_concurrent_jobs == 0 || i == total_jobs - 1 )); then
+        echo "waiting for jobs to finish at $(date)" >> $log_file
 
-    # Manage batches of max_jobs
-    if (( i % SLURM_NTASKS == 0 || i == total_jobs )); then
-        echo "Waiting for jobs to finish at $(date)" >> $log_file
-        
-        # Wait for each job in the current batch
         for pid in "${job_pids[@]}"; do
             wait $pid
-            exit_code=$?  # Capture the exit code of the job
-            
-            if [ $exit_code -eq 0 ]; then
-                echo "Job with PID $pid finished successfully at $(date)" >> $log_file
-            elif [ $exit_code -eq 124 ]; then
-                echo "Job with PID $pid timed out at $(date)" >> $log_file
-            elif [ $exit_code -gt 128 ]; then
-                signal=$((exit_code - 128))
-                echo "Job with PID $pid terminated by signal $signal at $(date)" >> $log_file
-            else
-                echo "Job with PID $pid failed with exit code $exit_code at $(date)" >> $log_file
-            fi
+            exit_code=$?  # capture the exit code of the job
+            job_index=${job_pid_map[$pid]}  # retrieve original job index
+            echo "job $job_index with pid $pid finished with exit code $exit_code at $(date)" >> $log_file
         done
-        
-        # Clear job PIDs for the next batch
+
+        # clear job pids for the next batch
         job_pids=()
+        job_pid_map=()
     fi
 done
 
-# Output results of all jobs and log them
-for i in $(seq 1 $total_jobs); do
-    result=$(cat subscript_$i.output 2>/dev/null || echo "No output (job may have failed)")
-    echo "Result of job $i is $result"              # simple echo goes to stdout
-    echo "Result of job $i: $result" >> $log_file   # redirected echo goes to log_file
-done
-
-echo "Starting the last job at $(date)"
-echo "Starting the last job at $(date)" >> $log_file
-timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # Run last script with a timeout
+echo "starting the last job at $(date)" >> $log_file
+timeout $timeout_duration srun -N 1 -n 1 -c 1 ./last.sh &  # run last script with a timeout
 last_pid=$!
-wait $last_pid  # Explicitly wait for last.sh to finish
-exit_code=$?  # Capture the exit code of the job
-if [ $exit_code -eq 0 ]; then
-    echo "Job with PID $last_pid finished successfully at $(date)" >> $log_file
-elif [ $exit_code -eq 124 ]; then
-    echo "Job with PID $last_pid timed out at $(date)" >> $log_file
-elif [ $exit_code -gt 128 ]; then
-    signal=$((exit_code - 128))
-    echo "Job with PID $last_pid terminated by signal $signal at $(date)" >> $log_file
-else
-    echo "Job with PID $last_pid failed with exit code $exit_code at $(date)" >> $log_file
-fi
-echo "Last job finished at $(date)"
-echo "Last job finished at $(date)" >> $log_file
+wait $last_pid  # explicitly wait for last.sh to finish
+exit_code=$?  # capture the exit code of the job
+echo "job 'last' with pid $last_pid finished with exit code $exit_code at $(date)" >> $log_file
+echo "the final job finished at $(date)" >> $log_file
+
+echo "all jobs completed at $(date)" >> $log_file
+echo "end script" >> $log_file
 
-# Log the end of the job process
-echo "All jobs completed at $(date)"
-echo "All jobs completed at $(date)" >> $log_file
-echo "End Script"
-echo "End Script" >> $log_file
\ No newline at end of file
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh
deleted file mode 100644
index 3ded5de1f4..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_1.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo "job1 start at $(date)" > subscript_1.output
-sleep 2
-echo "job1 finished at $(date)" >> subscript_1.output
-exit 0
-
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh
deleted file mode 100644
index 066fbd60e5..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_2.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo "job2 start at $(date)" > subscript_2.output
-sleep 2
-echo "job2 finished at $(date)" >> subscript_2.output
-exit 99
-
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
deleted file mode 100644
index a4d0582914..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_3.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo "job3 start at $(date)" > subscript_3.output
-sleep 7
-echo "job3 finished at $(date)" >> subscript_3.output
-exit 0
-
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
deleted file mode 100644
index 67940ff5f0..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_4.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo "job4 start at $(date)" > subscript_4.output
-sleep 15
-echo "job4 finished at $(date)" >> subscript_4.output
-exit 0
-
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh
deleted file mode 100644
index 64a899bb49..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_5.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo "job5 start at $(date)" > subscript_5.output
-sleep 2
-echo "job5 finished at $(date)" >> subscript_5.output
-exit 0
-
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh
deleted file mode 100644
index 91988d3961..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_6.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo "job6 start at $(date)" > subscript_6.output
-sleep 2
-echo "job6 finished at $(date)" >> subscript_6.output
-exit 130
-
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_7.sh b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_7.sh
deleted file mode 100644
index d79eadd9db..0000000000
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/subscript_7.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-echo "job7 start at $(date)" > subscript_7.output
-sleep 2
-echo "job7 finished at $(date)" >> subscript_7.output
-exit 0
-

From 3af873c03f3303cc51ee5e393143433bd57ef8e9 Mon Sep 17 00:00:00 2001
From: Dan Vasilescu <vasilescu@uchc.edu>
Date: Tue, 3 Jun 2025 15:42:57 -0400
Subject: [PATCH 12/12] renaminghe log, to match the simulation name

---
 .../slurm_fixtures/langevin/slurm_array_poc/advanced.sub      | 4 ++--
 .../slurm_fixtures/langevin/slurm_array_poc/dynamic.sub       | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
index 1e1345e3f4..0e0fa8e7e3 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/advanced.sub
@@ -1,5 +1,5 @@
 #!/bin/bash --login
-#SBATCH --job-name=advancedJob
+#SBATCH --job-name=advanced
 #SBATCH --nodes=1
 #SBATCH --ntasks=4
 #SBATCH --qos=vcell
@@ -9,7 +9,7 @@
 #SBATCH --cpus-per-task=1
 #SBATCH --time=01:00:00
 
-log_file="job_status.log"
+log_file="${SLURM_JOB_NAME}.log"
 total_jobs=10  # total number of jobs to run
 timeout_duration=15s  # maximum allowed runtime for each job
 max_concurrent_jobs=$SLURM_NTASKS  # number of jobs allowed at once
diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
index 400689eb99..c61083833d 100644
--- a/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
+++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/slurm_array_poc/dynamic.sub
@@ -1,5 +1,5 @@
 #!/bin/bash --login
-#SBATCH --job-name=dynamicJob
+#SBATCH --job-name=dynamic
 #SBATCH --nodes=1
 #SBATCH --ntasks=4     # use 3 for fast testing
 #SBATCH --qos=vcell
@@ -9,7 +9,7 @@
 #SBATCH --cpus-per-task=1
 #SBATCH --time=01:00:00
 
-log_file="job_status.log"
+log_file="${SLURM_JOB_NAME}.log"
 total_jobs=10  # total number of jobs to run (use 7 for fast testing)
 timeout_duration=15s  # maximum allowed runtime for each job
 max_concurrent_jobs=$SLURM_NTASKS  # number of jobs allowed at once