Update templates and settings

Author: calvinp0

arc/job/pipe/pipe_coordinator.py

@@ -154,6 +154,7 @@ def submit_pipe_run(self, run_id: str, tasks: List[TaskSpec],
             tasks=tasks,
             cluster_software=cluster_software,
             max_workers=pipe_settings.get('max_workers', 100),
+            max_concurrent=pipe_settings.get('max_concurrent'),
             max_attempts=pipe_settings.get('max_attempts', 3),
             pipe_root=pipe_root,
         )

arc/settings/settings.py

@@ -306,6 +306,7 @@
     'enabled': False,          # Set to True to enable pipe mode (it is off by default, use it for large compute campaigns).
     'min_tasks': 10,           # Minimum batch size to trigger pipe mode.
     'max_workers': 100,        # Upper bound on array worker slots per PipeRun.
+    'max_concurrent': None,    # Max array workers running simultaneously (like PBS `%N`). None = unthrottled.
     'max_attempts': 3,         # Retry budget per task before terminal failure.
     'lease_duration_hrs': 1,   # Worker lease duration in hours (default 1h).
     'env_setup': {},           # Engine-specific shell setup commands, e.g.,

arc/settings/submit.py

@@ -54,10 +54,10 @@
 #SBATCH -N 1
 #SBATCH -n {cpus}
 #SBATCH --mem={memory}
-#SBATCH --array=1-{max_task_num}
+#SBATCH --array={array_range}
 #SBATCH -o {pipe_root}/out_%a.txt
 #SBATCH -e {pipe_root}/err_%a.txt
-
+{extra_directives}
 {env_setup}
 WORKER_ID=$SLURM_ARRAY_TASK_ID
 
@@ -68,8 +68,8 @@
 #PBS -q {queue}
 #PBS -l ncpus={cpus}
 #PBS -l mem={memory}mb
-#PBS -J 1-{max_task_num}
-
+#PBS -J {array_range}
+{extra_directives}
 {env_setup}
 WORKER_ID="$PBS_ARRAY_INDEX"
 
@@ -80,10 +80,10 @@
 #$ -q {queue}
 #$ -pe smp {cpus}
 #$ -l h_vmem={memory}M
-#$ -t 1-{max_task_num}
+#$ -t {array_range}
 #$ -o {pipe_root}/out_$SGE_TASK_ID.txt
 #$ -e {pipe_root}/err_$SGE_TASK_ID.txt
-
+{extra_directives}
 {env_setup}
 WORKER_ID=$SGE_TASK_ID
 
@@ -96,7 +96,8 @@
 output = {pipe_root}/out_$(Process).txt
 error = {pipe_root}/err_$(Process).txt
 log = {pipe_root}/condor.log
-queue {max_task_num}
+{extra_directives}
+queue {array_range}
 """,
 }