Merge pull request #1625 from virtualcell/dan-ss-results3

Using the default singularity on batch run node

Author: danv61

docker/build/Dockerfile-submit-dev

@@ -87,6 +87,9 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \
     slurm_singularity_cachedir="slurm_singularity_cachedir-not-set" \
     slurm_singularity_pullfolder="slurm_singularity_pullfolder-not-set" \
     slurm_singularity_module_name="slurm_singularity_module_name-not-set" \
+    slurm_langevin_timeoutPerTaskSeconds="slurm_langevin_timeoutPerTaskSeconds-not-set" \
+    slurm_langevin_batchMemoryLimitPerTaskMB="slurm_langevin_batchMemoryLimitPerTaskMB-not-set" \
+    slurm_langevin_memoryBlockSizeMB="slurm_langevin_memoryBlockSizeMB-not-set" \
     jmsblob_minsize=100000 \
     vcell_ssh_cmd_cmdtimeout="cmdSrvcSshCmdTimeoutMS-not-set" \
     vcell_ssh_cmd_restoretimeout="cmdSrvcSshCmdRestoreTimeoutFactor-not-set" \
@@ -143,6 +146,9 @@ ENTRYPOINT java \
 	-Dvcell.slurm.singularity.cachedir="${slurm_singularity_cachedir}" \
 	-Dvcell.slurm.singularity.pullfolder="${slurm_singularity_pullfolder}" \
 	-Dvcell.slurm.singularity.module.name="${slurm_singularity_module_name}" \
+	-Dvcell.slurm.langevin.timeoutPerTaskSeconds="${slurm_langevin_timeoutPerTaskSeconds}" \
+	-Dvcell.slurm.langevin.batchMemoryLimitPerTaskMB="${slurm_langevin_batchMemoryLimitPerTaskMB}" \
+	-Dvcell.slurm.langevin.memoryBlockSizeMB="${slurm_langevin_memoryBlockSizeMB}" \
 	-Dvcell.htc.vcellfvsolver.docker.name="${htc_vcellfvsolver_docker_name}" \
 	-Dvcell.htc.vcellfvsolver.solver.list="${htc_vcellfvsolver_solver_list}" \
 	-Dvcell.htc.vcellsolvers.docker.name="${htc_vcellsolvers_docker_name}" \

vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java

@@ -115,6 +115,12 @@ public static void setConfigProvider(VCellConfigProvider configProvider) {
 	public static final String MPI_HOME_EXTERNAL		        = record("vcell.htc.mpi.home",ValueType.GEN);
 	public static final String nativeSolverDir_External      = record("vcell.nativesolverdir.external",ValueType.GEN);
 
+	// specific to langevin solver when running in batch mode
+	public static final String slurm_langevin_timeoutPerTaskSeconds = record("vcell.slurm.langevin.timeoutPerTaskSeconds",ValueType.GEN);
+	public static final String slurm_langevin_batchMemoryLimitPerTaskMB = record("vcell.slurm.langevin.batchMemoryLimitPerTaskMB",ValueType.GEN);
+	public static final String slurm_langevin_memoryBlockSizeMB = record("vcell.slurm.langevin.memoryBlockSizeMB",ValueType.GEN);
+
+
 //	public static final String finiteVolumeExecutableProperty = record("vcell.finitevolume.executable",ValueType.EXE);
 //
 //	//

vcell-server/src/main/java/cbit/vcell/message/server/htc/slurm/SlurmProxy.java

@@ -523,38 +523,23 @@ private void writeSingularitySetup(LineStringBuilder lsb) {
 		String singularityPullfolder = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_singularity_pullfolder);
 		String singularityModuleName = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_singularity_module_name);
 
-		lsb.write("echo \"=== Singularity check BEFORE module load ===\"");
+		lsb.write("echo \"=== Singularity check ===\"");
 		lsb.write("if command -v singularity >/dev/null 2>&1; then");
 		lsb.write("    echo \"Singularity found at: $(command -v singularity)\"");
 		lsb.write("    singularity --version");
 		lsb.write("else");
-		lsb.write("    echo \"Singularity not found before module load\"");
+		lsb.write("    echo \"Singularity not found\"");
+		lsb.write("    exit 127");
 		lsb.write("fi");
 		lsb.write("");
 
 		lsb.write("TMPDIR=" + slurmTmpDir);
 		lsb.write("if [ ! -e $TMPDIR ]; then mkdir -p $TMPDIR ; fi");
 		lsb.write("echo `hostname`");
 		lsb.write("export MODULEPATH=/isg/shared/modulefiles:/tgcapps/modulefiles");
-		lsb.write("if [ -f /usr/share/modules/init/bash ]; then");
-		lsb.write("    source /usr/share/modules/init/bash");
-		lsb.write("    module load " + singularityModuleName);
-		lsb.write("else");
-		lsb.write("    echo \"[Warning] Module init script not found - skipping module setup\"");
-		lsb.write("fi");
 		lsb.write("export SINGULARITY_CACHEDIR=" + singularityCachedir);
 		lsb.write("export SINGULARITY_PULLFOLDER=" + singularityPullfolder);
 		lsb.write("");
-
-		lsb.write("echo \"=== Singularity check AFTER module load ===\"");
-		lsb.write("if command -v singularity >/dev/null 2>&1; then");
-		lsb.write("    echo \"Singularity found at: $(command -v singularity)\"");
-		lsb.write("    singularity --version");
-		lsb.write("else");
-		lsb.write("    echo \"Singularity not found after module load\"");
-		lsb.write("    exit 127");
-		lsb.write("fi");
-		lsb.write("");
 	}
 	private void writeSlurmJobMetadata(LineStringBuilder lsb) {
 		lsb.write("# Compute memory per task and per job");
@@ -661,10 +646,12 @@ String  generateLangevinBatchScript(String jobName, ExecutableCommand.Container
 		SolverDescription solverDescription = std.getSolverDescription();
 		MemLimitResults memoryMBAllowed = HtcProxy.getMemoryLimit(vcellUserid, simID, solverDescription, memSizeMB, simTask.isPowerUser());
 
-		// TODO: do we hardcode these? Should it be part of LangevinSimulationOptions? Or, even better, properties?
-		int timeoutPerTaskSeconds = 86400;			// seconds. 24 hours
-		long hardbBtchMemoryLimitPerTask = 1024;	// MB. we hard limit mem to 1G for langevin batch jobs
-		int blockSizeMB = 256; 						// MB. SLURM memory allocation granularity
+		String sTimeoutPerTaskSeconds = PropertyLoader.getProperty(PropertyLoader.slurm_langevin_timeoutPerTaskSeconds, "86400");
+		String sHardbBtchMemoryLimitPerTask = PropertyLoader.getProperty(PropertyLoader.slurm_langevin_batchMemoryLimitPerTaskMB, "1024");
+		String sBlockSizeMB =  PropertyLoader.getProperty(PropertyLoader.slurm_langevin_memoryBlockSizeMB, "256");
+		int timeoutPerTaskSeconds = Integer.parseInt(sTimeoutPerTaskSeconds);				// seconds. 24 hours
+		long hardbBtchMemoryLimitPerTask = Long.parseLong(sHardbBtchMemoryLimitPerTask);	// MB. we hard limit mem to 2G for langevin batch jobs
+		int blockSizeMB = Integer.parseInt(sBlockSizeMB); 						// MB. SLURM memory allocation granularity
 		String slurmJobTimeout = computeSlurmTimeLimit(totalNumberOfJobs, numberOfConcurrentTasks, timeoutPerTaskSeconds);
 		long batchMemoryLimitPerTask = memoryMBAllowed.getMemLimit();
 		batchMemoryLimitPerTask = Math.min(batchMemoryLimitPerTask, hardbBtchMemoryLimitPerTask);

vcell-server/src/test/resources/slurm_fixtures/langevin/V_TEST2_999999999_0_0.slurm.sub

@@ -36,36 +36,22 @@ rm -f /share/apps/vcell3/users/${USERID}/SimID_${SIM_ID}_0_.functions
 rm -f /share/apps/vcell3/users/${USERID}/SimID_${SIM_ID}_0_.langevinInput
 rm -f /share/apps/vcell3/users/${USERID}/SimID_${SIM_ID}_0_.langevinMessagingConfig
 
-echo "=== Singularity check BEFORE module load ==="
+echo "=== Singularity check ==="
 if command -v singularity >/dev/null 2>&1; then
     echo "Singularity found at: $(command -v singularity)"
     singularity --version
 else
-    echo "Singularity not found before module load"
+    echo "Singularity not found"
+    exit 127
 fi
 
 TMPDIR=/scratch/vcell
 if [ ! -e $TMPDIR ]; then mkdir -p $TMPDIR ; fi
 echo `hostname`
 export MODULEPATH=/isg/shared/modulefiles:/tgcapps/modulefiles
-if [ -f /usr/share/modules/init/bash ]; then
-    source /usr/share/modules/init/bash
-    module load singularity/vcell-3.10.0
-else
-    echo "[Warning] Module init script not found - skipping module setup"
-fi
 export SINGULARITY_CACHEDIR=/share/apps/vcell3/singularity/cachdir
 export SINGULARITY_PULLFOLDER=/share/apps/vcell3/singularity/pullfolder
 
-echo "=== Singularity check AFTER module load ==="
-if command -v singularity >/dev/null 2>&1; then
-    echo "Singularity found at: $(command -v singularity)"
-    singularity --version
-else
-    echo "Singularity not found after module load"
-    exit 127
-fi
-
 # Compute memory per task and per job
 MEM_TASK=$(( SLURM_MEM_PER_CPU * SLURM_CPUS_PER_TASK ))
 MEM_JOB=$(( MEM_TASK * SLURM_NTASKS ))