Generalized use of some slurm batch langevin properties on cluent and server, make them required

Author: danv61

vcell-client/src/main/java/cbit/vcell/solver/ode/gui/SimulationSummaryPanel.java

@@ -358,7 +358,7 @@ private void displayTask() {
 			getJLabelAbsTol().setEnabled(true);
 			getJLabelAbsTolValue().setText(lso.getIntervalImage() + "s");
 			// default task timeout to 7 days, old default was 86400 s (1 day), but that is too short for some of the larger models
-			String sTimeoutPerTaskSeconds = PropertyLoader.getProperty(PropertyLoader.slurm_langevin_timeoutPerTaskSeconds, "604800");
+			String sTimeoutPerTaskSeconds = PropertyLoader.getProperty(PropertyLoader.slurm_langevin_timeoutPerTaskSeconds, "345600");
 			long value = Long.parseLong(sTimeoutPerTaskSeconds);
 			String str = DurationFormatUtils.formatDuration(value * 1000, "d'd' H'h' m'm' s's'");
 			getJLabel10().setText("Timeout per task ");

vcell-server/src/main/java/cbit/vcell/message/server/batch/sim/HtcSimulationWorker.java

@@ -724,7 +724,10 @@ public static void main(String[] args) throws IOException {
 		PropertyLoader.htcMinMemoryMB,
 		PropertyLoader.htcMaxMemoryMB,
 		PropertyLoader.htcPowerUserMemoryFloorMB,
-		PropertyLoader.htcPowerUserMemoryMaxMB
+		PropertyLoader.htcPowerUserMemoryMaxMB,
+		PropertyLoader.slurm_langevin_timeoutPerTaskSeconds,
+		PropertyLoader.slurm_langevin_batchMemoryLimitPerTaskMB,
+		PropertyLoader.slurm_langevin_memoryBlockSizeMB
 	};
 
 

vcell-server/src/main/java/cbit/vcell/message/server/htc/slurm/SlurmProxy.java

@@ -489,9 +489,7 @@ public static String computeSlurmTimeLimit(int totalNumberOfJobs,
 		int perTaskMinutes = (timeoutSeconds + 59) / 60; // ceiling(timeoutSeconds/60)
 		int batches = (totalNumberOfJobs + numberOfConcurrentTasks - 1) / numberOfConcurrentTasks;
 		long workMinutes = (long) batches * perTaskMinutes;
-		long extraMinutes = 3L * perTaskMinutes;
-		long totalMinutes = workMinutes + extraMinutes;
-		long cushionedMinutes = (long) Math.ceil(totalMinutes * 1.10);
+		long cushionedMinutes = (long) Math.ceil(workMinutes * 1.20);
 
 		long totalHours = cushionedMinutes / 60;
 		long minutes = cushionedMinutes % 60;
@@ -500,6 +498,9 @@ public static String computeSlurmTimeLimit(int totalNumberOfJobs,
 			return String.format("%02d:%02d:00", totalHours, minutes);
 		} else {
 			long days = totalHours / 24;
+			if(days >= 21) {
+				return("20-23:59:00");	// maxwall for vcell is 21-00:00:00
+			}
 			long hours = totalHours % 24;
 			return String.format("%d-%02d:%02d:00", days, hours, minutes);
 		}
@@ -665,9 +666,10 @@ String  generateLangevinBatchScript(String jobName, ExecutableCommand.Container
 		SolverDescription solverDescription = std.getSolverDescription();
 		MemLimitResults memoryMBAllowed = HtcProxy.getMemoryLimit(vcellUserid, simID, solverDescription, memSizeMB, simTask.isPowerUser());
 
-		String sTimeoutPerTaskSeconds = PropertyLoader.getProperty(PropertyLoader.slurm_langevin_timeoutPerTaskSeconds, "604800");	// seconds. 7 days
-		String sHardbBtchMemoryLimitPerTask = PropertyLoader.getProperty(PropertyLoader.slurm_langevin_batchMemoryLimitPerTaskMB, "1024");
-		String sBlockSizeMB =  PropertyLoader.getProperty(PropertyLoader.slurm_langevin_memoryBlockSizeMB, "256");
+		// next 3 will fire exception if prop not set
+		String sTimeoutPerTaskSeconds = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_langevin_timeoutPerTaskSeconds);	// seconds. 7 days
+		String sHardbBtchMemoryLimitPerTask = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_langevin_batchMemoryLimitPerTaskMB);
+		String sBlockSizeMB =  PropertyLoader.getRequiredProperty(PropertyLoader.slurm_langevin_memoryBlockSizeMB);
 		int timeoutPerTaskSeconds = Integer.parseInt(sTimeoutPerTaskSeconds);				// seconds. 24 hours
 		long hardbBtchMemoryLimitPerTask = Long.parseLong(sHardbBtchMemoryLimitPerTask);	// MB. we hard limit mem to 2G for langevin batch jobs
 		int blockSizeMB = Integer.parseInt(sBlockSizeMB); 						// MB. SLURM memory allocation granularity

vcell-server/src/test/java/cbit/vcell/message/server/htc/slurm/SlurmProxyTest.java

@@ -90,6 +90,10 @@ public void setup()
 		setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, "51200");
 		setProperty(PropertyLoader.htcMinMemoryMB, "4096");
 		setProperty(PropertyLoader.htcMaxMemoryMB, "81920");
+
+		setProperty(PropertyLoader.slurm_langevin_timeoutPerTaskSeconds, "345600");
+		setProperty(PropertyLoader.slurm_langevin_batchMemoryLimitPerTaskMB, "1024");
+		setProperty(PropertyLoader.slurm_langevin_memoryBlockSizeMB, "256");
 	}
 
 	@AfterEach

vcell-server/src/test/resources/slurm_fixtures/langevin/V_TEST2_999999999_0_0.slurm.sub

@@ -9,7 +9,7 @@
 #SBATCH --cpus-per-task=1
 #SBATCH --mem-per-cpu=4096M
 #SBATCH --nodes=1
-#SBATCH --time=6-14:24:00		# timeout for the entire job
+#SBATCH --time=14-09:36:00		# timeout for the entire job
 #SBATCH --nodelist=mantis-040
 #SBATCH --no-kill
 #SBATCH --no-requeue
@@ -23,7 +23,7 @@ set +e
 USERID=danv
 SIM_ID=999999999
 TOTAL_JOBS=8            # to be set by generator to lso.getTotalNumberOfJobs()
-JOB_TIMEOUT_SECONDS=86400  # per-job timeout (seconds), adjust per generator
+JOB_TIMEOUT_SECONDS=345600  # per-job timeout (seconds), adjust per generator
 LOG_FILE="/share/apps/vcell3/htclogs/V_TEST2_999999999_0_.submit.log"
 MESSAGING_CONFIG_FILE="/share/apps/vcell3/users/danv/SimID_999999999_0_.langevinMessagingConfig"