44"""
55
66import datetime
7+ import logging
78import os
89import shlex
910import shutil
11+ import subprocess
1012import time
1113from typing import TYPE_CHECKING , List , Optional , Tuple
1214
@@ -161,9 +163,27 @@ def rmg_job_converged(project_directory: str) -> Tuple[bool, Optional[str]]:
161163 return rmg_converged , error
162164
163165
166+ _DEFAULT_RMG_TIMEOUT_S = 6 * 3600 # 6 hours
167+
168+ logger = logging .getLogger (__name__ )
169+
170+
171+ def _parse_walltime_to_seconds (walltime : str ) -> int :
172+ """Parse a 'DD:HH:MM:SS' walltime string to total seconds. Returns 0 for '00:00:00:00'."""
173+ parts = walltime .split (':' )
174+ if len (parts ) != 4 :
175+ return 0
176+ try :
177+ days , hours , minutes , seconds = (int (p ) for p in parts )
178+ except ValueError :
179+ return 0
180+ return days * 86400 + hours * 3600 + minutes * 60 + seconds
181+
182+
164183def run_rmg_incore (rmg_input_file_path : str ,
165184 verbose : Optional [int ] = None ,
166185 max_iterations : Optional [int ] = None ,
186+ walltime : Optional [str ] = None ,
167187 ) -> bool :
168188 """
169189 Run RMG incore under the rmg_env.
@@ -172,10 +192,14 @@ def run_rmg_incore(rmg_input_file_path: str,
172192 rmg_input_file_path (str): The path to the RMG input file.
173193 max_iterations (int, optional): Max RMG iterations.
174194 verbose (int, optional): Level of verbosity.
195+ walltime (str, optional): Max walltime in 'DD:HH:MM:SS' format. Defaults to 6 hours.
175196
176197 Returns:
177198 bool: Whether an exception was raised.
178199 """
200+ timeout_s = _parse_walltime_to_seconds (walltime ) if walltime else 0
201+ if timeout_s <= 0 :
202+ timeout_s = _DEFAULT_RMG_TIMEOUT_S
179203 project_directory = os .path .abspath (os .path .dirname (rmg_input_file_path ))
180204 verbose = f' -v { verbose } ' if verbose is not None else ''
181205 max_iterations = f' -m { max_iterations } ' if max_iterations is not None else ''
@@ -192,8 +216,16 @@ def run_rmg_incore(rmg_input_file_path: str,
192216 echo "Micromamba/Mamba/Conda required" >&2
193217 exit 1
194218fi' '''
195- stdout , stderr = execute_command (shell_script , shell = True , no_fail = True , executable = '/bin/bash' )
196- stderr_text = '' .join (stderr ) if isinstance (stderr , list ) else (stderr or '' )
219+ try :
220+ result = subprocess .run (shell_script , shell = True , executable = '/bin/bash' ,
221+ capture_output = True , text = True , timeout = timeout_s )
222+ stderr_text = result .stderr or ''
223+ except subprocess .TimeoutExpired :
224+ logger .error (f'RMG incore timed out after { timeout_s } s' )
225+ return True
226+ if result .returncode != 0 :
227+ logger .error (f'RMG incore exited with code { result .returncode } ' )
228+ return True
197229 if 'RMG threw an exception and did not converge.' in stderr_text :
198230 return True
199231 return False
@@ -271,6 +303,7 @@ def rmg_runner(rmg_input_file_path: str,
271303 t3_project_name : Optional [str ] = None ,
272304 rmg_execution_type : Optional [str ] = None ,
273305 restart_rmg : bool = False ,
306+ walltime : Optional [str ] = None ,
274307 ) -> bool :
275308 """
276309 Run an RMG job as a subprocess under the rmg_env.
@@ -286,6 +319,7 @@ def rmg_runner(rmg_input_file_path: str,
286319 t3_project_name (str, optional): The T3 project name, used for setting a job name on the server for the RMG run.
287320 rmg_execution_type (str, optional): The RMG execution type (incore or local). Also set via settings.py.
288321 restart_rmg (bool, optional): Whether to restart RMG from seed.
322+ walltime (str, optional): Max walltime in 'DD:HH:MM:SS' format. Defaults to 6 hours.
289323
290324 Returns:
291325 bool: Whether an exception was raised.
@@ -299,6 +333,7 @@ def rmg_runner(rmg_input_file_path: str,
299333 rmg_exception_encountered = run_rmg_incore (rmg_input_file_path = rmg_input_file_path ,
300334 verbose = verbose ,
301335 max_iterations = max_iterations ,
336+ walltime = walltime ,
302337 )
303338 return rmg_exception_encountered
304339 elif rmg_execution_type == 'local' :
0 commit comments