Allowing a timestamp in output-dir-eos (#492)

* Allowing a timestamp in output-dir-eos

* Do not check if EOS output directory exists when running analysis

Author: kjvbrt

examples/FCCee/higgs/mH-recoil/mumu/analysis_stage1_batch.py

@@ -3,6 +3,7 @@
 This analysis stage runs on HTCondor.
 '''
 from argparse import ArgumentParser
+from string import Template
 
 
 # Mandatory: Analysis class where the user defines the operations on the
@@ -52,9 +53,15 @@ def __init__(self, cmdline_args):
         # is 'group_u_FCC.local_gen'
         self.comp_group = 'group_u_FCC.local_gen'
 
-        # Optional: output directory on eos, if specified files will be copied
+        # Optional: output directory on EOS, if specified files will be copied
         # there once the batch job is done, default is empty
         self.output_dir_eos = '/eos/user/j/jsmiesko/mH-recoil-output'
+        # or it could also be a template:
+        # self.output_dir_eos = Template(
+        #     '/eos/user/j/jsmiesko/mH-recoil-output/$timestamp'
+        # )
+        # variable $timestamp is internally defined to have this form:
+        # '%Y-%m-%d_%H-%M-%S'
 
         # Optional: type of EOS proxy used when <outputDirEos> is specified.
         # The default is eosuser

man/man7/fccanalysis-script.7

@@ -164,10 +164,12 @@ Computing account when running on HTCondor.
 Default value: "group_u_FCC.local_gen"
 .TP
 \fBoutput_dir_eos\fR (optional)
-Output directory on EOS, if specified files will be copied there once the batch
-job is done.
+Output directory on EOS, recognized only when submitting to HTCondor. If
+specified files will be copied there once the batch job is done. A string
+(\fBstr\fR) or a template (\fBstring.Template\fR) can be provided. In case of
+the template only one variable is currently supported: \fB$timestamp\fR.
 .br
-Default value: empty string
+Default value: None
 .TP
 \fBeos_type\fR (optional)
 Type of the EOS proxy to be used.
@@ -177,7 +179,7 @@ Default value: user
 \fBtest_file\fR (optional)
 Location of the test file provided as a string (\fBstr\fR) or a template
 (\fBstring.Template\fR). In case of the template two variables are supported:
-\fBkey4hep_os\fR and \fBkey4hep_stack\fR.
+\fB$key4hep_os\fR and \fB$key4hep_stack\fR.
 .br
 Default value: empty string
 .TP

python/batch.py

@@ -9,6 +9,7 @@
 import subprocess
 import datetime
 import argparse
+import string
 from typing import Any
 
 from process import get_process_info
@@ -53,15 +54,16 @@ def determine_os(fccana_dir: str) -> str | None:
 def create_condor_config(config: dict[str, Any],
                          batch_dir: str,
                          sample_name: str,
-                         subjob_scripts: list[str]) -> str:
+                         subjob_scripts: list[str],
+                         output_dir_eos: str | None) -> str:
     '''
     Creates contents of HTCondor submit description file.
     '''
     cfg = 'executable = $(scriptfile)\n'
 
     cfg += f'log = {batch_dir}/condor_job.{sample_name}.$(ClusterId).log\n'
 
-    if config['output-dir-eos'] is None:
+    if output_dir_eos is None:
         cfg += f'output = {config["output-dir"]}/log/{sample_name}/'
         cfg += f'condor_job.{sample_name}.$(ClusterId).$(ProcId).out\n'
         cfg += f'error = {config["output-dir"]}/log/{sample_name}/'
@@ -95,13 +97,13 @@ def create_condor_config(config: dict[str, Any],
     cfg += 'when_to_transfer_output = on_exit\n'
     cfg += f'transfer_output_files = {sample_name}\n'
 
-    if config['output-dir-eos'] is None:
+    if output_dir_eos is None:
         cfg += 'transfer_output_remaps = '
         cfg += f'"{sample_name}={config["output-dir"]}/{sample_name}"\n'
     else:
         cfg += 'output_destination = '
         cfg += f'root://{config["eos-type"]}.cern.ch/'
-        cfg += f'{config["output-dir-eos"]}\n'
+        cfg += f'{output_dir_eos}\n'
         cfg += 'MY.XRDCP_CREATE_DIR = True\n\n'
 
     # Add user batch configuration if any.
@@ -292,11 +294,11 @@ def send_sample(config: dict[str, Any],
     '''
     sample_dict = config['sample-list'][sample_name]
 
-    # Create log directory
-    current_date = datetime.datetime.fromtimestamp(
+    timestamp = datetime.datetime.fromtimestamp(
         datetime.datetime.now().timestamp()).strftime('%Y-%m-%d_%H-%M-%S')
-    batch_dir = os.path.join('batch-submission-files',
-                             current_date, sample_name)
+
+    # Create log directory
+    batch_dir = os.path.join('batch-submission-files', timestamp, sample_name)
     if not os.path.exists(batch_dir):
         os.system(f'mkdir -p {batch_dir}')
 
@@ -373,21 +375,24 @@ def send_sample(config: dict[str, Any],
 
     condor_config_path = f'{batch_dir}/job_desc_{sample_name}.cfg'
 
-    for i in range(3):
-        try:
-            with open(condor_config_path, 'w', encoding='utf-8') as cfgfile:
-                condor_config = create_condor_config(config,
-                                                     batch_dir,
-                                                     sample_name,
-                                                     subjob_scripts)
-                cfgfile.write(condor_config)
-        except IOError as err:
-            LOGGER.warning('I/O error(%i): %s', err.errno, err.strerror)
-            if i == 2:
-                sys.exit(3)
-        else:
-            break
-        time.sleep(10)
+    # Convert possible output-dir-eos template to string
+    if isinstance(config['output-dir-eos'], string.Template):
+        output_dir_eos = config['output-dir-eos'].substitute(
+            timestamp=timestamp)
+    else:
+        output_dir_eos = config['output-dir-eos']
+
+    # Check if EOS output directory exist and if not create it
+    if output_dir_eos is not None and not os.path.exists(output_dir_eos):
+        os.system(f'mkdir -p {output_dir_eos}')
+
+    with open(condor_config_path, 'w', encoding='utf-8') as cfgfile:
+        condor_config = create_condor_config(config,
+                                             batch_dir,
+                                             sample_name,
+                                             subjob_scripts,
+                                             output_dir_eos)
+        cfgfile.write(condor_config)
 
     if config['submission-filesystem-type'] == 'eos':
         batch_cmd = f'condor_submit -spool {condor_config_path}'

python/run_fccanalysis.py

@@ -477,11 +477,6 @@ def run_fccanalysis(args, analysis_module):
     if output_dir is not None and not os.path.exists(output_dir):
         os.system(f'mkdir -p {output_dir}')
 
-    # Check if EOS output directory exist and if not create it
-    output_dir_eos = get_attribute(analysis, 'output_dir_eos', None)
-    if output_dir_eos is not None and not os.path.exists(output_dir_eos):
-        os.system(f'mkdir -p {output_dir_eos}')
-
     if config['do-weighted']:
         LOGGER.info('Using generator weights...')