First implementation of parallel testing

Author: jackal1-66

MC/config/PWGLF/ini/GeneratorLF_Coalescence2.ini

@@ -0,0 +1,9 @@
+#---> GeneratorLF_Coalescence.ini
+[GeneratorExternal]
+fileName = ${O2DPG_MC_CONFIG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_coalescence.C
+funcName = generateCoalescence({1000010030, 1000020030, 1010010030}, 1, 0.239)
+
+[GeneratorPythia8]
+config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg
+
+ 

test/run_generator_tests.sh

@@ -46,6 +46,9 @@ TEST_COUNTER=1
 # whether or not to delete everything except logs (default is to delete)
 KEEP_ONLY_LOGS=1
 
+# Number of workers for parallel test runs
+NWORKERS=${NWORKERS:-8}
+
 # Prepare some colored output
 SRED="\033[0;31m"
 SGREEN="\033[0;32m"
@@ -132,7 +135,7 @@ exec_test()
     [[ "${RET}" != "0" ]] && { remove_artifacts ; return ${RET} ; }
     # run the simulation, fail if not successful
     echo "### Testing base o2-sim executable ###" >> ${LOG_FILE_SIM}
-    o2-sim -g ${generator_lower} ${trigger} --noGeant -n ${nev} -j 4 --configFile ${ini_path} --configKeyValues "GeneratorPythia8.includePartonEvent=true" >> ${LOG_FILE_SIM} 2>&1
+    o2-sim -g ${generator_lower} ${trigger} --noGeant -n ${nev} -j 1 --configFile ${ini_path} --configKeyValues "GeneratorPythia8.includePartonEvent=true" >> ${LOG_FILE_SIM} 2>&1
     RET=${?}
     [[ "${RET}" != "0" ]] && { remove_artifacts ; return ${RET} ; }
 
@@ -152,6 +155,14 @@ exec_test()
 }
 
 
+wait_for_slot()
+{
+    # Wait until the number of background jobs is less than NWORKERS
+    while (( $(jobs -r | wc -l) >= NWORKERS )) ; do
+        sleep 0.1
+    done
+}
+
 check_generators()
 {
     # check all possible generators incorporated in the INI file
@@ -169,24 +180,31 @@ check_generators()
             local look_for=$(grep " ${g}.*\(\)" ${test_script})
             local has_trigger="$(grep Trigger${g} ${ini_path})"
             [[ -z "${look_for}" ]] && continue
-            echo -n "Test ${TEST_COUNTER}: ${ini_path} with generator ${g}"
+            echo "Test ${TEST_COUNTER}: ${ini_path} with generator ${g} - STARTED"
             tested_any=1
             # prepare the test directory
             local test_dir=${TEST_COUNTER}_$(basename ${ini})_${g}_dir
             rm -rf ${test_dir} 2> /dev/null
             mkdir ${test_dir}
-            pushd ${test_dir} > /dev/null
-                # one single test
-                exec_test ${ini_path} ${g} ${has_trigger}
-                RET=${?}
-            popd > /dev/null
-            if [[ "${RET}" != "0" ]] ; then
-                echo_red " -> FAILED"
-                ret_this=${RET}
-            else
-                echo_green " -> PASSED"
-            fi
+            local test_num=${TEST_COUNTER}
             ((TEST_COUNTER++))
+            
+            # Wait for an available slot before starting a new test
+            wait_for_slot
+            
+            # Run test in background
+            (
+                cd ${test_dir}
+                exec_test ${ini_path} ${g} ${has_trigger}
+                exit $?
+            ) &
+            local pid=$!
+            
+            # Store test information in global arrays
+            test_pids+=(${pid})
+            test_numbers+=(${test_num})
+            test_generators+=(${g})
+            test_ini_paths+=("${ini_path}")
         fi
     done
     [[ -z "${tested_any}" ]] && { echo_red "No test scenario was found for any generator. There must be at least one generator to be tested." ; ret_this=1 ; }
@@ -469,6 +487,12 @@ pushd ${TEST_PARENT_DIR} > /dev/null
 # global return code to be returned at the end
 ret_global=0
 
+# Global arrays to track all test jobs (across all INI files)
+declare -a test_pids
+declare -a test_numbers
+declare -a test_generators
+declare -a test_ini_paths
+
 # check each of the INI files
 for ini in ${ini_files_full_paths} ; do
 
@@ -495,6 +519,32 @@ for ini in ${ini_files_full_paths} ; do
         [[ "${fail_immediately}" == "1" ]] && break
     fi
 done
+
+# Wait for all test jobs to complete and collect results
+idx=0
+for pid in "${test_pids[@]}" ; do
+    wait ${pid}
+    RET=$?
+    test_num="${test_numbers[$idx]}"
+    generator="${test_generators[$idx]}"
+    ini_path="${test_ini_paths[$idx]}"
+    
+    if [[ "${RET}" != "0" ]] ; then
+        echo_red "Test ${test_num}: ${ini_path} with generator ${generator} -> FAILED"
+        ret_global=${RET}
+        if [[ "${fail_immediately}" == "1" ]] ; then
+            # Kill remaining background jobs
+            for remaining_pid in "${test_pids[@]}" ; do
+                kill ${remaining_pid} 2>/dev/null
+            done
+            break
+        fi
+    else
+        echo_green "Test ${test_num}: ${ini_path} with generator ${generator} -> PASSED"
+    fi
+    ((idx++))
+done
+
 # return to where we came from
 popd > /dev/null