Make sure the ports are unique when running the multinode test (#1538)

* Make sure the ports are unique when running the multinode test

* Update o2-qc-multinode-test.sh

* update the path in curl

* do not generate randomly UNIQUE_PORT_2 anymore

Co-authored-by: Piotr Konopka <piotr.jan.konopka@cern.ch>

Author: 2 people

Framework/CMakeLists.txt

@@ -332,7 +332,9 @@ set_property(TEST functional_test PROPERTY TIMEOUT 45)
 
 include(GenerateUniquePort)
 o2_generate_unique_port(UNIQUE_PORT_1)
-o2_generate_unique_port(UNIQUE_PORT_2)
+# UNIQUE_PORT_2 is UNIQUE_PORT_1 + 1
+# since we are guaranteed UNIQUE_PORT_1 is even.
+math(EXPR UNIQUE_PORT_2 "${UNIQUE_PORT_1} + 1")
 configure_file(multinode-test.json.in ${CMAKE_BINARY_DIR}/tests/multinode-test.json)
 add_test(NAME multinode_test COMMAND o2-qc-multinode-test.sh)
 set_tests_properties(multinode_test

Framework/script/o2-qc-multinode-test.sh

@@ -21,6 +21,13 @@ for sig in INT QUIT HUP TERM; do
 done
 trap cleanup EXIT
 
+if [ -z "$UNIQUE_PORT_1" ]
+then
+  echo "UNIQUE_PORT_1 must be set when calling o2-qc-multinode-test.sh"
+  exit 1
+fi
+export UNIQUE_TEST_NAME="multinode-test-${UNIQUE_PORT_1}"
+
 function check_if_port_in_use() {
   OS=`uname`
   if [[ $OS == Linux ]] ; then
@@ -42,29 +49,21 @@ function delete_data() {
   curl -i -L ccdb-test.cern.ch:8080/truncate/qc/TST/QO/MultiNodeLocalTest
   curl -i -L ccdb-test.cern.ch:8080/truncate/qc/TST/QO/MultiNodeRemoteTest
 
-  rm -f /tmp/multinode_test_obj${UNIQUE_PORT_1}.root
-  rm -f /tmp/multinode_test_obj${UNIQUE_PORT_2}.root
+  cd /tmp
+  # mv in /tmp is guaranteed to be atomic
+  mv -f /tmp/${UNIQUE_TEST_NAME}{,.todelete}
+  rm -rf /tmp/${UNIQUE_TEST_NAME}.todelete
 }
 
-if [ -z "$UNIQUE_PORT_1" ]
-then
-  echo "UNIQUE_PORT_1 must be set when calling o2-qc-multinode-test.sh"
-  exit 1
-fi
+delete_data
+# mkdir in /tmp is guaranteed to be atomic
+mkdir /tmp/${UNIQUE_TEST_NAME} || { echo "Concurrent usage of the same port ${UNIQUE_PORT_1} detected, exiting"; exit 1; }
+pushd /tmp/${UNIQUE_TEST_NAME}
+
+UNIQUE_PORT_2=$((UNIQUE_PORT_1+1))
+
 check_if_port_in_use $UNIQUE_PORT_1
-if [ -z "$UNIQUE_PORT_2" ]
-then
-  echo "UNIQUE_PORT_2 must be set when calling o2-qc-multinode-test.sh"
-  exit 1
-fi
 check_if_port_in_use $UNIQUE_PORT_2
-if [ "$UNIQUE_PORT_1" == "$UNIQUE_PORT_2" ]
-then
-  echo "UNIQUE_PORT_1 must be different than UNIQUE_PORT_2 when calling o2-qc-multinode-test.sh"
-  echo "You were probably very unlucky to have the same port randomly selected twice."
-  echo "Just run the tests again please (or complain to the QC developers if that happens suspiciously often)."
-  exit 1
-fi
 if [ -z "$JSON_DIR" ]
 then
   echo "JSON_DIR must be set when calling o2-qc-multinode-test.sh"
@@ -80,8 +79,6 @@ else
   exit 0
 fi
 
-delete_data
-
 # store data
 o2-qc-run-producer --producers 2 --message-amount 15  --message-rate 1 -b | timeout -s INT 40s o2-qc --config json://${JSON_DIR}/multinode-test.json -b --local --host localhost --run &
 
@@ -92,21 +89,21 @@ wait
 
 # check MonitorObject
 # first the return code must be 200
-code=$(curl -L ccdb-test.cern.ch:8080/qc/TST/MO/MultiNodeLocalTest${UNIQUE_PORT_1}/example/`date +%s`999 --write-out %{http_code} --silent --output /tmp/multinode_test_obj${UNIQUE_PORT_1}.root)
+code=$(curl -L ccdb-test.cern.ch:8080/qc/TST/MO/MultiNodeLocalTest${UNIQUE_PORT_1}/example/`date +%s`999 --write-out %{http_code} --silent --output /tmp/${UNIQUE_TEST_NAME}/multinode_test_obj${UNIQUE_PORT_1}.root)
 if (( $code != 200 )); then
   echo "Error, monitor object of the local QC Task could not be found."
   delete_data
   exit 2
 fi
 # try to check that we got a valid root object
-root -b -l -q -e 'TFile f("/tmp/multinode_test_obj${UNIQUE_PORT_1}.root"); f.Print();'
+root -b -l -q -e 'TFile f("/tmp/${UNIQUE_TEST_NAME}/multinode_test_obj${UNIQUE_PORT_1}.root"); f.Print();'
 if (( $? != 0 )); then
   echo "Error, monitor object of the local QC Task is invalid."
   delete_data
   exit 2
 fi
 # try if it is a non empty histogram
-entries=`root -b -l -q -e 'TFile f("/tmp/multinode_test_obj${UNIQUE_PORT_1}.root"); TH1F *h = (TH1F*)f.Get("ccdb_object"); cout << h->GetEntries() << endl;' | tail -n 1`
+entries=`root -b -l -q -e 'TFile f("/tmp/${UNIQUE_TEST_NAME}/multinode_test_obj${UNIQUE_PORT_1}.root"); TH1F *h = (TH1F*)f.Get("ccdb_object"); cout << h->GetEntries() << endl;' | tail -n 1`
 if ! [ $entries -gt 0 ] 2>/dev/null
 then
   echo "The histogram of the local QC Task is empty or the object is not a histogram."
@@ -116,21 +113,21 @@ fi
 
 # check MonitorObject
 # first the return code must be 200
-code=$(curl -L ccdb-test.cern.ch:8080/qc/TST/MO/MultiNodeRemoteTest${UNIQUE_PORT_2}/example/`date +%s`999 --write-out %{http_code} --silent --output /tmp/multinode_test_obj${UNIQUE_PORT_2}.root)
+code=$(curl -L ccdb-test.cern.ch:8080/qc/TST/MO/MultiNodeRemoteTest${UNIQUE_PORT_2}/example/`date +%s`999 --write-out %{http_code} --silent --output /tmp/${UNIQUE_TEST_NAME}/multinode_test_obj${UNIQUE_PORT_2}.root)
 if (( $code != 200 )); then
   echo "Error, monitor object of the remote QC Task could not be found."
   delete_data
   exit 2
 fi
 # try to check that we got a valid root object
-root -b -l -q -e 'TFile f("/tmp/multinode_test_obj${UNIQUE_PORT_2}.root"); f.Print();'
+root -b -l -q -e 'TFile f("/tmp/${UNIQUE_TEST_NAME}/multinode_test_obj${UNIQUE_PORT_2}.root"); f.Print();'
 if (( $? != 0 )); then
   echo "Error, monitor object of the remote QC Task is invalid."
   delete_data
   exit 2
 fi
 # try if it is a non empty histogram
-entries=`root -b -l -q -e 'TFile f("/tmp/multinode_test_obj${UNIQUE_PORT_2}.root"); TH1F *h = (TH1F*)f.Get("ccdb_object"); cout << h->GetEntries() << endl;' | tail -n 1`
+entries=`root -b -l -q -e 'TFile f("/tmp/${UNIQUE_TEST_NAME}/multinode_test_obj${UNIQUE_PORT_2}.root"); TH1F *h = (TH1F*)f.Get("ccdb_object"); cout << h->GetEntries() << endl;' | tail -n 1`
 if ! [ $entries -gt 0 ] 2>/dev/null
 then
   echo "The histogram of the remote QC Task is empty or the object is not a histogram."

cmake/GenerateUniquePort.cmake

@@ -18,9 +18,10 @@ include_guard()
 function(o2_generate_unique_port VAR_NAME)
 
   string(RANDOM LENGTH 1 ALPHABET 345 FIRST_DIGIT)
-  string(RANDOM LENGTH 4 ALPHABET 0123456789 OTHER_DIGITS)
-  string(CONCAT ${VAR_NAME} ${FIRST_DIGIT} ${OTHER_DIGITS})
+  string(RANDOM LENGTH 3 ALPHABET 0123456789 OTHER_DIGITS)
+  string(RANDOM LENGTH 1 ALPHABET 02468 LAST_DIGIT)
+  string(CONCAT ${VAR_NAME} ${FIRST_DIGIT} ${OTHER_DIGITS} ${LAST_DIGIT})
 
   set(${VAR_NAME} ${${VAR_NAME}} PARENT_SCOPE)
 
-endfunction()
+endfunction()