[QC-1038] fix Race condition in Service Discovery (#2024)

* [QC-1038] fix Race condition in Service Discovery

* format

* add missing include

* add missing include

Author: Barthelemy

Framework/include/QualityControl/ServiceDiscovery.h

@@ -20,6 +20,8 @@
 #include <memory>
 #include <string>
 #include <thread>
+#include <mutex>
+#include <condition_variable>
 
 typedef void CURL;
 
@@ -65,6 +67,9 @@ class ServiceDiscovery
   const std::string mId;            ///< Instance (service) ID
   std::string mHealthUrl;           ///< hostname of health check endpoint
   size_t mHealthPort;               ///< Port of the health check endpoint
+  std::mutex mHealthPortMutex;      ///< Mutex for the port
+  std::condition_variable mHealthPortCV; ///< Condition varaible for the port
+  std::atomic<bool> mHealthPortAssigned; ///< Port of the health check is ready.
   std::thread mHealthThread;        ///< Health check thread
   std::atomic<bool> mThreadRunning; ///< Health check thread running flag
 

Framework/src/ServiceDiscovery.cxx

@@ -38,6 +38,7 @@ ServiceDiscovery::ServiceDiscovery(const std::string& url, const std::string& na
   : curlHandle(initCurl(), &ServiceDiscovery::deleteCurl), mConsulUrl(url), mName(name), mId(id), mHealthUrl(healthEndUrl)
 {
   mHealthUrl = mHealthUrl.empty() ? boost::asio::ip::host_name() : mHealthUrl;
+  mHealthPortAssigned = false;
 
   mHealthThread = std::thread([=] {
 #ifdef __linux__
@@ -96,6 +97,11 @@ bool ServiceDiscovery::_register(const std::string& objects)
   check.put("Name", "Health check " + mId);
   check.put("Interval", "5s");
   check.put("DeregisterCriticalServiceAfter", "1m");
+  // Wait until port is set by the thread
+  {
+    std::unique_lock<std::mutex> lk(mHealthPortMutex);
+    mHealthPortCV.wait(lk, [this] { return mHealthPortAssigned == true; });
+  }
   check.put("TCP", mHealthUrl + ":" + std::to_string(mHealthPort));
   checks.push_back(std::make_pair("", check));
 
@@ -151,12 +157,18 @@ void ServiceDiscovery::runHealthServer()
       ILOG(Debug, Trace) << "ServiceDiscovery::runHealthServer - cound not bind to " << port << ENDM;
       continue; // try the next one
     }
-    // if we reach this point it means that we could bind
-    mHealthPort = port;
-    ILOG(Debug, Devel) << "ServiceDiscovery selected port: " << mHealthPort << ENDM;
     break;
   }
 
+  // assign the port and unblock the main thread (we got a port or we failed)
+  {
+    ILOG(Debug, Devel) << "ServiceDiscovery selected port: " << mHealthPort << ENDM;
+    std::lock_guard<std::mutex> lk(mHealthPortMutex);
+    mHealthPort = port;
+    mHealthPortAssigned = true;
+  }
+  mHealthPortCV.notify_one();
+
   if (cycle == rangeLength) {
     ILOG(Error, Support) << "Could not find a free port for the ServiceDiscovery, aborting the ServiceDiscovery health check" << ENDM;
     return;