Stabilize CCDB tests and update API (QC-221) (#237)

* remove getListOfTasksWithPublications
* fix tests

Author: Barthelemy

Framework/CMakeLists.txt

@@ -246,16 +246,14 @@ foreach(t testTaskInterface testWorkflow testTaskRunner
   target_include_directories(${t} PRIVATE ${CMAKE_SOURCE_DIR})
 endforeach()
 
-set_property(TEST testTaskInterface PROPERTY LABELS slow)
-set_property(TEST testWorkflow PROPERTY TIMEOUT 120)
+set_property(TEST testWorkflow PROPERTY TIMEOUT 10)
 set_property(TEST testWorkflow PROPERTY LABELS slow)
-set_property(TEST testObjectsManager PROPERTY TIMEOUT 120)
+set_property(TEST testObjectsManager PROPERTY TIMEOUT 60)
 set_property(TEST testObjectsManager PROPERTY LABELS slow)
+set_property(TEST testCcdbDatabase PROPERTY TIMEOUT 60)
+set_property(TEST testCcdbDatabase PROPERTY LABELS slow)
 # Temporarily disable tests that fail on CI
 set_property(TEST testWorkflow PROPERTY LABELS manual)
-set_property(TEST testCcdbDatabaseExtra PROPERTY LABELS manual)
-set_property(TEST testCcdbDatabase PROPERTY LABELS manual)
-set_property(TEST testDbFactory PROPERTY LABELS manual)
 
 # ---- Install ----
 

Framework/include/QualityControl/CcdbDatabase.h

@@ -56,16 +56,21 @@ class CcdbDatabase : public DatabaseInterface
   void connect(std::string host, std::string database, std::string username, std::string password) override;
   void connect(const std::unordered_map<std::string, std::string>& config) override;
   void store(std::shared_ptr<o2::quality_control::core::MonitorObject> mo) override;
-  core::MonitorObject* retrieve(std::string taskName, std::string objectName, long timestamp = 0) override;
-  std::string retrieveJson(std::string taskName, std::string objectName) override;
+  core::MonitorObject* retrieve(std::string path, std::string objectName, long timestamp = 0) override;
+  std::string retrieveJson(std::string path, std::string objectName) override;
   void disconnect() override;
   void prepareTaskDataContainer(std::string taskName) override;
-  std::vector<std::string> getListOfTasksWithPublications() override;
   std::vector<std::string> getPublishedObjectNames(std::string taskName) override;
   void truncate(std::string taskName, std::string objectName) override;
   void storeStreamerInfosToFile(std::string filename);
   static long getCurrentTimestamp();
   static long getFutureTimestamp(int secondsInFuture);
+  /**
+  * Return the listing of folder and/or objects in the subpath.
+  * @param subpath The folder we want to list the children of.
+  * @return The listing of folder and/or objects at the subpath.
+  */
+  std::vector<std::string> getListing(std::string subpath = "");
 
  private:
   /**
@@ -83,7 +88,7 @@ class CcdbDatabase : public DatabaseInterface
    * @param accept The format of the returned string as an \"Accept\", i.e. text/plain, application/json, text/xml
    * @return The listing of folder and/or objects in the format requested and as returned by the http server.
    */
-  std::string getListing(std::string subpath = "", std::string accept = "text/plain");
+  std::string getListingAsString(std::string subpath = "", std::string accept = "text/plain");
   o2::ccdb::CcdbApi ccdbApi;
   std::string mUrl = "";
 };

Framework/include/QualityControl/DatabaseInterface.h

@@ -79,7 +79,6 @@ class DatabaseInterface
    * the given Task. If the container already exists, we do nothing.
    */
   virtual void prepareTaskDataContainer(std::string taskName) = 0;
-  virtual std::vector<std::string> getListOfTasksWithPublications() = 0;
   virtual std::vector<std::string> getPublishedObjectNames(std::string taskName) = 0;
   /**
    * Delete all versions of a given object

Framework/include/QualityControl/MySqlDatabase.h

@@ -44,7 +44,7 @@ class MySqlDatabase : public DatabaseInterface
   std::string retrieveJson(std::string taskName, std::string objectName) override;
   void disconnect() override;
   std::vector<std::string> getPublishedObjectNames(std::string taskName) override;
-  std::vector<std::string> getListOfTasksWithPublications() override;
+  std::vector<std::string> getListOfTasksWithPublications();
   void truncate(std::string taskName, std::string objectName) override;
 
  private:

Framework/src/CcdbDatabase.cxx

@@ -30,6 +30,7 @@
 #include <sstream>
 
 #include <fairlogger/Logger.h>
+#include <boost/algorithm/string.hpp>
 
 using namespace std::chrono;
 using namespace AliceO2::Common;
@@ -114,32 +115,32 @@ void CcdbDatabase::store(std::shared_ptr<o2::quality_control::core::MonitorObjec
   ccdbApi.storeAsTFile(mo.get(), path, metadata, from, to);
 }
 
-core::MonitorObject* CcdbDatabase::retrieve(std::string taskName, std::string objectName, long timestamp)
+core::MonitorObject* CcdbDatabase::retrieve(std::string path, std::string objectName, long timestamp)
 {
-  string path = taskName + "/" + objectName;
+  string fullPath = path + "/" + objectName;
   map<string, string> metadata;
   long when = timestamp == 0 ? getCurrentTimestamp() : timestamp;
 
   // we try first to load a TFile
-  TObject* object = ccdbApi.retrieveFromTFile(path, metadata, when);
+  TObject* object = ccdbApi.retrieveFromTFile(fullPath, metadata, when);
   if (object == nullptr) {
     // We could not open a TFile we should now try to open an object directly serialized
-    object = ccdbApi.retrieve(path, metadata, when);
-    LOG(DEBUG) << "We could retrieve the object " << path << " as a streamed object.";
+    object = ccdbApi.retrieve(fullPath, metadata, when);
+    LOG(DEBUG) << "We could retrieve the object " << fullPath << " as a streamed object.";
     if (object == nullptr) {
       return nullptr;
     }
   }
   auto* mo = dynamic_cast<core::MonitorObject*>(object);
   if (mo == nullptr) {
-    LOG(ERROR) << "Could not cast the object " << taskName << "/" << objectName << " to MonitorObject";
+    LOG(ERROR) << "Could not cast the object " << fullPath << " to MonitorObject";
   }
   return mo;
 }
 
-std::string CcdbDatabase::retrieveJson(std::string taskName, std::string objectName)
+std::string CcdbDatabase::retrieveJson(std::string path, std::string objectName)
 {
-  std::unique_ptr<core::MonitorObject> monitor(retrieve(taskName, objectName));
+  std::unique_ptr<core::MonitorObject> monitor(retrieve(path, objectName));
   if (monitor == nullptr) {
     return std::string();
   }
@@ -159,9 +160,9 @@ void CcdbDatabase::prepareTaskDataContainer(std::string /*taskName*/)
   // NOOP for CCDB
 }
 
-std::string CcdbDatabase::getListing(std::string path, std::string accept)
+std::string CcdbDatabase::getListingAsString(std::string subpath, std::string accept)
 {
-  std::string tempString = ccdbApi.list(path, false, accept);
+  std::string tempString = ccdbApi.list(subpath, false, accept);
 
   return tempString;
 }
@@ -180,12 +181,12 @@ static inline void rtrim(std::string& s)
   s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
 }
 
-std::vector<std::string> CcdbDatabase::getListOfTasksWithPublications()
+std::vector<std::string> CcdbDatabase::getListing(std::string subpath)
 {
   std::vector<string> result;
 
-  // Get the listing from CCDB
-  string listing = getListing();
+  // Get the listing from CCDB (folder qc)
+  string listing = getListingAsString(subpath);
 
   // Split the string we received, by line. Also trim it and remove empty lines.
   std::stringstream ss(listing);
@@ -210,11 +211,12 @@ std::vector<std::string> CcdbDatabase::getPublishedObjectNames(std::string taskN
   // Split the string we received, by line. Also trim it and remove empty lines. Select the lines starting with "path".
   std::stringstream ss(listing);
   std::string line;
+  std::string taskNameEscaped = boost::replace_all_copy(taskName, "/", "\\/");
   while (std::getline(ss, line, '\n')) {
     ltrim(line);
     rtrim(line);
     if (line.length() > 0 && line.find("\"path\"") == 0) {
-      unsigned long objNameStart = 9 + taskName.length();
+      unsigned long objNameStart = 9 + taskNameEscaped.length();
       string path = line.substr(objNameStart, line.length() - 2 /*final 2 char*/ - objNameStart);
       result.push_back(path);
     }

Framework/test/testCcdbDatabase.cxx

@@ -68,7 +68,8 @@ BOOST_AUTO_TEST_CASE(ccdb_getobjects_name)
 {
   test_fixture f;
 
-  auto tasks = f.backend->getListOfTasksWithPublications();
+  CcdbDatabase* ccdb = static_cast<CcdbDatabase*>(f.backend.get());
+  auto tasks = ccdb->getListing();
   for (auto& task : tasks) {
     auto objects = f.backend->getPublishedObjectNames(task);
     for (auto& object : objects) {
@@ -84,15 +85,15 @@ BOOST_AUTO_TEST_CASE(ccdb_store)
   test_fixture f;
   TH1F* h1 = new TH1F("asdf/asdf", "asdf", 100, 0, 99);
   h1->FillRandom("gaus", 10000);
-  shared_ptr<MonitorObject> mo1 = make_shared<MonitorObject>(h1, "my/task");
+  shared_ptr<MonitorObject> mo1 = make_shared<MonitorObject>(h1, "my/task", "TST");
   oldTimestamp = CcdbDatabase::getCurrentTimestamp();
   f.backend->store(mo1);
 }
 
 BOOST_AUTO_TEST_CASE(ccdb_retrieve, *utf::depends_on("ccdb_store"))
 {
   test_fixture f;
-  MonitorObject* mo = f.backend->retrieve("my/task", "asdf/asdf");
+  MonitorObject* mo = f.backend->retrieve("qc/TST/my/task", "asdf/asdf");
   BOOST_CHECK_NE(mo, nullptr);
   TH1F* h1 = dynamic_cast<TH1F*>(mo->getObject());
   BOOST_CHECK_NE(h1, nullptr);
@@ -105,25 +106,25 @@ BOOST_AUTO_TEST_CASE(ccdb_retrieve_former_versions, *utf::depends_on("ccdb_store
   test_fixture f;
   TH1F* h1 = new TH1F("asdf/asdf", "asdf", 100, 0, 99);
   h1->FillRandom("gaus", 10001);
-  shared_ptr<MonitorObject> mo1 = make_shared<MonitorObject>(h1, "my/task");
+  shared_ptr<MonitorObject> mo1 = make_shared<MonitorObject>(h1, "my/task", "TST");
   f.backend->store(mo1);
 
   // Retrieve old object stored at timestampStorage
-  MonitorObject* mo = f.backend->retrieve("my/task", "asdf/asdf", oldTimestamp);
+  MonitorObject* mo = f.backend->retrieve("qc/TST/my/task", "asdf/asdf", oldTimestamp);
   BOOST_CHECK_NE(mo, nullptr);
   TH1F* old = dynamic_cast<TH1F*>(mo->getObject());
   BOOST_CHECK_NE(old, nullptr);
   BOOST_CHECK_EQUAL(old->GetEntries(), 10000);
 
   // Retrieve latest object with timestamp
-  MonitorObject* mo2 = f.backend->retrieve("my/task", "asdf/asdf", CcdbDatabase::getCurrentTimestamp());
+  MonitorObject* mo2 = f.backend->retrieve("qc/TST/my/task", "asdf/asdf", CcdbDatabase::getCurrentTimestamp());
   BOOST_CHECK_NE(mo2, nullptr);
   TH1F* latest = dynamic_cast<TH1F*>(mo2->getObject());
   BOOST_CHECK_NE(latest, nullptr);
   BOOST_CHECK_EQUAL(latest->GetEntries(), 10001);
 
   // Retrieve latest object without timetsamp
-  MonitorObject* mo3 = f.backend->retrieve("my/task", "asdf/asdf");
+  MonitorObject* mo3 = f.backend->retrieve("qc/TST/my/task", "asdf/asdf");
   BOOST_CHECK_NE(mo3, nullptr);
   TH1F* latest2 = dynamic_cast<TH1F*>(mo3->getObject());
   BOOST_CHECK_NE(latest2, nullptr);