Skip to content

Commit 826bd42

Browse files
BarthelemyBarthelemy
authored
Unify retrieve ccdb functions (#334)
* first attempt, unfinished * working proto * some tests * comment * format * No default param in virtual methods for thew new methods in DatabaseInterface. Default timestamp is -1, not 0. Add doxygen. Fix trending task test broken by recent changes. OGUI-367 * fix merge * fix merge * clean memory * format
1 parent 0a872f7 commit 826bd42

10 files changed

Lines changed: 193 additions & 55 deletions

File tree

‎Framework/include/QualityControl/CcdbDatabase.h‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,23 @@ class CcdbDatabase : public DatabaseInterface
5555

5656
void connect(std::string host, std::string database, std::string username, std::string password) override;
5757
void connect(const std::unordered_map<std::string, std::string>& config) override;
58-
// MonitorObject
58+
59+
// storage
5960
void storeMO(std::shared_ptr<o2::quality_control::core::MonitorObject> q) override;
61+
void storeQO(std::shared_ptr<o2::quality_control::core::QualityObject> q) override;
62+
63+
// retrieval - MO
6064
std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string taskName, std::string objectName, long timestamp = 0) override;
6165
std::string retrieveMOJson(std::string taskName, std::string objectName, long timestamp = 0) override;
62-
// QualityObject
63-
void storeQO(std::shared_ptr<o2::quality_control::core::QualityObject> q) override;
66+
67+
// retrieval - QO
6468
std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = 0) override;
6569
std::string retrieveQOJson(std::string qoPath, long timestamp = 0) override;
6670

71+
// retrieval - general
72+
std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp = -1) override;
73+
std::string retrieveJson(std::string path, long timestamp = -1) override;
74+
6775
void disconnect() override;
6876
void prepareTaskDataContainer(std::string taskName) override;
6977
std::vector<std::string> getPublishedObjectNames(std::string taskName) override;

‎Framework/include/QualityControl/DatabaseInterface.h‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,20 +68,65 @@ class DatabaseInterface
6868
virtual void storeQO(std::shared_ptr<o2::quality_control::core::QualityObject> qo) = 0;
6969

7070
/**
71-
* Look up an object of a task and return it.
72-
* \details It returns the object if found or nullptr if not.
73-
* TODO evaluate whether we should have more methods to retrieve objects of different types (with or without
74-
* templates)
75-
* TODO evaluate whether we should have a method to retrieve a list of objects (optimization)
71+
* \brief Look up a monitor object and return it.
72+
* Look up a monitor object and return it if found or nullptr if not.
73+
* @deprecated
7674
*/
7775
virtual std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string taskName, std::string objectName, long timestamp = 0) = 0;
76+
/**
77+
* \brief Look up a quality object and return it.
78+
* Look up a quality object and return it if found or nullptr if not.
79+
* @deprecated
80+
*/
7881
virtual std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = 0) = 0;
82+
/**
83+
* \brief Look up an object and return it.
84+
* Look up an object and return it if found or nullptr if not.
85+
* \param path the path of the object
86+
* \param timestamp the timestamp to query the object
87+
*/
88+
virtual std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp) = 0;
89+
/**
90+
* \brief Look up an object and return it.
91+
* Look up an object and return it if found or nullptr if not.
92+
* A default timestamp of -1 is used, usually meaning to use the current timestamp.
93+
* \param path the path of the object
94+
*/
95+
virtual std::shared_ptr<TObject> retrieveTObject(std::string path)
96+
{
97+
return retrieveTObject(path, -1);
98+
}
7999

80100
/**
81-
* Returns JSON encoded object
101+
* \brief Look up a monitor object and return it in JSON format.
102+
* Look up a monitor object and return it in JSON format if found or an empty string if not.
103+
* @deprecated
82104
*/
83105
virtual std::string retrieveMOJson(std::string taskName, std::string objectName, long timestamp = 0) = 0;
106+
/**
107+
* \brief Look up a quality object and return it in JSON format.
108+
* Look up a quality object and return it in JSON format if found or an empty string if not.
109+
* @deprecated
110+
*/
84111
virtual std::string retrieveQOJson(std::string qoPath, long timestamp = 0) = 0;
112+
/**
113+
* \brief Look up an object and return it in JSON format.
114+
* Look up an object and return it in JSON format if found or an empty string if not.
115+
* \param path the path of the object
116+
* \param timestamp the timestamp to query the object
117+
*/
118+
virtual std::string retrieveJson(std::string path, long timestamp) = 0;
119+
/**
120+
* \brief Look up an object and return it in JSON format.
121+
* Look up an object and return it in JSON format if found or an empty string if not.
122+
* A default timestamp of -1 is used, usually meaning to use the current timestamp.
123+
* \param path the path to the object
124+
*/
125+
virtual std::string retrieveJson(std::string path)
126+
{
127+
return retrieveJson(path, -1);
128+
}
129+
85130
virtual void disconnect() = 0;
86131
/**
87132
* \brief Prepare the container, such as a table in a relational database, that will contain the MonitorObject's for

‎Framework/include/QualityControl/DummyDatabase.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ class DummyDatabase : public DatabaseInterface
3838
void storeQO(std::shared_ptr<o2::quality_control::core::QualityObject> q) override;
3939
std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string checkerName, long timestamp = 0) override;
4040
std::string retrieveQOJson(std::string checkName, long timestamp = 0) override;
41+
// General
42+
std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp) override;
43+
std::string retrieveJson(std::string path, long timestamp = 0) override;
4144

4245
void disconnect() override;
4346
void prepareTaskDataContainer(std::string taskName) override;

‎Framework/include/QualityControl/MySqlDatabase.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ class MySqlDatabase : public DatabaseInterface
4747
void storeQO(std::shared_ptr<o2::quality_control::core::QualityObject> q) override;
4848
std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = 0) override;
4949
std::string retrieveQOJson(std::string qoPath, long timestamp = 0) override;
50+
// General
51+
std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp) override;
52+
std::string retrieveJson(std::string path, long timestamp = 0) override;
5053

5154
void disconnect() override;
5255
std::vector<std::string> getPublishedObjectNames(std::string taskName) override;

‎Framework/src/CcdbDatabase.cxx‎

Lines changed: 52 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,8 @@ void CcdbDatabase::storeMO(std::shared_ptr<o2::quality_control::core::MonitorObj
130130
ccdbApi.storeAsTFile(mo.get(), path, metadata, from, to);
131131
}
132132

133-
std::shared_ptr<core::MonitorObject> CcdbDatabase::retrieveMO(std::string taskName, std::string objectName, long timestamp)
133+
std::shared_ptr<TObject> CcdbDatabase::retrieveTObject(std::string path, long timestamp)
134134
{
135-
string path = taskName + "/" + objectName;
136135
map<string, string> metadata;
137136
long when = timestamp == 0 ? getCurrentTimestamp() : timestamp;
138137

@@ -146,22 +145,66 @@ std::shared_ptr<core::MonitorObject> CcdbDatabase::retrieveMO(std::string taskNa
146145
return nullptr;
147146
}
148147
}
149-
std::shared_ptr<core::MonitorObject> mo(dynamic_cast<core::MonitorObject*>(object));
148+
std::shared_ptr<TObject> result(object);
149+
return result;
150+
}
151+
152+
std::shared_ptr<core::MonitorObject> CcdbDatabase::retrieveMO(std::string taskName, std::string objectName, long timestamp)
153+
{
154+
string path = taskName + "/" + objectName;
155+
std::shared_ptr<TObject> obj = retrieveTObject(path, timestamp);
156+
std::shared_ptr<MonitorObject> mo = std::dynamic_pointer_cast<MonitorObject>(obj);
150157
if (mo == nullptr) {
151158
ILOG(Error) << "Could not cast the object " << taskName << "/" << objectName << " to MonitorObject" << ENDM;
152159
}
153160
return mo;
154161
}
155162

156-
std::string CcdbDatabase::retrieveMOJson(std::string taskName, std::string objectName, long /*timestamp*/)
163+
std::shared_ptr<QualityObject> CcdbDatabase::retrieveQO(std::string qoPath, long timestamp)
164+
{
165+
std::shared_ptr<TObject> obj = retrieveTObject(qoPath, timestamp);
166+
std::shared_ptr<QualityObject> qo = std::dynamic_pointer_cast<QualityObject>(obj);
167+
if (qo == nullptr) {
168+
LOG(ERROR) << "Could not cast the object " << qoPath << " to QualityObject";
169+
}
170+
return qo;
171+
}
172+
173+
std::string CcdbDatabase::retrieveQOJson(std::string qoPath, long timestamp)
157174
{
158-
auto monitor = retrieveMO(taskName, objectName);
159-
if (monitor == nullptr) {
175+
return retrieveJson(qoPath, timestamp);
176+
}
177+
178+
std::string CcdbDatabase::retrieveMOJson(std::string taskName, std::string objectName, long timestamp)
179+
{
180+
string path = taskName + "/" + objectName;
181+
return retrieveJson(path, timestamp);
182+
}
183+
184+
std::string CcdbDatabase::retrieveJson(std::string path, long timestamp)
185+
{
186+
auto tobj = retrieveTObject(path, timestamp);
187+
if (tobj == nullptr) {
188+
return std::string();
189+
}
190+
TObject* toConvert;
191+
if (tobj->IsA() == MonitorObject::Class()) {
192+
std::shared_ptr<MonitorObject> mo = std::dynamic_pointer_cast<MonitorObject>(tobj);
193+
toConvert = mo->getObject();
194+
mo->setIsOwner(false);
195+
} else if (tobj->IsA() == QualityObject::Class()) {
196+
toConvert = dynamic_cast<QualityObject*>(tobj.get());
197+
if (toConvert == nullptr) {
198+
return std::string();
199+
}
200+
} else {
201+
LOG(ERROR) << "Unknown type of object : " << path << " -> " << tobj->ClassName();
160202
return std::string();
161203
}
162-
std::unique_ptr<TObject> obj(monitor->getObject());
163-
monitor->setIsOwner(false);
164-
TString json = TBufferJSON::ConvertToJSON(obj.get());
204+
TString json = TBufferJSON::ConvertToJSON(toConvert);
205+
if (toConvert) {
206+
delete toConvert;
207+
}
165208
return json.Data();
166209
}
167210

@@ -182,39 +225,6 @@ void CcdbDatabase::storeQO(std::shared_ptr<QualityObject> qo)
182225
ccdbApi.storeAsTFile(qo.get(), path, metadata, from, to);
183226
}
184227

185-
std::shared_ptr<QualityObject> CcdbDatabase::retrieveQO(std::string qoPath, long timestamp)
186-
{
187-
string fullPath = qoPath;
188-
map<string, string> metadata;
189-
long when = timestamp == 0 ? getCurrentTimestamp() : timestamp;
190-
191-
// we try first to load a TFile
192-
TObject* object = ccdbApi.retrieveFromTFile(fullPath, metadata, when);
193-
if (object == nullptr) {
194-
// We could not open a TFile we should now try to open an object directly serialized
195-
object = ccdbApi.retrieve(fullPath, metadata, when);
196-
LOG(DEBUG) << "We could retrieve the object " << fullPath << " as a streamed object.";
197-
if (object == nullptr) {
198-
return nullptr;
199-
}
200-
}
201-
std::shared_ptr<QualityObject> qo(dynamic_cast<QualityObject*>(object));
202-
if (qo == nullptr) {
203-
LOG(ERROR) << "Could not cast the object " << qoPath << " to QualityObject";
204-
}
205-
return qo;
206-
}
207-
208-
std::string CcdbDatabase::retrieveQOJson(std::string qoPath, long /*timestamp*/)
209-
{
210-
auto qualityObject = retrieveQO(qoPath);
211-
if (qualityObject == nullptr) {
212-
return std::string();
213-
}
214-
TString json = TBufferJSON::ConvertToJSON(qualityObject.get());
215-
return json.Data();
216-
}
217-
218228
void CcdbDatabase::disconnect()
219229
{
220230
// NOOP for CCDB

‎Framework/src/DummyDatabase.cxx‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
12
// Copyright CERN and copyright holders of ALICE O2. This software is
23
// distributed under the terms of the GNU General Public License v3 (GPL
34
// Version 3), copied verbatim in the file "COPYING".
@@ -72,4 +73,13 @@ void DummyDatabase::truncate(std::string, std::string)
7273
{
7374
}
7475

76+
std::shared_ptr<TObject> DummyDatabase::retrieveTObject(std::string /*path*/, long /*timestamp*/)
77+
{
78+
return std::shared_ptr<TObject>();
79+
}
80+
std::string DummyDatabase::retrieveJson(std::string /*path*/, long /*timestamp*/)
81+
{
82+
return std::string();
83+
}
84+
7585
} // namespace o2::quality_control::repository

‎Framework/src/MySqlDatabase.cxx‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,4 +461,13 @@ void MySqlDatabase::truncate(std::string taskName, std::string objectName)
461461
}
462462
}
463463

464+
std::shared_ptr<TObject> MySqlDatabase::retrieveTObject(std::string /*path*/, long /*timestamp*/)
465+
{
466+
return std::shared_ptr<TObject>(); // TODO
467+
}
468+
std::string MySqlDatabase::retrieveJson(std::string /*path*/, long /*timestamp*/)
469+
{
470+
return std::string(); // TODO
471+
}
472+
464473
} // namespace o2::quality_control::repository

‎Framework/test/testCcdbDatabase.cxx‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,25 @@ BOOST_AUTO_TEST_CASE(ccdb_store)
101101
BOOST_AUTO_TEST_CASE(ccdb_retrieve, *utf::depends_on("ccdb_store"))
102102
{
103103
test_fixture f;
104-
//MonitorObject* mo = f.backend->retrieve("qc/TST/my/task", "asdf/asdf");
104+
105+
std::shared_ptr<TObject> obj = f.backend->retrieveTObject("qc/TST/my/task/asdf/asdf");
106+
auto mo = dynamic_pointer_cast<MonitorObject>(obj);
107+
auto mo2 = f.backend->retrieveMO("qc/TST/my/task", "asdf/asdf");
108+
BOOST_CHECK_NE(mo, nullptr);
109+
BOOST_CHECK_EQUAL(mo->getName(), mo2->getName());
110+
TH1F* h1 = dynamic_cast<TH1F*>(mo->getObject());
111+
BOOST_CHECK_NE(h1, nullptr);
112+
BOOST_CHECK_EQUAL(h1->GetEntries(), 10000);
113+
114+
std::shared_ptr<TObject> obj2 = f.backend->retrieveTObject("qc/checks/TST/test-ccdb-check");
115+
auto qo = dynamic_pointer_cast<QualityObject>(obj2);
116+
BOOST_CHECK_NE(qo, nullptr);
117+
BOOST_CHECK_EQUAL(qo->getQuality(), Quality::Bad);
118+
}
119+
120+
BOOST_AUTO_TEST_CASE(ccdb_retrieve_mo, *utf::depends_on("ccdb_store"))
121+
{
122+
test_fixture f;
105123
std::shared_ptr<MonitorObject> mo = f.backend->retrieveMO("qc/TST/my/task", "asdf/asdf");
106124
BOOST_REQUIRE_NE(mo, nullptr);
107125
TH1F* h1 = dynamic_cast<TH1F*>(mo->getObject());
@@ -117,13 +135,44 @@ BOOST_AUTO_TEST_CASE(ccdb_retrieve, *utf::depends_on("ccdb_store"))
117135
BOOST_CHECK_EQUAL(qo->getDetectorName(), "TST");
118136
}
119137

138+
BOOST_AUTO_TEST_CASE(ccdb_retrieve_qo, *utf::depends_on("ccdb_store"))
139+
{
140+
test_fixture f;
141+
std::shared_ptr<QualityObject> qo = f.backend->retrieveQO("qc/checks/TST/test-ccdb-check");
142+
BOOST_CHECK_NE(qo, nullptr);
143+
Quality q = qo->getQuality();
144+
BOOST_CHECK_EQUAL(q.getLevel(), 3);
145+
}
146+
120147
BOOST_AUTO_TEST_CASE(ccdb_retrieve_json, *utf::depends_on("ccdb_store"))
121148
{
122149
test_fixture f;
123150

124151
std::string task = "qc/TST/my/task";
125152
std::string object = "asdf/asdf";
126-
std::shared_ptr<MonitorObject> mo = f.backend->retrieveMO(task, object);
153+
std::cout << "[json retrieve]: " << task << "/" << object << std::endl;
154+
auto json = f.backend->retrieveJson(task + "/" + object);
155+
auto json2 = f.backend->retrieveMOJson(task, object);
156+
157+
cout << "mo json : " << json << endl;
158+
159+
BOOST_CHECK(!json.empty());
160+
BOOST_CHECK_EQUAL(json, json2);
161+
162+
string qualityPath = "qc/checks/TST/test-ccdb-check";
163+
std::cout << "[json retrieve]: " << qualityPath << std::endl;
164+
auto json3 = f.backend->retrieveJson(qualityPath);
165+
auto json4 = f.backend->retrieveQOJson(qualityPath);
166+
cout << "qo json : " << json3 << endl;
167+
BOOST_CHECK(!json3.empty());
168+
BOOST_CHECK_EQUAL(json3, json4);
169+
}
170+
171+
BOOST_AUTO_TEST_CASE(ccdb_retrieve_mo_json, *utf::depends_on("ccdb_store"))
172+
{
173+
test_fixture f;
174+
std::string task = "qc/TST/my/task";
175+
std::string object = "asdf/asdf";
127176
std::cout << "[json retrieve]: " << task << "/" << object << std::endl;
128177
auto jsonMO = f.backend->retrieveMOJson(task, object);
129178

‎Framework/test/testTrendingTask.cxx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ BOOST_AUTO_TEST_CASE(test_task)
5757
std::shared_ptr<MonitorObject> mo = std::make_shared<MonitorObject>(histo, taskName, "TST");
5858
repository->storeMO(mo);
5959

60-
std::shared_ptr<QualityObject> qo = std::make_shared<QualityObject>("testTrendingTaskCheck");
60+
std::vector<std::string> inputs;
61+
std::shared_ptr<QualityObject> qo = std::make_shared<QualityObject>("testTrendingTaskCheck", inputs, "TST");
6162
qo->updateQuality(Quality::Bad);
6263
repository->storeQO(qo);
6364
}

‎Framework/test/testTrendingTask.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
},
3939
{
4040
"type": "repository-quality",
41-
"path": "qc/checks",
41+
"path": "qc/checks/TST",
4242
"name": "testTrendingTaskCheck",
4343
"reductorName": "o2::quality_control_modules::common::QualityReductor",
4444
"moduleName": "QcCommon"

0 commit comments

Comments
 (0)