Skip to content

Commit 3ccfac3

Browse files
BarthelemyBarthelemy
authored
(QC-239) Storage and retrieval of the encapsulated object, not the MO (#362)
* use storeAsTFileAny * retrieveTObject is not part of the interface * add getCheckName * Storage and retrieval of the encapsulated object, not the MO. * Addition of backward compatibility tests
1 parent 0daf2cb commit 3ccfac3

16 files changed

Lines changed: 293 additions & 121 deletions

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ include(GNUInstallDirs)
1212
# ---- Project ----
1313

1414
project(QualityControl
15-
VERSION 0.24.0 # TODO update this automatically when there are new releases
15+
VERSION 0.25.0 # TODO update this automatically when there are new releases
1616
DESCRIPTION "O2 Data Quality Control Framework"
1717
LANGUAGES CXX)
1818

‎Framework/include/QualityControl/CcdbDatabase.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@ class CcdbDatabase : public DatabaseInterface
6969
std::string retrieveQOJson(std::string qoPath, long timestamp = 0) override;
7070

7171
// retrieval - general
72-
std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp = -1) override;
73-
std::string retrieveJson(std::string path, long timestamp = -1) override;
72+
std::string retrieveJson(std::string path, long timestamp, const std::map<std::string, std::string>& metadata) override;
73+
TObject* retrieveTObject(std::string path, const std::map<std::string, std::string>& metadata, long timestamp = -1, std::map<std::string, std::string>* headers = nullptr) override;
7474

7575
void disconnect() override;
7676
void prepareTaskDataContainer(std::string taskName) override;

‎Framework/include/QualityControl/DatabaseInterface.h‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -81,21 +81,13 @@ class DatabaseInterface
8181
virtual std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = 0) = 0;
8282
/**
8383
* \brief Look up an object and return it.
84-
* Look up an object and return it if found or nullptr if not.
84+
* Look up an object and return it if found or nullptr if not. It is a raw pointer because we might need it to build a MO.
8585
* \param path the path of the object
8686
* \param timestamp the timestamp to query the object
87+
* \param headers Map to be populated with the headers we received, if it is not null.
88+
* \param metadata filters under the form of key-value pairs to select data
8789
*/
88-
virtual std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp) = 0;
89-
/**
90-
* \brief Look up an object and return it.
91-
* Look up an object and return it if found or nullptr if not.
92-
* A default timestamp of -1 is used, usually meaning to use the current timestamp.
93-
* \param path the path of the object
94-
*/
95-
virtual std::shared_ptr<TObject> retrieveTObject(std::string path)
96-
{
97-
return retrieveTObject(path, -1);
98-
}
90+
virtual TObject* retrieveTObject(std::string path, const std::map<std::string, std::string>& metadata, long timestamp = -1, std::map<std::string, std::string>* headers = nullptr) = 0;
9991

10092
/**
10193
* \brief Look up a monitor object and return it in JSON format.
@@ -114,8 +106,9 @@ class DatabaseInterface
114106
* Look up an object and return it in JSON format if found or an empty string if not.
115107
* \param path the path of the object
116108
* \param timestamp the timestamp to query the object
109+
* \param metadata filters under the form of key-value pairs to select data
117110
*/
118-
virtual std::string retrieveJson(std::string path, long timestamp) = 0;
111+
virtual std::string retrieveJson(std::string path, long timestamp, const std::map<std::string, std::string>& metadata) = 0;
119112
/**
120113
* \brief Look up an object and return it in JSON format.
121114
* Look up an object and return it in JSON format if found or an empty string if not.
@@ -124,7 +117,8 @@ class DatabaseInterface
124117
*/
125118
virtual std::string retrieveJson(std::string path)
126119
{
127-
return retrieveJson(path, -1);
120+
std::map<std::string, std::string> metadata;
121+
return retrieveJson(path, -1, metadata);
128122
}
129123

130124
virtual void disconnect() = 0;

‎Framework/include/QualityControl/DummyDatabase.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ class DummyDatabase : public DatabaseInterface
3939
std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string checkerName, long timestamp = 0) override;
4040
std::string retrieveQOJson(std::string checkName, long timestamp = 0) override;
4141
// General
42-
std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp) override;
43-
std::string retrieveJson(std::string path, long timestamp = 0) override;
42+
std::string retrieveJson(std::string path, long timestamp, const std::map<std::string, std::string>& metadata) override;
43+
TObject* retrieveTObject(std::string path, const std::map<std::string, std::string>& metadata, long timestamp = -1, std::map<std::string, std::string>* headers = nullptr) override;
4444

4545
void disconnect() override;
4646
void prepareTaskDataContainer(std::string taskName) override;

‎Framework/include/QualityControl/MonitorObject.h‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ struct DuplicateObjectError : virtual AliceO2::Common::ExceptionBase {
3535
}
3636
};
3737

38-
/// \brief This class keeps the metadata about one published object.
38+
/// \brief This class keeps the meta data about one published object.
3939
///
4040
/// \author Barthelemy von Haller
4141
class MonitorObject : public TObject
@@ -82,12 +82,19 @@ class MonitorObject : public TObject
8282
const std::string& getDetectorName() const { return mDetectorName; }
8383
void setDetectorName(const std::string& detectorName) { mDetectorName = detectorName; }
8484

85-
/// \brief Add key value pair that will end up in the database
86-
/// Add a metadata (key value pair) to the MonitorObject. It will be stored in the database.
87-
/// If the key already exists the value will be updated.
85+
/// \brief Add key value pair that will end up in the database as metadata of the object
86+
/// Add a metadata (key value pair) to the MonitorObject. It will be stored in the database as metadata.
87+
/// If the key already exists the value will NOT be updated.
8888
void addMetadata(std::string key, std::string value);
89-
90-
std::map<std::string, std::string> getMetadataMap() const;
89+
/// \brief Add key value pairs that will end up in the database as metadata of the object
90+
/// Add all the key-value pairs in the map to the MonitorObject. It will be stored in the database as metadata.
91+
/// If a key already exists the value will NOT be updated.
92+
void addMetadata(std::map<std::string, std::string> pairs);
93+
/// \brief Update the value of metadata.
94+
/// If the key does not exist it will ignore it.
95+
void updateMetadata(std::string key, std::string value);
96+
/// \brief Get the full map of user's metadata
97+
const std::map<std::string, std::string>& getMetadataMap() const;
9198

9299
void Draw(Option_t* option) override;
93100
TObject* DrawClone(Option_t* option) const override;

‎Framework/include/QualityControl/MySqlDatabase.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ class MySqlDatabase : public DatabaseInterface
4848
std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = 0) override;
4949
std::string retrieveQOJson(std::string qoPath, long timestamp = 0) override;
5050
// General
51-
std::shared_ptr<TObject> retrieveTObject(std::string path, long timestamp) override;
52-
std::string retrieveJson(std::string path, long timestamp = 0) override;
51+
std::string retrieveJson(std::string path, long timestamp, const std::map<std::string, std::string>& metadata) override;
52+
TObject* retrieveTObject(std::string path, const std::map<std::string, std::string>& metadata, long timestamp = -1, std::map<std::string, std::string>* headers = nullptr) override;
5353

5454
void disconnect() override;
5555
std::vector<std::string> getPublishedObjectNames(std::string taskName) override;

‎Framework/include/QualityControl/QualityObject.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,15 @@ class QualityObject : public TObject
7575
/// Add a metadata (key value pair) to the QualityObject. It will be stored in the database.
7676
/// If the key already exists the value will be updated.
7777
void addMetadata(std::string key, std::string value);
78-
79-
std::map<std::string, std::string> getMetadataMap();
78+
/// \brief Add key value pairs that will end up in the database as metadata of the object
79+
/// Add all the key-value pairs in the map to the MonitorObject. It will be stored in the database as metadata.
80+
/// If a key already exists the value will NOT be updated.
81+
void addMetadata(std::map<std::string, std::string> pairs);
82+
/// \brief Update the value of metadata.
83+
/// If the key does not exist it will ignore it.
84+
void updateMetadata(std::string key, std::string value);
85+
/// \brief Get the full map of user's metadata
86+
const std::map<std::string, std::string>& getMetadataMap() const;
8087

8188
/// \brief Build the path to this object.
8289
/// Build the path to this object as it will appear in the GUI.
@@ -87,6 +94,7 @@ class QualityObject : public TObject
8794
void setDetectorName(const std::string& detectorName);
8895

8996
void setQuality(const Quality& quality);
97+
const std::string& getCheckName() const;
9098

9199
private:
92100
Quality mQuality;

‎Framework/src/CcdbDatabase.cxx‎

Lines changed: 74 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -126,103 +126,135 @@ void CcdbDatabase::storeMO(std::shared_ptr<o2::quality_control::core::MonitorObj
126126
long from = getCurrentTimestamp();
127127
long to = getFutureTimestamp(60 * 60 * 24 * 365 * 10);
128128

129-
ccdbApi.storeAsTFile(mo.get(), path, metadata, from, to);
129+
// extract object and metadata from MonitorObject
130+
TObject* obj = mo->getObject();
131+
metadata["qc_detector_name"] = mo->getDetectorName();
132+
metadata["qc_task_name"] = mo->getTaskName();
133+
metadata["ObjectType"] = mo->getObject()->IsA()->GetName(); // ObjectType says TObject and not MonitorObject due to a quirk in the API. Once fixed, remove this.
134+
135+
ccdbApi.storeAsTFileAny<TObject>(obj, path, metadata, from, to);
130136
}
131137

132-
std::shared_ptr<TObject> CcdbDatabase::retrieveTObject(std::string path, long timestamp)
138+
void CcdbDatabase::storeQO(std::shared_ptr<QualityObject> qo)
133139
{
140+
// metadata
134141
map<string, string> metadata;
142+
// QC metadata (prefix qc_)
143+
metadata["qc_version"] = Version::GetQcVersion().getString();
144+
metadata["qc_quality"] = std::to_string(qo->getQuality().getLevel());
145+
metadata["qc_detector_name"] = qo->getDetectorName();
146+
metadata["qc_check_name"] = qo->getCheckName();
147+
148+
// other attributes
149+
string path = qo->getPath();
150+
long from = getCurrentTimestamp();
151+
long to = getFutureTimestamp(60 * 60 * 24 * 365 * 10);
152+
153+
ccdbApi.storeAsTFileAny<QualityObject>(qo.get(), path, metadata, from, to);
154+
}
155+
156+
TObject* CcdbDatabase::retrieveTObject(std::string path, std::map<std::string, std::string> const& metadata, long timestamp, std::map<std::string, std::string>* headers)
157+
{
135158
long when = timestamp == 0 ? getCurrentTimestamp() : timestamp;
136159

137160
// we try first to load a TFile
138-
TObject* object = ccdbApi.retrieveFromTFile(path, metadata, when);
161+
auto* object = ccdbApi.retrieveFromTFileAny<TObject>(path, metadata, when, headers);
139162
if (object == nullptr) {
140163
// We could not open a TFile we should now try to open an object directly serialized
141164
object = ccdbApi.retrieve(path, metadata, when);
142-
ILOG(Debug) << "We could retrieve the object " << path << " as a streamed object." << ENDM;
143165
if (object == nullptr) {
166+
ILOG(Error) << "We could NOT retrieve the object " << path << "." << ENDM;
144167
return nullptr;
145168
}
146169
}
147-
std::shared_ptr<TObject> result(object);
148-
return result;
170+
ILOG(Debug) << "Retrieved object " << path << ENDM;
171+
return object;
149172
}
150173

151174
std::shared_ptr<core::MonitorObject> CcdbDatabase::retrieveMO(std::string taskName, std::string objectName, long timestamp)
152175
{
153176
string path = taskName + "/" + objectName;
154-
std::shared_ptr<TObject> obj = retrieveTObject(path, timestamp);
155-
std::shared_ptr<MonitorObject> mo = std::dynamic_pointer_cast<MonitorObject>(obj);
156-
if (mo == nullptr) {
157-
ILOG(Error) << "Could not cast the object " << taskName << "/" << objectName << " to MonitorObject" << ENDM;
177+
long when = timestamp == 0 ? getCurrentTimestamp() : timestamp;
178+
map<string, string> headers;
179+
map<string, string> metadata;
180+
TObject* obj = retrieveTObject(path, metadata, when, &headers);
181+
Version objectVersion(headers["qc_version"]); // retrieve headers to determine the version of the QC framework
182+
183+
std::shared_ptr<MonitorObject> mo;
184+
if (objectVersion == Version("0.0.0") || objectVersion < Version("0.25")) {
185+
ILOG(Debug) << "Version of object " << taskName << "/" << objectName << " is < 0.25" << ENDM;
186+
// The object is either in a TFile or is a blob but it was stored with storeAsTFile as a full MO
187+
mo.reset(dynamic_cast<MonitorObject*>(obj));
188+
if (mo == nullptr) {
189+
ILOG(Error) << "Could not cast the object " << taskName << "/" << objectName << " to MonitorObject" << ENDM;
190+
}
191+
} else {
192+
// Version >= 0.25 -> the object is stored directly unencapsulated
193+
ILOG(Debug) << "Version of object " << taskName << "/" << objectName << " is >= 0.25" << ENDM;
194+
mo = make_shared<MonitorObject>(obj, headers["qc_task_name"], headers["qc_detector_name"]);
195+
// TODO should we remove the headers we know are general such as ETag and qc_task_name ?
196+
mo->addMetadata(headers);
158197
}
159198
return mo;
160199
}
161200

162201
std::shared_ptr<QualityObject> CcdbDatabase::retrieveQO(std::string qoPath, long timestamp)
163202
{
164-
std::shared_ptr<TObject> obj = retrieveTObject(qoPath, timestamp);
165-
std::shared_ptr<QualityObject> qo = std::dynamic_pointer_cast<QualityObject>(obj);
203+
long when = timestamp == 0 ? getCurrentTimestamp() : timestamp;
204+
map<string, string> headers;
205+
map<string, string> metadata;
206+
TObject* obj = retrieveTObject(qoPath, metadata, when, &headers);
207+
std::shared_ptr<QualityObject> qo(dynamic_cast<QualityObject*>(obj));
166208
if (qo == nullptr) {
167-
LOG(ERROR) << "Could not cast the object " << qoPath << " to QualityObject";
209+
ILOG(Error) << "Could not cast the object " << qoPath << " to QualityObject" << ENDM;
168210
}
211+
// TODO should we remove the headers we know are general such as ETag and qc_task_name ?
212+
qo->addMetadata(headers);
169213
return qo;
170214
}
171215

172216
std::string CcdbDatabase::retrieveQOJson(std::string qoPath, long timestamp)
173217
{
174-
return retrieveJson(qoPath, timestamp);
218+
map<string, string> metadata;
219+
return retrieveJson(qoPath, timestamp, metadata);
175220
}
176221

177222
std::string CcdbDatabase::retrieveMOJson(std::string taskName, std::string objectName, long timestamp)
178223
{
179224
string path = taskName + "/" + objectName;
180-
return retrieveJson(path, timestamp);
225+
map<string, string> metadata;
226+
return retrieveJson(path, timestamp, metadata);
181227
}
182228

183-
std::string CcdbDatabase::retrieveJson(std::string path, long timestamp)
229+
std::string CcdbDatabase::retrieveJson(std::string path, long timestamp, const std::map<std::string, std::string>& metadata)
184230
{
185-
auto tobj = retrieveTObject(path, timestamp);
231+
map<string, string> headers;
232+
auto tobj = retrieveTObject(path, metadata, timestamp, &headers);
233+
186234
if (tobj == nullptr) {
187235
return std::string();
188236
}
189-
TObject* toConvert = 0;
190-
if (tobj->IsA() == MonitorObject::Class()) {
191-
std::shared_ptr<MonitorObject> mo = std::dynamic_pointer_cast<MonitorObject>(tobj);
237+
238+
TObject* toConvert = nullptr;
239+
if (tobj->IsA() == MonitorObject::Class()) { // a full MO -> pre-v0.25
240+
std::shared_ptr<MonitorObject> mo(dynamic_cast<MonitorObject*>(tobj));
192241
toConvert = mo->getObject();
193-
mo->setIsOwner(true);
242+
mo->setIsOwner(false);
194243
} else if (tobj->IsA() == QualityObject::Class()) {
195-
toConvert = dynamic_cast<QualityObject*>(tobj.get());
196-
} else {
197-
ILOG(Error) << "Unknown type of object : " << path << " -> " << tobj->ClassName() << ENDM;
198-
return std::string();
244+
toConvert = dynamic_cast<QualityObject*>(tobj);
245+
} else { // something else but a TObject
246+
toConvert = tobj;
199247
}
200248
if (toConvert == nullptr) {
201249
ILOG(Error) << "Unable to get the object to convert" << ENDM;
202250
return std::string();
203251
}
204252
TString json = TBufferJSON::ConvertToJSON(toConvert);
253+
delete toConvert;
205254

206255
return json.Data();
207256
}
208257

209-
//Quality Object
210-
void CcdbDatabase::storeQO(std::shared_ptr<QualityObject> qo)
211-
{
212-
// metadata
213-
map<string, string> metadata;
214-
// QC metadata (prefix qc_)
215-
metadata["qc_version"] = Version::GetQcVersion().getString();
216-
metadata["qc_quality"] = std::to_string(qo->getQuality().getLevel());
217-
218-
// other attributes
219-
string path = qo->getPath();
220-
long from = getCurrentTimestamp();
221-
long to = getFutureTimestamp(60 * 60 * 24 * 365 * 10);
222-
223-
ccdbApi.storeAsTFile(qo.get(), path, metadata, from, to);
224-
}
225-
226258
void CcdbDatabase::disconnect()
227259
{
228260
// NOOP for CCDB

‎Framework/src/DummyDatabase.cxx‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,12 @@ void DummyDatabase::truncate(std::string, std::string)
7373
{
7474
}
7575

76-
std::shared_ptr<TObject> DummyDatabase::retrieveTObject(std::string /*path*/, long /*timestamp*/)
76+
TObject* DummyDatabase::retrieveTObject(std::string, const std::map<std::string, std::string>&, long, std::map<std::string, std::string>*)
7777
{
78-
return std::shared_ptr<TObject>();
78+
return nullptr;
7979
}
80-
std::string DummyDatabase::retrieveJson(std::string /*path*/, long /*timestamp*/)
80+
81+
std::string DummyDatabase::retrieveJson(std::string, long, const std::map<std::string, std::string>&)
8182
{
8283
return std::string();
8384
}

‎Framework/src/MonitorObject.cxx‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,27 @@ const char* MonitorObject::GetName() const
6565

6666
void MonitorObject::addMetadata(std::string key, std::string value)
6767
{
68-
mUserMetadata[key] = value;
68+
mUserMetadata.insert({ key, value });
6969
}
7070

71-
std::map<std::string, std::string> MonitorObject::getMetadataMap() const
71+
void MonitorObject::addMetadata(std::map<std::string, std::string> pairs)
72+
{
73+
// we do not use "merge" because it would ignore the items whose key already exist in mUserMetadata.
74+
mUserMetadata.insert(pairs.begin(), pairs.end());
75+
}
76+
77+
const std::map<std::string, std::string>& MonitorObject::getMetadataMap() const
7278
{
7379
return mUserMetadata;
7480
}
7581

82+
void MonitorObject::updateMetadata(std::string key, std::string value)
83+
{
84+
if (mUserMetadata.count(key) > 0) {
85+
mUserMetadata[key] = value;
86+
}
87+
}
88+
7689
std::string MonitorObject::getPath() const
7790
{
7891
string path = "qc/" + getDetectorName() + "/" + getTaskName() + "/" + getName();

0 commit comments

Comments
 (0)