Skip to content

Commit c88c616

Browse files
aferrero2707aferrero2707
authored
Add a generic reference histogram comparator (#2317)
* Add a generic reference data comparator The code introduces some tools that allow to compare the plots produced by given task(s) with that of a reference run: - `ReferenceComparatorTask` draws histograms with their reference superimposed, as well as the ratio between histograms and reference - `ReferenceComparatorCheck` compares the histograms with their corresponding reference, and assesses their compatibility. The comparison is performed by a dynamically loadable module that provides the actual comparison algorithm - `ReferenceComparatorPlot` is an utility class to draw the current and reference histograms, and their ratio - `ObjectComparator*` are dynamically loadable modules than implement different histogram comparison methods Three examples are provided: - `ObjectComparatorDeviation` evaluates the average relative difference between the histogram bins - `ObjectComparatorChi2` compares the histograms using a chi2-based test - `ObjectComparatorKolmogorov` compares the histograms using a Kolmogorov test * [Common] implement review comments
1 parent 78bc952 commit c88c616

20 files changed

Lines changed: 1927 additions & 0 deletions

‎Modules/Common/CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ target_sources(O2QcCommon
2222
src/CcdbInspectorTask.cxx
2323
src/CcdbInspectorTaskConfig.cxx
2424
src/CcdbInspectorCheck.cxx
25+
src/ObjectComparatorInterface.cxx
26+
src/ObjectComparatorDeviation.cxx
27+
src/ObjectComparatorChi2.cxx
28+
src/ObjectComparatorKolmogorov.cxx
29+
src/ReferenceComparatorPlot.cxx
30+
src/ReferenceComparatorTask.cxx
31+
src/ReferenceComparatorTaskConfig.cxx
32+
src/ReferenceComparatorCheck.cxx
2533
src/NonEmpty.cxx
2634
src/MeanIsAbove.cxx
2735
src/TH1Reductor.cxx
@@ -54,6 +62,12 @@ add_root_dictionary(O2QcCommon
5462
include/Common/BigScreen.h
5563
include/Common/CcdbInspectorTask.h
5664
include/Common/CcdbInspectorCheck.h
65+
include/Common/ObjectComparatorInterface.h
66+
include/Common/ObjectComparatorDeviation.h
67+
include/Common/ObjectComparatorChi2.h
68+
include/Common/ObjectComparatorKolmogorov.h
69+
include/Common/ReferenceComparatorTask.h
70+
include/Common/ReferenceComparatorCheck.h
5771
include/Common/MeanIsAbove.h
5872
include/Common/TH1Ratio.h
5973
include/Common/TH2Ratio.h

‎Modules/Common/etc/reference-comparator-example.json‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080",
7+
"username": "not_applicable",
8+
"password": "not_applicable",
9+
"name": "not_applicable"
10+
},
11+
"Activity": {
12+
},
13+
"monitoring": {
14+
"url": "infologger:///debug?qc"
15+
},
16+
"consul": {
17+
"url": ""
18+
},
19+
"conditionDB": {
20+
"url": "ccdb-test.cern.ch:8080"
21+
}
22+
},
23+
"postprocessing": {
24+
"ExampleRefComp": {
25+
"active": "true",
26+
"className": "o2::quality_control_modules::common::ReferenceComparatorTask",
27+
"moduleName": "QualityControl",
28+
"detectorName": "MCH",
29+
"extendedTaskParameters": {
30+
"default": {
31+
"default": {
32+
"notOlderThan" : "300",
33+
"referenceRun" : "551875"
34+
}
35+
},
36+
"PHYSICS": {
37+
"PROTON-PROTON": {
38+
"referenceRun" : "551890"
39+
}
40+
}
41+
},
42+
"dataGroups": [
43+
{
44+
"name": "Tracks",
45+
"inputPath": "MCH/MO/Tracks/WithCuts",
46+
"referencePath": "MCH/MO/Tracks",
47+
"outputPath": "Tracks/WithCuts",
48+
"normalizeReference": "true",
49+
"drawRatioOnly": "false",
50+
"drawOption1D": "E",
51+
"drawOption2D": "COL",
52+
"inputObjects": [
53+
"TrackEta",
54+
"TrackEtaPhi"
55+
]
56+
}
57+
],
58+
"initTrigger": [
59+
"userorcontrol"
60+
],
61+
"updateTrigger": [
62+
"60 seconds"
63+
],
64+
"stopTrigger": [
65+
"userorcontrol"
66+
]
67+
}
68+
},
69+
"checks": {
70+
"ExampleRefCheck": {
71+
"active": "true",
72+
"className": "o2::quality_control_modules::common::ReferenceComparatorCheck",
73+
"moduleName": "QualityControl",
74+
"detectorName": "MCH",
75+
"policy": "OnAny",
76+
"extendedCheckParameters": {
77+
"default": {
78+
"default": {
79+
"moduleName" : "QualityControl",
80+
"comparatorName" : "o2::quality_control_modules::common::ObjectComparatorChi2",
81+
"threshold" : "0.5"
82+
}
83+
}
84+
},
85+
"dataSource": [
86+
{
87+
"type": "PostProcessing",
88+
"name": "ExampleRefComp",
89+
"MOs" : [
90+
"Tracks/WithCuts/TrackEta",
91+
"Tracks/WithCuts/TrackEtaPhi"
92+
]
93+
}
94+
]
95+
}
96+
}
97+
}
98+
}

‎Modules/Common/include/Common/LinkDef.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
#pragma link C++ class o2::quality_control_modules::common::BigScreen + ;
2020
#pragma link C++ class o2::quality_control_modules::common::CcdbInspectorTask + ;
2121
#pragma link C++ class o2::quality_control_modules::common::CcdbInspectorCheck + ;
22+
#pragma link C++ class o2::quality_control_modules::common::ObjectComparatorDeviation + ;
23+
#pragma link C++ class o2::quality_control_modules::common::ObjectComparatorChi2 + ;
24+
#pragma link C++ class o2::quality_control_modules::common::ObjectComparatorKolmogorov + ;
25+
#pragma link C++ class o2::quality_control_modules::common::ReferenceComparatorTask + ;
26+
#pragma link C++ class o2::quality_control_modules::common::ReferenceComparatorCheck + ;
2227
#pragma link C++ class o2::quality_control_modules::common::WorstOfAllAggregator + ;
2328
#pragma link C++ class o2::quality_control_modules::common::IncreasingEntries + ;
2429
#pragma link C++ class o2::quality_control_modules::common::TH1SliceReductor + ;

‎Modules/Common/include/Common/ObjectComparatorChi2.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file ObjectComparatorChi2.h
14+
/// \author Andrea Ferrero
15+
/// \brief A class for comparing two histogram objects based on a chi2 test
16+
///
17+
18+
#ifndef QUALITYCONTROL_ObjectComparatorChi2_H
19+
#define QUALITYCONTROL_ObjectComparatorChi2_H
20+
21+
#include "Common/ObjectComparatorInterface.h"
22+
23+
namespace o2::quality_control_modules::common
24+
{
25+
26+
/// \brief A class for comparing two histogram objects based on a chi2 test
27+
class ObjectComparatorChi2 : public ObjectComparatorInterface
28+
{
29+
public:
30+
/// \brief Constructor
31+
ObjectComparatorChi2() = default;
32+
/// \brief Destructor
33+
virtual ~ObjectComparatorChi2() = default;
34+
35+
/// \brief objects comparison function
36+
/// \return the quality resulting from the object comparison
37+
o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) override;
38+
};
39+
40+
} // namespace o2::quality_control_modules::common
41+
42+
#endif // QUALITYCONTROL_ObjectComparatorChi2_H

‎Modules/Common/include/Common/ObjectComparatorDeviation.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file ObjectComparatorDeviation.h
14+
/// \author Andrea Ferrero
15+
/// \brief A class for comparing two histogram objects based on the average of the relative deviation between the bins
16+
///
17+
18+
#ifndef QUALITYCONTROL_ObjectComparatorDeviation_H
19+
#define QUALITYCONTROL_ObjectComparatorDeviation_H
20+
21+
#include "Common/ObjectComparatorInterface.h"
22+
23+
namespace o2::quality_control_modules::common
24+
{
25+
26+
/// \brief A class for comparing two histogram objects based on the average of the relative deviation between the bins
27+
class ObjectComparatorDeviation : public ObjectComparatorInterface
28+
{
29+
public:
30+
/// \brief Constructor
31+
ObjectComparatorDeviation() = default;
32+
/// \brief Destructor
33+
virtual ~ObjectComparatorDeviation() = default;
34+
35+
/// \brief objects comparison function
36+
/// \return the quality resulting from the object comparison
37+
o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) override;
38+
};
39+
40+
} // namespace o2::quality_control_modules::common
41+
42+
#endif // QUALITYCONTROL_ObjectComparatorDeviation_H

‎Modules/Common/include/Common/ObjectComparatorInterface.h‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file ObjectComparatorInterface.h
14+
/// \author Andrea Ferrero
15+
/// \brief An interface for comparing two TObject
16+
///
17+
18+
#ifndef QUALITYCONTROL_ObjectComparatorInterface_H
19+
#define QUALITYCONTROL_ObjectComparatorInterface_H
20+
21+
#include "QualityControl/Quality.h"
22+
#include "QualityControl/CustomParameters.h"
23+
#include "QualityControl/Activity.h"
24+
25+
#include <tuple>
26+
27+
class TObject;
28+
class TH1;
29+
30+
namespace o2::quality_control_modules::common
31+
{
32+
33+
/// \brief An interface for comparing two TObject
34+
class ObjectComparatorInterface
35+
{
36+
public:
37+
/// \brief Constructor
38+
ObjectComparatorInterface() = default;
39+
/// \brief Destructor
40+
virtual ~ObjectComparatorInterface() = default;
41+
42+
/// \brief comparator configuration via CustomParameters
43+
// virtual void configure(const o2::quality_control::core::CustomParameters& customParameters, const o2::quality_control::core::Activity activity = {}){};
44+
45+
/// setter/getter methods for the threshold to define the goodness of the comparison
46+
void setThreshold(double threshold) { mThreshold = threshold; }
47+
double getThreshold() { return mThreshold; }
48+
49+
/// perform a number of sanity checks on the input objects
50+
/// \return a tuple containing pointers to the histogram, the reference histogram, and a boolean indicating the success of the checks
51+
std::tuple<TH1*, TH1*, bool> checkInputObjects(TObject* object, TObject* referenceObject, std::string& message);
52+
53+
/// \brief objects comparison function
54+
/// \return the quality resulting from the object comparison
55+
virtual o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) = 0;
56+
57+
private:
58+
/// the threshold to define the goodness of the comparison
59+
double mThreshold{ 0 };
60+
};
61+
62+
} // namespace o2::quality_control_modules::common
63+
64+
#endif // QUALITYCONTROL_ObjectComparatorInterface_H

‎Modules/Common/include/Common/ObjectComparatorKolmogorov.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file ObjectComparatorKolmogorov.h
14+
/// \author Andrea Ferrero
15+
/// \brief A class for comparing two histogram objects based on a Kolmogorov test
16+
///
17+
18+
#ifndef QUALITYCONTROL_ObjectComparatorKolmogorov_H
19+
#define QUALITYCONTROL_ObjectComparatorKolmogorov_H
20+
21+
#include "Common/ObjectComparatorInterface.h"
22+
23+
namespace o2::quality_control_modules::common
24+
{
25+
26+
/// \brief A class for comparing two histogram objects based on a Kolmogorov test
27+
class ObjectComparatorKolmogorov : public ObjectComparatorInterface
28+
{
29+
public:
30+
/// \brief Constructor
31+
ObjectComparatorKolmogorov() = default;
32+
/// \brief Destructor
33+
virtual ~ObjectComparatorKolmogorov() = default;
34+
35+
/// \brief objects comparison function
36+
/// \return the quality resulting from the object comparison
37+
o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) override;
38+
};
39+
40+
} // namespace o2::quality_control_modules::common
41+
42+
#endif // QUALITYCONTROL_ObjectComparatorKolmogorov_H

‎Modules/Common/include/Common/ReferenceComparatorCheck.h‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file ReferenceComparatorCheck.h
14+
/// \author Andrea Ferrero
15+
/// \brief A generic QC check that compares a given set of histograms with their corresponding references
16+
///
17+
18+
#ifndef QUALITYCONTROL_ReferenceComparatorCheck_H
19+
#define QUALITYCONTROL_ReferenceComparatorCheck_H
20+
21+
#include "QualityControl/CheckInterface.h"
22+
#include "Common/ObjectComparatorInterface.h"
23+
24+
#include <sstream>
25+
26+
class TPaveText;
27+
28+
namespace o2::quality_control_modules::common
29+
{
30+
31+
/// \brief A generic QC check that compares a given set of histograms with their corresponding references
32+
/// \author Andrea Ferrero
33+
class ReferenceComparatorCheck : public o2::quality_control::checker::CheckInterface
34+
{
35+
public:
36+
/// Default constructor
37+
ReferenceComparatorCheck() = default;
38+
/// Destructor
39+
~ReferenceComparatorCheck() override = default;
40+
41+
// Override interface
42+
void configure() override;
43+
Quality check(std::map<std::string, std::shared_ptr<MonitorObject>>* moMap) override;
44+
void reset() override;
45+
void beautify(std::shared_ptr<MonitorObject> mo, Quality checkResult = Quality::Null) override;
46+
std::string getAcceptedType() override;
47+
48+
void startOfActivity(const Activity& activity) override;
49+
void endOfActivity(const Activity& activity) override;
50+
51+
private:
52+
std::unique_ptr<ObjectComparatorInterface> mComparator;
53+
std::map<std::string, Quality> mQualityFlags;
54+
std::map<std::string, std::shared_ptr<TPaveText>> mQualityLabels;
55+
};
56+
57+
} // namespace o2::quality_control_modules::common
58+
59+
#endif // QC_MODULE_SKELETON_ReferenceComparatorCheck_H

0 commit comments

Comments
 (0)