Skip to content

Commit b7f065e

Browse files
andrpieandrpie
committed
initial implementation of the dataset
1 parent 5af0a17 commit b7f065e

17 files changed

Lines changed: 33639 additions & 0 deletions

‎nnpdf_data/nnpdf_data/commondata/CMS_2JET_8TEV/data.yaml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
data_central:
2+
- 1304.0
3+
- 638.1
4+
- 329.4
5+
- 176.1
6+
- 95.42
7+
- 53.14
8+
- 30.04
9+
- 17.36
10+
- 10.07
11+
- 5.904
12+
- 3.53
13+
- 2.101
14+
- 1.257
15+
- 0.7631
16+
- 0.4598
17+
- 0.2762
18+
- 0.1661
19+
- 0.1004
20+
- 0.06007
21+
- 0.03592
22+
- 0.02109
23+
- 0.01254
24+
- 0.007197
25+
- 0.004058
26+
- 0.002391
27+
- 0.001419
28+
- 0.0007634
29+
- 0.0003934
30+
- 0.0002155
31+
- 0.0001198
32+
- 4.445e-05
33+
- 518.7
34+
- 239.1
35+
- 118.0
36+
- 59.7
37+
- 30.5
38+
- 15.88
39+
- 8.356
40+
- 4.508
41+
- 2.401
42+
- 1.303
43+
- 0.6989
44+
- 0.3753
45+
- 0.2016
46+
- 0.1091
47+
- 0.05782
48+
- 0.03049
49+
- 0.01591
50+
- 0.008247
51+
- 0.004217
52+
- 0.002188
53+
- 0.001044
54+
- 0.0004743
55+
- 0.000229
56+
- 9.465e-05
57+
- 4.074e-05
58+
- 2.321e-05
59+
- 39.11
60+
- 15.49
61+
- 5.749
62+
- 2.689
63+
- 1.138
64+
- 0.4543
65+
- 0.1875
66+
- 0.08166
67+
- 0.03022
68+
- 0.009612
69+
- 0.004567
70+
- 0.00146
71+
- 0.0004647
72+
- 0.0001141
73+
- 790.8
74+
- 376.0
75+
- 176.4
76+
- 90.12
77+
- 46.08
78+
- 23.09
79+
- 11.96
80+
- 6.188
81+
- 3.148
82+
- 1.634
83+
- 0.8323
84+
- 0.4201
85+
- 0.2114
86+
- 0.1044
87+
- 0.0504
88+
- 0.02377
89+
- 0.01105
90+
- 0.005018
91+
- 0.00204
92+
- 0.00086
93+
- 0.0003546
94+
- 0.0001093
95+
- 3.493e-05
96+
- 188.4
97+
- 79.95
98+
- 36.65
99+
- 16.05
100+
- 7.067
101+
- 3.108
102+
- 1.352
103+
- 0.593
104+
- 0.2592
105+
- 0.1068
106+
- 0.04124
107+
- 0.01593
108+
- 0.005859
109+
- 0.002166
110+
- 0.0006671
111+
- 0.0001653
112+
- 4.066e-05
113+
- 126.0
114+
- 47.86
115+
- 18.92
116+
- 7.102
117+
- 2.474
118+
- 0.8486
119+
- 0.2511
120+
- 0.06703
121+
- 0.01889
122+
- 0.002968
123+
- 0.0006449

‎nnpdf_data/nnpdf_data/commondata/CMS_2JET_8TEV/filter.py‎

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
import yaml
2+
import numpy as np
3+
import copy
4+
from nnpdf_data.filter_utils.utils import cormat_to_covmat, covmat_to_artunc
5+
6+
def read_metadata() -> tuple:
7+
'''
8+
takes the important information from the metadata file
9+
'''
10+
with open('metadata.yaml', 'r') as file:
11+
info = yaml.safe_load(file)
12+
obs_info = info['implemented_observables'][0]
13+
return obs_info['tables'], obs_info['ndata']
14+
15+
16+
def read_kinematics_and_centrals(tables: list) -> tuple:
17+
'''
18+
using the table numbers from metadata,
19+
reads the bins and central values from rawdata
20+
return two lists: one for kinematics, one for central values
21+
'''
22+
bins = list()
23+
centrals = list()
24+
for table_num in tables[:6]: # select just the tables with kinematic data
25+
with open(f'rawdata/table_{table_num}.yaml', 'r') as file:
26+
kins = yaml.safe_load(file)
27+
# get ystar, yboost, sqrts bins:
28+
current_yys_bin = dict()
29+
for yys_dict in kins['dependent_variables'][0]['qualifiers']:
30+
for k, v in yys_dict.items():
31+
if k == 'name':
32+
bin_name = v.lower().replace('(', '').replace(')', '')
33+
current_yys_bin[bin_name] = dict()
34+
if k == 'value':
35+
if type(v) == str:
36+
lower = float(v[:3])
37+
upper = float(v[-3:])
38+
middle = (lower+upper)/2
39+
current_yys_bin[bin_name] = {'min': lower, 'mid': middle, 'max': upper}
40+
else:
41+
current_yys_bin[bin_name] = {'min': None, 'mid': v, 'max': None}
42+
# get the ptavg bins and combine them with yys bins
43+
for ptavg_dict in kins['independent_variables'][0]['values']:
44+
lower = ptavg_dict['low']
45+
upper = ptavg_dict['high']
46+
middle = (lower + upper)/2
47+
current_p_bin = {'pTavg': {'min': lower, 'mid': middle, 'max': upper}}
48+
copy_yys = copy.deepcopy(current_yys_bin)
49+
current_pyys_bin = current_p_bin | copy_yys
50+
bins.append(current_pyys_bin)
51+
# get the central values
52+
for vals in kins['dependent_variables'][0]['values']:
53+
centrals.append(vals['value'])
54+
return bins, centrals
55+
56+
57+
def dump_kinematics_and_centrals(bins: list, centrals: list) -> None:
58+
# dump the dictionaries into files
59+
with open('kinematics.yaml', 'w') as file:
60+
yaml.safe_dump({'bins': bins}, file, sort_keys=False)
61+
with open('data.yaml', 'w') as file:
62+
yaml.safe_dump({'data_central': centrals}, file, sort_keys=False)
63+
64+
65+
def read_kinematics_lengths(tables: list) -> list:
66+
ndata = list()
67+
for table_id in tables:
68+
with open(f'rawdata/table_{table_id}.yaml', 'r') as file:
69+
working_dict = yaml.safe_load(file)
70+
ndata.append(len(working_dict['independent_variables'][0]['values']))
71+
return ndata
72+
73+
74+
def read_correlation_matrix(table_no: int) -> list:
75+
corr_mat = list()
76+
with open(f'rawdata/table_{table_no}.yaml', 'r') as file:
77+
temp_mat = yaml.safe_load(file)['dependent_variables'][0]['values']
78+
corr_mat = [small_dict['value'] for small_dict in temp_mat]
79+
return corr_mat
80+
81+
82+
def read_rel_errors(tables: list) -> list:
83+
errors = list()
84+
for table_num in tables:
85+
temp_list = list()
86+
with open(f'rawdata/table_{table_num}.yaml', 'r') as file:
87+
temp_list = yaml.safe_load(file)['dependent_variables'][0]['values']
88+
errors_in_table = list()
89+
for data_point in temp_list:
90+
errors_in_table.append(data_point['errors'])
91+
errors += errors_in_table
92+
return errors
93+
94+
95+
def make_errors_absolute(errors: list, centrals: list):
96+
if len(errors) != len(centrals):
97+
print('lengths dont match')
98+
else:
99+
abs_errors = list()
100+
for i in range(len(errors)):
101+
c_val = centrals[i]
102+
extracted_errors_at_dp = {small_dict['label']: c_val*float(small_dict['symerror'][:-1])/100 for small_dict in errors[i]}
103+
abs_errors.append(extracted_errors_at_dp)
104+
return abs_errors
105+
106+
107+
def generate_stat_art_unc(abs_errors, kin_lengths):
108+
stat_errors = [item['stat'] for item in abs_errors]
109+
split_indices = [kin_lengths[0]]+[0]*(len(kin_lengths)-1)
110+
for i in range(1, len(kin_lengths)):
111+
split_indices[i] = split_indices[i-1]+kin_lengths[i]
112+
split_indices = [0]+split_indices
113+
art_unc = list()
114+
for i in range(1, len(split_indices)):
115+
current_errors = stat_errors[split_indices[i-1]:split_indices[i]]
116+
current_corr_mat = read_correlation_matrix(i+6)
117+
current_ndata = kin_lengths[i-1]
118+
if not (len(current_errors) == current_ndata and len(current_corr_mat) == current_ndata**2):
119+
print('lengths not matching:')
120+
else:
121+
current_cov_mat = cormat_to_covmat(err_list=current_errors, cormat_list=current_corr_mat)
122+
current_art_unc = covmat_to_artunc(ndata = current_ndata, covmat_list = current_cov_mat)
123+
big_art_unc = []
124+
for small_row in current_art_unc:
125+
big_art_unc.append([0]*split_indices[i-1]+small_row+[0]*(122-split_indices[i]))
126+
art_unc += big_art_unc
127+
return art_unc
128+
129+
130+
def aggregate_uncertainties(abs_errors, art_unc):
131+
all_uncertainties = []
132+
for i in range(len(abs_errors)):
133+
current_dict = abs_errors[i]
134+
current_dict.pop('stat')
135+
art_unc_list = art_unc[i]
136+
art_unc_dict = {f'stat_art_unc_{j+1}': art_unc_list[j] for j in range(len(art_unc_list))}
137+
total_dict = current_dict | art_unc_dict
138+
all_uncertainties.append(total_dict)
139+
return all_uncertainties
140+
141+
142+
def dump_uncertainties(all_unc):
143+
singular_art_unc_desc = {'description': 'artificial uncertainty originating correlated statistical uncertainties',
144+
'treatment': 'ADD',
145+
'type': 'CORR'}
146+
all_art_unc_desc = {f'stat_art_unc_{i+1}': singular_art_unc_desc.copy() for i in range(122)}
147+
other_unc = {
148+
'uncor': {'description': 'stems from residual effects of small inefficiencies in the jet identification',
149+
'treatment': 'ADD',
150+
'type': 'UNCORR'},
151+
'jererr': {'description': 'jet energy resolution',
152+
'treatment': 'MULT',
153+
'type': 'CORR'},
154+
'lumi': {'description': 'luminosity uncertainty',
155+
'treatment': 'MULT',
156+
'type': 'CMSLUMI19P7'},
157+
'nongaussiantails': {'description': 'non-Gaussian tails in detector response to jets',
158+
'treatment': 'MULT',
159+
'type': 'CORR'},
160+
'AbsoluteScale': {'description': 'absolute jet energy scale calibration',
161+
'treatment': 'MULT',
162+
'type': 'CORR'},
163+
'AbsoluteStat': {'description': 'statistical uncertainty of absolute JES',
164+
'treatment': 'MULT',
165+
'type': 'CORR'},
166+
'AbsoluteMPFBias': {'description': 'bias in MPF response method',
167+
'treatment': 'MULT',
168+
'type': 'CORR'},
169+
'Fragmentation': {'description': 'fragmentation uncertainty',
170+
'treatment': 'MULT',
171+
'type': 'CORR'},
172+
'SinglePionECAL': {'description': 'e-calorimeter response to single pions',
173+
'treatment': 'MULT',
174+
'type': 'CORR'},
175+
'SinglePionHCAL': {'description': 'h-calorimeter response to single pions',
176+
'treatment': 'MULT',
177+
'type': 'CORR'},
178+
'FlavorQCD': {'description': 'jet flavour composition uncertainty',
179+
'treatment': 'MULT',
180+
'type': 'CORR'},
181+
'RelativeJEREC1': {'description': 'JER relative uncertainty',
182+
'treatment': 'MULT',
183+
'type': 'CORR'},
184+
'RelativeJEREC2': {'description': 'JER relative uncertainty',
185+
'treatment': 'MULT',
186+
'type': 'CORR'},
187+
'RelativeJERHF': {'description': 'JER relative uncertainty',
188+
'treatment': 'MULT',
189+
'type': 'CORR'},
190+
'RelativePtBB': {'description': 'Relative JES vs pT',
191+
'treatment': 'MULT',
192+
'type': 'CORR'},
193+
'RelativePtEC1': {'description': 'Relative JES vs pT',
194+
'treatment': 'MULT',
195+
'type': 'CORR'},
196+
'RelativePtEC2': {'description': 'Relative JES vs pT',
197+
'treatment': 'MULT',
198+
'type': 'CORR'},
199+
'RelativePtHF': {'description': 'Relative JES vs pT',
200+
'treatment': 'MULT',
201+
'type': 'CORR'},
202+
'RelativeFSR': {'description': 'Final-state radiation modeling',
203+
'treatment': 'MULT',
204+
'type': 'CORR'},
205+
'RelativeStatEC2': {'description': 'Relative JES statistical uncertainty',
206+
'treatment': 'MULT',
207+
'type': 'CORR'},
208+
'RelativeStatHF': {'description': 'Relative JES statistical uncertainty',
209+
'treatment': 'MULT',
210+
'type': 'CORR'},
211+
'RelativeStatFSR': {'description': 'Relative JES statistical uncertainty',
212+
'treatment': 'MULT',
213+
'type': 'CORR'},
214+
'PileUpDataMC': {'description': 'Data–MC pileup mismatch',
215+
'treatment': 'MULT',
216+
'type': 'CORR'},
217+
'PileUpPtRef': {'description': 'Pileup pT reference uncertainty',
218+
'treatment': 'MULT',
219+
'type': 'CORR'},
220+
'PileUpPtBB': {'description': 'Pileup pT uncertainty',
221+
'treatment': 'MULT',
222+
'type': 'CORR'},
223+
'PileUpPtEC1': {'description': 'Pileup pT uncertainty',
224+
'treatment': 'MULT',
225+
'type': 'CORR'},
226+
'PileUpPtEC2': {'description': 'Pileup pT uncertainty',
227+
'treatment': 'MULT',
228+
'type': 'CORR'},
229+
'PileUpPtHF': {'description': 'Pileup pT uncertainty',
230+
'treatment': 'MULT',
231+
'type': 'CORR'}
232+
}
233+
definitions = {'definitions': other_unc | all_art_unc_desc}
234+
uncertainties_yaml = definitions | {'bins': all_unc}
235+
with open('uncertainties.yaml', 'w') as file:
236+
yaml.safe_dump(uncertainties_yaml, file, sort_keys = False)
237+
238+
def main_filter():
239+
tables = read_metadata()[0][:6]
240+
bins, centrals = read_kinematics_and_centrals(tables)
241+
kin_lengths = read_kinematics_lengths(tables)
242+
errors = read_rel_errors([1,2,3,4,5,6])
243+
abs_errors = make_errors_absolute(errors, centrals)
244+
art_unc = generate_stat_art_unc(abs_errors, kin_lengths)
245+
all_unc = aggregate_uncertainties(abs_errors, art_unc)
246+
247+
dump_kinematics_and_centrals(bins, centrals)
248+
dump_uncertainties(all_unc)
249+
250+
if __name__ == '__main__':
251+
main_filter()

0 commit comments

Comments
 (0)