Merge pull request #2362 from NNPDF/ATLAS_ZPT_8TEV_variant

ATLAS ZpT 8 TeV uncertainty variant

Author: scarlehoff

nnpdf_data/nnpdf_data/commondata/ATLAS_Z0J_8TEV/filter.py

@@ -1,5 +1,6 @@
 import logging
 
+from filter_decorr_unc import filter_unc_ATLAS_Z0J_8TEV
 from filter_utils import Extractor
 import numpy as np
 import yaml
@@ -75,6 +76,9 @@ def check_dat_with_legacy(observable, rtol=1e-03):
     ATLAS_Z0J_8TEV_PT_M.generate_uncertainties('./rawdata/hepdata/unnormalized/output')
     ATLAS_Z0J_8TEV_PT_M.generate_uncertainties('./rawdata/hepdata/unnormalized/output', True)
 
+    filter_unc_ATLAS_Z0J_8TEV(MC=False)
+    filter_unc_ATLAS_Z0J_8TEV(MC=True)
+
     # Comparing new anf legacy uncertainties
     # Deprecated
     if CHECK:

nnpdf_data/nnpdf_data/commondata/ATLAS_Z0J_8TEV/filter_decorr_unc.py

@@ -0,0 +1,132 @@
+"""
+This piece of code generates a simplified variant of the uncertainties.yaml
+files in which the 99 correlated systematic uncertainties of the legacy
+version are replaced with two systematic uncertainties, one that is uncorrelated
+and that is correlated. Statistical, luminosity and )optional) Monte Carlo
+uncertainties are as in the legacy version.
+"""
+
+import yaml
+
+
+def get_tables(observable):
+    """
+    get the Hepdata tables, given the tables and version specified in metadata
+    """
+    prefix = "rawdata/Table"
+    with open("metadata.yaml", "r") as file:
+        metadata = yaml.safe_load(file)
+
+    if observable == "PT-Y":
+        tables = metadata["implemented_observables"][0]["tables"]
+    elif observable == "PT-M":
+        tables = metadata["implemented_observables"][1]["tables"]
+    else:
+        print("Data set not implemented")
+        print("Available data sets are:")
+        print("- ATLAS_Z0J_8TEV_PT-Y")
+        print("- ATLAS_Z0J_8TEV_PT-M")
+        exit()
+
+    hepdata_tables = []
+
+    for table in tables:
+        hepdata_tables.append(f"{prefix}{table}.yaml")
+
+    return hepdata_tables
+
+
+def get_uncertainties(observable):
+    """
+    Returns uncertainties for dumping in the -yaml file
+    """
+    data_central = []
+    uncertainties = []
+
+    hepdata_tables = get_tables(observable)
+    for table in hepdata_tables:
+        with open(table, 'r') as f:
+            input = yaml.safe_load(f)
+        # Central values
+        data_values = input["dependent_variables"][5]["values"]
+        for data_value in data_values:
+            data_central.append(data_value["value"])
+
+        # Uncertainties
+        for data_value in data_values:
+            errors = data_value["errors"]
+            uncertainty = {}
+            for error in errors:
+                uncertainty[error["label"]] = error["symerror"]
+                uncertainty.update(uncertainty)
+
+            uncertainties.append(uncertainty)
+
+    return (data_central, uncertainties)
+
+
+def filter_unc_ATLAS_Z0J_8TEV(MC=False):
+    """
+    Dumps uncertainties on .yaml files
+    """
+    lumi_unc = 2.8  # %
+    mc_unc = 1.0  # %
+    observables = ["PT-Y", "PT-M"]
+    for observable in observables:
+        if MC == False:
+            unc_file = "uncertainties_decorr_" + observable + ".yaml"
+        else:
+            unc_file = "uncertainties_decorr_sys_10_" + observable + ".yaml"
+        central_values, uncertainties = get_uncertainties(observable)
+
+        for i in range(len(central_values)):
+            for k in uncertainties[i]:
+                uncertainties[i][k] = (
+                    float(uncertainties[i][k].replace("%", "")) / 100.0 * central_values[i] * 1000.0
+                )
+            uncertainties[i].update({"sys_lumi_corr": lumi_unc / 100 * central_values[i] * 1000.0})
+            if MC == True:
+                uncertainties[i].update(
+                    {"sys_mc_uncorr": mc_unc / 100 * central_values[i] * 1000.0}
+                )
+
+        treatment = {
+            "stat": "ADD",
+            "sys,Uncorrelated": "ADD",
+            "sys,Correlated": "MULT",
+            "sys_lumi_corr": "MULT",
+        }
+        correlation = {
+            "stat": "UNCORR",
+            "sys,Uncorrelated": "UNCORR",
+            "sys,Correlated": "CORR",
+            "sys_lumi_corr": "ATLASLUMI12",
+        }
+        if MC == True:
+            treatment.update({"sys_mc_uncorr": "ADD"})
+            correlation.update({"sys_mc_uncorr": "UNCORR"})
+
+        definitions = {}
+        for key, value in uncertainties[0].items():
+            definition = {
+                key: {
+                    "description": key + " unc. from HepData",
+                    "treatment": treatment[key],
+                    "type": correlation[key],
+                }
+            }
+            definitions.update(definition)
+            uncertainties_yaml = {"definitions": definitions, "bins": uncertainties}
+
+        if MC:
+            uncertainties_yaml["definitions"][key][
+                "description"
+            ] = "extra Monte Carlo statistical uncertainty"
+
+        with open(unc_file, "w") as file:
+            yaml.dump(uncertainties_yaml, file, sort_keys=False)
+
+
+if __name__ == "__main__":
+    filter_unc_ATLAS_Z0J_8TEV(MC=False)
+    filter_unc_ATLAS_Z0J_8TEV(MC=True)

nnpdf_data/nnpdf_data/commondata/ATLAS_Z0J_8TEV/metadata.yaml

@@ -69,7 +69,13 @@ implemented_observables:
     legacy_10:
       data_uncertainties:
       - uncertainties_legacy_sys_10_PT-Y.yaml
-
+    decorr:
+      data_uncertainties:
+      - uncertainties_decorr_PT-Y.yaml
+    decorr_sys_10:
+      data_uncertainties:
+      - uncertainties_decorr_sys_10_PT-Y.yaml
+      
 - observable_name: PT-M
   observable:
     description: Drell-Yan Transverse Momentum Distribution
@@ -143,3 +149,9 @@ implemented_observables:
     legacy_data_10:
       data_uncertainties:
       - uncertainties_legacy_sys_10_PT-M.yaml
+    decorr:
+      data_uncertainties:
+      - uncertainties_decorr_PT-M.yaml
+    decorr_sys_10:
+      data_uncertainties:
+      - uncertainties_decorr_sys_10_PT-M.yaml