MetaDetect Ingester for DP2

txpipe/binning/random_forest.py

@@ -83,7 +83,7 @@ def apply_classifier(classifier, features, bands, shear_catalog_type, shear_data
         prefixes = ["", "", "", "", ""]
         suffixes = ["", "_1p", "_2p", "_1m", "_2m"]
     elif shear_catalog_type == "metadetect":
-        prefixes = ["00/", "1p/", "2p/", "1m/", "2m/"]
+        prefixes = ["ns/", "1p/", "2p/", "1m/", "2m/"]
         suffixes = ["", "", "", "", ""]
     else:
         prefixes = [""]

txpipe/calibrate.py

@@ -89,9 +89,9 @@ def run(self):
 
             # cat_cols is everything we are reading in
             if cat_type == "metadetect":
-                cat_cols = cat_cols + [f"00/{c}" for c in extra_cols + mag_cols_in]
-                mag_cols_in = [f"00/{c}" for c in mag_cols_in]
-                renames.update({f"00/{c}": c for c in extra_cols})
+                cat_cols = cat_cols + [f"ns/{c}" for c in extra_cols + mag_cols_in]
+                mag_cols_in = [f"ns/{c}" for c in mag_cols_in]
+                renames.update({f"ns/{c}": c for c in extra_cols})
             else:
                 cat_cols = cat_cols + extra_cols + mag_cols_in
 

txpipe/data_types.py

@@ -73,29 +73,29 @@ def catalog_type(self):
 
     def get_size(self):
         if self.catalog_type == "metadetect":
-            return self.file["shear/00/ra"].size
+            return self.file["shear/ns/ra"].size
         else:
             return self.file["shear/ra"].size
 
     def get_primary_catalog_group(self):
         if self.catalog_type == "metadetect":
-            return "shear/00"
+            return "shear/ns"
         else:
             return "shear"
 
     def get_true_redshift_column(self):
         if self.catalog_type == "metadetect":
-            return "00/redshift_true"
+            return "ns/redshift_true"
         else:
             return "redshift_true"
 
     def get_primary_catalog_names(self, true_shear=False):
         if true_shear:
             if self.catalog_type == "metadetect":
-                shear_cols = ["00/true_g1", "00/true_g2", "00/ra", "00/dec", "00/weight"]
+                shear_cols = ["ns/true_g1", "ns/true_g2", "ns/ra", "ns/dec", "ns/weight"]
                 rename = {c: c[3:] for c in shear_cols}
-                rename["00/true_g1"] = "g1"
-                rename["00/true_g2"] = "g2"
+                rename["ns/true_g1"] = "g1"
+                rename["ns/true_g2"] = "g2"
             else:
                 rename = {"true_g1": "g1", "true_g2": "g2"}
                 rename = {}
@@ -106,7 +106,7 @@ def get_primary_catalog_names(self, true_shear=False):
             shear_cols = ["g1", "g2", "c1", "c2", "ra", "dec", "weight"]
             rename = {}
         elif self.catalog_type == "metadetect":
-            shear_cols = ["00/g1", "00/g2", "00/ra", "00/dec", "00/weight"]
+            shear_cols = ["ns/g1", "ns/g2", "ns/ra", "ns/dec", "ns/weight"]
             rename = {c: c[3:] for c in shear_cols}
         else:
             shear_cols = ["g1", "g2", "ra", "dec", "weight"]

txpipe/diagnostics.py

@@ -237,7 +237,10 @@ def run(self):
                 "m",
             ] + [f"mag_{b}" for b in self.config["bands"]]
 
-        shear_tomo_cols = ["bin"]
+        if self.config["shear_catalog_type"] == "metadetect":
+            shear_tomo_cols = ["bin_ns", "bin_1p", "bin_1m", "bin_2p", "bin_2m"]
+        else:
+            shear_tomo_cols = ["bin"]
 
         if self.config["shear_catalog_type"] == "metacal":
             more_iters = ["shear_tomography_catalog", "response", ["R_gamma"]]
@@ -292,7 +295,7 @@ def plot_psf_shear(self):
         delta_gamma = self.config["delta_gamma"]
 
         psf_g_edges = self.get_bin_edges("psf_g1")
-        shear_prefix = "00/" if self.config["shear_catalog_type"] == "metadetect" else ""
+        shear_prefix = "ns/" if self.config["shear_catalog_type"] == "metadetect" else ""
 
         p1 = MeanShearInBins(
             f"{shear_prefix}psf_g1",
@@ -391,7 +394,7 @@ def plot_psf_size_shear(self):
         delta_gamma = self.config["delta_gamma"]
 
         psf_T_edges = self.get_bin_edges("psf_T_mean")
-        shear_prefix = "00/" if self.config["shear_catalog_type"] == "metadetect" else ""
+        shear_prefix = "ns/" if self.config["shear_catalog_type"] == "metadetect" else ""
 
         binnedShear = MeanShearInBins(
             f"{shear_prefix}psf_T_mean",
@@ -451,7 +454,7 @@ def plot_snr_shear(self):
 
         # Parameters of the binning in SNR
         shear_catalog_type = self.config["shear_catalog_type"]
-        shear_prefix = "00/" if shear_catalog_type == "metadetect" else ""
+        shear_prefix = "ns/" if shear_catalog_type == "metadetect" else ""
         delta_gamma = self.config["delta_gamma"]
 
         snr_edges = self.get_bin_edges("s2n")
@@ -516,7 +519,7 @@ def plot_size_shear(self):
         from scipy import stats
 
         shear_catalog_type = self.config["shear_catalog_type"]
-        shear_prefix = "00/" if shear_catalog_type == "metadetect" else ""
+        shear_prefix = "ns/" if shear_catalog_type == "metadetect" else ""
         delta_gamma = self.config["delta_gamma"]
 
         T_edges = self.get_bin_edges("T")
@@ -580,7 +583,7 @@ def plot_mag_shear(self):
         from scipy import stats
 
         shear_catalog_type = self.config["shear_catalog_type"]
-        shear_prefix = "00/" if shear_catalog_type == "metadetect" else ""
+        shear_prefix = "ns/" if shear_catalog_type == "metadetect" else ""
         delta_gamma = self.config["delta_gamma"]
         nbins = self.config["nbins"]
 
@@ -712,16 +715,16 @@ def plot_g_histogram(self):
             if data is None:
                 break
 
-            qual_cut = data["bin"] != -1
+            #qual_cut = data["bin"] != -1
 
             if cat_type == "metacal":
                 g1 = data["g1"]
                 g2 = data["g2"]
                 w = data["weight"]
             elif cat_type == "metadetect":
-                g1 = data["00/g1"]
-                g2 = data["00/g2"]
-                w = data["00/weight"]
+                g1 = data["ns/g1"]
+                g2 = data["ns/g2"]
+                w = data["ns/weight"]
             elif cat_type == "lensfit":
                 dec = data["dec"]
                 g1 = data["g1"]
@@ -790,7 +793,8 @@ def plot_snr_histogram(self):
 
         delta_gamma = self.config["delta_gamma"]
         shear_catalog_type = self.config["shear_catalog_type"]
-        shear_prefix = "00/" if shear_catalog_type == "metadetect" else ""
+        shear_prefix = "ns/" if shear_catalog_type == "metadetect" else ""
+        bin_type = "bin_ns" if self.config["shear_catalog_type"] == "metadetect" else "bin"
         bins = 10
         edges = np.logspace(1, 3, bins + 1)
         mids = 0.5 * (edges[1:] + edges[:-1])
@@ -802,7 +806,7 @@ def plot_snr_histogram(self):
             if data is None:
                 break
 
-            qual_cut = data["bin"] != -1
+            qual_cut = data[bin_type] != -1
 
             b1 = np.digitize(data[f"{shear_prefix}s2n"][qual_cut], edges) - 1
 
@@ -979,7 +983,8 @@ def plot_mag_histograms(self):
         mid = 0.5 * (edges[1:] + edges[:-1])
         width = edges[1] - edges[0]
         bands = self.config["bands"]
-        shear_prefix = "00/" if self.config["shear_catalog_type"] == "metadetect" else ""
+        shear_prefix = "ns/" if self.config["shear_catalog_type"] == "metadetect" else ""
+        bin_type = "bin_ns" if self.config["shear_catalog_type"] == "metadetect" else "bin"
         nband = len(bands)
         full_hists = [np.zeros(size, dtype=int) for b in bands]
         source_hists = [np.zeros(size, dtype=int) for b in bands]
@@ -998,7 +1003,7 @@ def plot_mag_histograms(self):
                     count = w.sum()
                     h1[i] += count
 
-                    w &= data["bin"] >= 0
+                    w &= data[bin_type] >= 0
                     count = w.sum()
                     h2[i] += count
 

txpipe/ingest/__init__.py

@@ -11,3 +11,4 @@
 )
 from .dp1 import TXIngestDataPreview1
 from .stage3 import TXIngestDESY3Gold
+from .metaDetect import TXIngestRubinMetaDetect

txpipe/ingest/lsst.py

@@ -64,3 +64,55 @@ def process_shear_data(data):
     output["psf_g2"] = psf_g2
 
     return output
+
+
+def process_metadetect_data(data):
+    output = {}
+    for variant in ["ns", "1p", "1m", "2p", "2m"]:
+        var_data = data[data["metaStep"] == variant]
+        var_data = sanitize(var_data)
+
+        var_output = {
+            "ra": var_data["ra"],
+            "dec": var_data["dec"],
+            "id": var_data["shearObjectId"],
+            "object_mask_fraction": var_data["mfrac"],
+            #"n_epoch": var_data["nEpochCell"],
+            "g1": var_data["gauss_g1"],
+            "g2": var_data["gauss_g2"],
+            "g1_err": var_data["gauss_g1_g1_Cov"],
+            "g2_err": var_data["gauss_g2_g2_Cov"],
+            "g_cross": var_data["gauss_g1_g2_Cov"],
+            "T": var_data["gauss_T"],
+            "s2n": var_data["gauss_snr"],
+            "T_err": var_data["gauss_TErr"],
+            "psf_g1_original": var_data["psfOriginal_e1"],
+            "psf_g2_original": var_data["psfOriginal_e2"],
+            "psf_T_mean_original": var_data["psfOriginal_T"],
+            "psf_g1": var_data["gauss_psfReconvolved_g1"],
+            "psf_g2": var_data["gauss_psfReconvolved_g1"],
+            "psf_T_mean": var_data["gauss_psfReconvolved_T"],
+            "flags": var_data["gauss_shape_flags"], # TO BE ADDRESSED!
+            "weight": 1 / (0.5 * (var_data["gauss_g1_g1_Cov"] + var_data["gauss_g2_g2_Cov"])),
+        }
+        for band in "gri": # For DP2, we only expect 4 bands
+            f = var_data[f"{band}_pgaussFlux"]
+            f_err = var_data[f"{band}_pgaussFluxErr"]
+            var_output[f"mag_{band}"] = nanojansky_to_mag_ab(f)
+            var_output[f"mag_err_{band}"] = nanojansky_err_to_mag_ab(f, f_err)
+        output[f"{variant}"] = var_output
+
+    return output
+
+def sanitize(data):
+    """
+    Convert unicode arrays into types that h5py can save
+    """
+    # convert unicode to strings
+    if data.dtype.kind == "U":
+        data = data.astype("S")
+    # convert dates to integers
+    elif data.dtype.kind == "M":
+        data = data.astype(int)
+
+    return data