U/eac/merge spec

src/rail/cli/rail_project/__init__.py

@@ -9,7 +9,6 @@
     run_group,
 )
 
-
 __all__ = [
     "project_cli",
     "inspect_command",

src/rail/cli/rail_project/project_commands.py

@@ -131,6 +131,46 @@ def split_command(
     return ok
 
 
+@project_cli.command(name="merge")
+@project_options.config_file()
+@project_options.run_mode()
+@project_options.catalog_template()
+@project_options.output_catalog_template()
+@project_options.merge_name()
+@project_options.merger_class_name()
+@project_options.selection()
+@project_options.flavor()
+def merge_command(
+    config_file: str, run_mode: project_options.RunMode, **kwargs: Any
+) -> int:
+    """Merged files from a particular catalog
+
+    This will:
+    create a catalog of input files from the catalog_template,
+    flavor, selection and basename parameters,
+    be merging varous input files
+    """
+
+    if run_mode == project_options.RunMode.slurm:
+        raise NotImplementedError("subsample_command not set up to run under slurm")
+
+    project = RailProject.load_config(config_file)
+    flavors = project.get_flavor_args(kwargs.pop("flavor"))
+    selections = project.get_selection_args(kwargs.pop("selection"))
+    iter_kwargs = project.generate_kwargs_iterable(flavor=flavors, selection=selections)
+
+    dry_run = run_mode == project_options.RunMode.dry_run
+
+    ok = 0
+    for kw in iter_kwargs:
+        _output_path = project.merge_data(
+            dry_run=dry_run,
+            **kw,
+            **kwargs,
+        )
+    return ok
+
+
 @project_cli.command(name="subsample")
 @project_options.config_file()
 @project_options.run_mode()
@@ -141,6 +181,8 @@ def split_command(
 @project_options.selection()
 @project_options.flavor()
 @project_options.basename()
+@project_options.label()
+@project_options.cone_cut()
 def subsample_command(
     config_file: str, run_mode: project_options.RunMode, **kwargs: Any
 ) -> int:
@@ -158,6 +200,12 @@ def subsample_command(
     if run_mode == project_options.RunMode.slurm:
         raise NotImplementedError("subsample_command not set up to run under slurm")
 
+    cone_cut_str = kwargs.pop('cone_cut')
+    if cone_cut_str:
+        tokens = cone_cut_str.replace('[', '').replace(']', '').split(',')
+        cone_cut = [float(tokens[0]), float(tokens[1]), float(tokens[2])]
+        kwargs['cone_cut'] = cone_cut
+
     project = RailProject.load_config(config_file)
     flavors = project.get_flavor_args(kwargs.pop("flavor"))
     selections = project.get_selection_args(kwargs.pop("selection"))
@@ -195,6 +243,7 @@ def subsample_command(
 @project_options.reducer_class_name()
 @project_options.input_selection()
 @project_options.selection()
+@project_options.sim_version()
 def reduce_command(
     config_file: str, run_mode: project_options.RunMode, **kwargs: Any
 ) -> int:
@@ -233,6 +282,7 @@ def run_group() -> None:
 @project_options.selection()
 @project_options.flavor()
 @project_options.run_mode()
+@project_options.convert_output()
 @project_options.site()
 def photmetric_errors_pipeline(config_file: str, **kwargs: Any) -> int:
     """Run the photometric errors analysis pipeline"""
@@ -257,6 +307,7 @@ def photmetric_errors_pipeline(config_file: str, **kwargs: Any) -> int:
 @project_options.selection()
 @project_options.flavor()
 @project_options.run_mode()
+@project_options.convert_output()
 @project_options.site()
 def prepare_pipeline(config_file: str, **kwargs: Any) -> int:
     """Run the truth-to-observed data pipeline"""
@@ -281,6 +332,7 @@ def prepare_pipeline(config_file: str, **kwargs: Any) -> int:
 @project_options.selection()
 @project_options.flavor()
 @project_options.run_mode()
+@project_options.convert_output()
 @project_options.site()
 def truth_to_observed_pipeline(config_file: str, **kwargs: Any) -> int:
     """Run the truth-to-observed data pipeline"""
@@ -305,6 +357,7 @@ def truth_to_observed_pipeline(config_file: str, **kwargs: Any) -> int:
 @project_options.selection()
 @project_options.flavor()
 @project_options.run_mode()
+@project_options.convert_output()
 @project_options.site()
 def blending_pipeline(config_file: str, **kwargs: Any) -> int:
     """Run the blending analysis pipeline"""
@@ -329,6 +382,7 @@ def blending_pipeline(config_file: str, **kwargs: Any) -> int:
 @project_options.selection()
 @project_options.flavor()
 @project_options.run_mode()
+@project_options.convert_output()
 @project_options.site()
 def spectroscopic_selection_pipeline(config_file: str, **kwargs: Any) -> int:
     """Run the spectroscopic selection data pipeline"""
@@ -395,6 +449,33 @@ def estimate_single(config_file: str, **kwargs: Any) -> int:
         )
     return ok
 
+@run_group.command(name="estimate-many")
+@project_options.config_file()
+@project_options.selection()
+@project_options.flavor()
+@project_options.basename()
+@project_options.run_mode()
+@project_options.site()
+def estimate_many(config_file: str, **kwargs: Any) -> int:
+    """Run the spectroscopic selection data pipeline"""
+    project = RailProject.load_config(config_file)
+    flavors = project.get_flavor_args(kwargs.pop("flavor"))
+    selections = project.get_selection_args(kwargs.pop("selection"))
+    the_basename = kwargs.pop("basename")
+    iter_kwargs = project.generate_kwargs_iterable(flavor=flavors, selection=selections)
+    ok = 0
+    pipeline_name = "estimate"
+
+    for kw in iter_kwargs:
+        kw.update(basename=the_basename)
+        ok |= project.run_pipeline_catalog(
+            pipeline_name,
+            **kw,
+            **kwargs,
+        )
+    return ok
+
+
 
 @run_group.command(name="evaluate")
 @project_options.config_file()

src/rail/cli/rail_project/project_options.py

@@ -7,7 +7,9 @@
     "RunMode",
     "args",
     "basename",
+    "cone_cut",
     "config_path",
+    "convert_output",
     "catalog_template",
     "file_template",
     "force",
@@ -18,6 +20,8 @@
     "input_tag",
     "label",
     "maglim",
+    "merge_name",
+    "merger_class_name",
     "model_dir",
     "model_name",
     "model_path",
@@ -27,6 +31,7 @@
     "reducer_class_name",
     "run_mode",
     "selection",
+    "sim_version",
     "site",
     "splitter_class_name",
     "subsampler_class_name",
@@ -61,12 +66,24 @@
 )
 
 
+cone_cut = PartialOption(
+    "--cone-cut",
+    help="[RA, DEC, SIZE] in degrees for a cone selection",
+    type=str,
+)
+
+
 config_path = PartialOption(
     "--config_path",
     help="Path to configuration file",
     type=click.Path(),
 )
 
+convert_output = PartialOption(
+    "--convert-output/--no-convert-output",
+    help="Convert outputfiles",
+    default=True,
+)
 
 catalog_template = PartialOption(
     "--catalog-template",
@@ -99,6 +116,8 @@
 label = PartialOption(
     "--label",
     help="File label (e.g., 'test' or 'train')",
+    required=False,
+    default=None,
     type=str,
 )
 
@@ -189,6 +208,20 @@
 )
 
 
+merge_name = PartialOption(
+    "--merge-name",
+    help="Configuration used to merge",
+    type=str,
+)
+
+
+merger_class_name = PartialOption(
+    "--merger-class-name",
+    help="Class for the merger",
+    type=str,
+)
+
+
 model_dir = PartialOption(
     "--model-dir",
     help="Path to directory with model files",
@@ -238,13 +271,21 @@
     help="Mode to run script",
 )
 
+sim_version = PartialOption(
+    "--sim-version",
+    type=str,
+    help="Optional override to simulation version",
+)
+
+
 size = PartialOption(
     "--size",
     type=int,
     default=100_000,
     help="Number of objects in file",
 )
 
+
 test_file_template = PartialOption(
     "--test-file-template",
     type=str,

src/rail/plotting/data_extraction_funcs.py

@@ -7,7 +7,7 @@
 import numpy as np
 import qp
 import tables_io
-from rail.utils.catalog_utils import CatalogConfigBase
+from rail.utils import catalog_utils
 
 from rail.projects import RailProject, path_funcs
 
@@ -322,9 +322,11 @@ def get_pz_point_estimate_data(
     z_estimate_data = extract_z_point(z_estimate_path)
     flavor_info = project.get_flavor(flavor)
     catalog_tag = flavor_info["catalog_tag"]
-    CatalogConfigBase.apply(catalog_tag)
-    catalog_class = CatalogConfigBase.active_class()
-    ref_band = catalog_class.band_template.format(band=catalog_class.ref_band)
+    catalog_utils.apply(catalog_tag)
+    active_catalog = catalog_utils.get_active_tag()
+    ref_band = active_catalog.config.mag_column_template.format(
+        band=active_catalog.config.ref_band
+    )
     mag_data = extract_mag(z_true_path, colname=ref_band)
     pz_data = make_z_true_z_point_dict(z_true_data, z_estimate_data, mag_data)
     return pz_data
@@ -360,18 +362,20 @@ def get_ztrue_and_magntidues(
     """
     flavor_info = project.get_flavor(flavor)
     catalog_tag = flavor_info["catalog_tag"]
-    CatalogConfigBase.apply(catalog_tag)
-    catalog_class = CatalogConfigBase.active_class()
+    catalog_utils.apply(catalog_tag)
+    active_catalog = catalog_utils.get_active_tag()
 
     z_true_path = path_funcs.get_z_true_path(project, selection, flavor, tag)
-    z_true_data = extract_z_true(z_true_path, catalog_class.redshift_col)
+    z_true_data = extract_z_true(z_true_path, active_catalog.config.redshift_col)
     mag_data = extract_magnitudes(
-        z_true_path, catalog_class.band_template, catalog_class.bandlist
+        z_true_path,
+        active_catalog.config.mag_column_template,
+        active_catalog.config.band_list,
     )
     out_data = dict(
         truth=z_true_data,
         magnitudes=mag_data,
-        bands=catalog_class.bandlist,
+        bands=active_catalog.config.band_list,
     )
     return out_data
 

src/rail/plotting/dataset_holder.py

@@ -5,6 +5,7 @@
 
 from ceci.config import StageParameter
 from rail.core.configurable import Configurable
+from rail.utils import catalog_utils
 
 from rail.projects import RailProject
 from rail.projects.dynamic_class import DynamicClass
@@ -288,4 +289,6 @@ def resolve(self) -> RailProject:
         """Read the associated yaml file and create a RailProject"""
         if self._project is None:
             self._project = RailProject.load_config(self.config.yaml_file)
+            for catalog_lib in self._project.config.CatalogLib:
+                catalog_utils.load_yaml(catalog_lib)
         return self._project

src/rail/projects/__init__.py

@@ -8,5 +8,4 @@
 from . import library, name_utils
 from .project import RailFlavor, RailProject
 
-
 __all__ = ["library", "name_utils", "RailFlavor", "RailProject"]

src/rail/projects/algorithm_factory.py

@@ -10,6 +10,7 @@
     RailClassificationAlgorithmHolder,
     RailErrorModelAlgorithmHolder,
     RailPZAlgorithmHolder,
+    RailMergerAlgorithmHolder,
     RailReducerAlgorithmHolder,
     RailSpecSelectionAlgorithmHolder,
     RailSplitterAlgorithmHolder,
@@ -23,6 +24,7 @@
     "Classifiers",
     "Summarizers",
     "ErrorModels",
+    "Mergers",
     "Splitters",
     "Subsamplers",
     "Reducers",
@@ -65,6 +67,7 @@ class RailAlgorithmFactory(RailFactoryMixin):
         RailClassificationAlgorithmHolder,
         RailSummarizerAlgorithmHolder,
         RailErrorModelAlgorithmHolder,
+        RailMergerAlgorithmHolder,
         RailSplitterAlgorithmHolder,
         RailSubsamplerAlgorithmHolder,
         RailReducerAlgorithmHolder,