change flarelist to L3

Author: nicHoch

processed.sqlite

@@ -15,6 +15,7 @@ parallel_batchsize_L0 = 300
 parallel_batchsize_L1 = 300
 parallel_batchsize_L2 = 100
 flareid_sdc_lut_file = ./stixcore/data/publish/flarelist_sdc_lut.csv
+flareid_sc_lut_file = ./stixcore/data/publish/flarelist_sc_lut.csv
 [Publish]
 db_file = ./stixcore/data/publish/published.sqlite
 include_levels = L0

stixcore/data/stixcore.ini

@@ -1084,7 +1084,8 @@ def generate_filename(self, product, *, version=0, suffix='.svg'):
         Path
             a Path object with full name and path
         """
-        p = Path(super().generate_filename(product=product, version=version, header=True))
+        p = Path(super().generate_filename(product=product, version=version,
+                                           header=True, status='C'))
         return p.with_suffix(suffix).name
 
     def generate_primary_header(self, filename, product, *, version=0):

stixcore/io/FlareListManager.py

@@ -39,6 +39,22 @@ def __init__(self, source_dir: Path, output_dir: Path):
         self.source_dir = Path(source_dir)
         self.output_dir = Path(output_dir)
 
+    def get_processing_files(self, phs) -> list[Path]:
+        """Returns a list of all files which should be processed by this step.
+
+        Returns
+        -------
+        list[Path]
+            a list of fits files candidates
+        """
+        hk_in_files = []
+
+        for fc in self.find_processing_candidates():
+            tr = self.test_for_processing(fc, phs)
+            if tr == TestForProcessingResult.Suitable:
+                hk_in_files.append(fc)
+        return hk_in_files
+
     def find_processing_candidates(self) -> list[Path]:
         """Performs file pattern search in the source directory
 

stixcore/io/fits/processors.py

@@ -13,8 +13,8 @@
     SingleProductProcessingStepMixin,
     TestForProcessingResult,
 )
-from stixcore.products.ANC.flarelist import (
-    FlareListProduct,
+from stixcore.products.level3.flarelist import (
+    FlareList,
     FlarePeekPreviewMixin,
     FlarePositionMixin,
     FlareSOOPMixin,
@@ -30,7 +30,7 @@
 
 
 class FLtoFL(SingleProductProcessingStepMixin):
-    """Processing step from a HK L1 fits file to a solo_ANC_stix-asp-ephemeris*.fits file.
+    """Processing step from enhance monthly flare list files to next processing steep
     """
     INPUT_PATTERN = "solo_ANC_stix-flarelist-sdc*.fits"
 
@@ -84,8 +84,8 @@ def get_processing_files(self, phs: ProcessingHistoryStorage) -> list[Path]:
         return fl_to_process
 
     def test_for_processing(self, candidate: Path,
-                            product_in: FlareListProduct,
-                            product_out: FlareListProduct,
+                            product_in: FlareList,
+                            product_out: FlareList,
                             phm: ProcessingHistoryStorage) -> TestForProcessingResult:
         """_summary_
 

stixcore/processing/AspectANC.py

@@ -12,17 +12,17 @@
     SingleProductProcessingStepMixin,
     TestForProcessingResult,
 )
-from stixcore.products.ANC.flarelist import FlarePositionMixin, FlareSOOPMixin
+from stixcore.products.level3.flarelist import FlarePositionMixin, FlareSOOPMixin
 from stixcore.soop.manager import SOOPManager
 from stixcore.util.logging import get_logger
 
-__all__ = ['FlareListANC', 'FlarePositionMixin']
+__all__ = ['FlareListL3']
 
 logger = get_logger(__name__)
 
 
-class FlareListANC(SingleProductProcessingStepMixin):
-    """Processing step from a FlareListManager to monthly solo_ANC_stix-flarelist-*.fits file.
+class FlareListL3(SingleProductProcessingStepMixin):
+    """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file.
     """
 
     STARTDATE = date(2024, 1, 1)
@@ -49,7 +49,7 @@ def find_processing_months(self, phs: ProcessingHistoryStorage) -> list[date]:
             a list of fits files candidates
         """
         fl_months = list()
-        for month in pd.date_range(FlareListANC.STARTDATE, date.today(), freq='MS').date:
+        for month in pd.date_range(FlareListL3.STARTDATE, date.today(), freq='MS').date:
             if self.test_for_processing(month, phs) == TestForProcessingResult.Suitable:
                 fl_months.append(month)
         return fl_months
@@ -75,7 +75,7 @@ def test_for_processing(self, month: date,
                                                  self.flm.productCls.LEVEL,
                                                  self.flm.productCls.TYPE,
                                                  self.flm.productCls.get_cls_processing_version(),
-                                                 str(month), FlareListANC.STARTDATE)
+                                                 str(month), FlareListL3.STARTDATE)
 
             # found already in the processing history
             if wp:
@@ -93,7 +93,7 @@ def test_for_processing(self, month: date,
     def process_fits_files(self, months: list[date], *, soopmanager: SOOPManager,
                            spice_kernel_path: Path, processor, config) -> list[Path]:
         """Performs the processing (expected to run in a dedicated python process) from a
-        list of solo_L1_stix-hk-maxi_*.fits into solo_ANC_stix-asp-ephemeris*.fits files.
+        list of time periods (months) and creates a FITS file for each one.
 
         Parameters
         ----------

stixcore/processing/FLtoFL.py

@@ -7,6 +7,8 @@
 from datetime import date, timedelta
 from concurrent.futures import ProcessPoolExecutor
 
+from stixpy.net.client import STIXClient
+
 from stixcore.config.config import CONFIG
 from stixcore.ephemeris.manager import Spice, SpiceKernelManager
 from stixcore.io.fits.processors import (
@@ -15,17 +17,24 @@
     FitsL3Processor,
     PlotProcessor,
 )
-from stixcore.io.FlareListManager import SDCFlareListManager
+from stixcore.io.FlareListManager import SCFlareListManager, SDCFlareListManager
 from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage
 from stixcore.io.RidLutManager import RidLutManager
 from stixcore.processing.AspectANC import AspectANC
-from stixcore.processing.FlareListANC import FlareListANC
+from stixcore.processing.FlareListL3 import FlareListL3
 from stixcore.processing.FLtoFL import FLtoFL
 from stixcore.processing.LL import LL03QL
 from stixcore.processing.pipeline import PipelineStatus
-from stixcore.processing.SingleStep import SingleProcessingStepResult, TestForProcessingResult
-from stixcore.products.ANC.flarelist import FlarelistSDCLoc, FlarelistSDCLocImg
+from stixcore.processing.SingleStep import SingleProcessingStepResult
 from stixcore.products.level1.quicklookL1 import LightCurve
+from stixcore.products.level3.flarelist import (
+    FlarelistSC,
+    FlarelistSCLoc,
+    FlarelistSCLocImg,
+    FlarelistSDC,
+    FlarelistSDCLoc,
+    FlarelistSDCLocImg,
+)
 from stixcore.products.lowlatency.quicklookLL import LightCurveL3
 from stixcore.soop.manager import SOOPManager
 from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger
@@ -184,6 +193,13 @@ def run_daily_pipeline(args):
 
         Path(CONFIG.get('Paths', 'fits_archive'))
 
+        fido_url = CONFIG.get('Paths', 'fido_search_url',
+                              fallback='https://pub099.cs.technik.fhnw.ch/data/fits')
+        fido_client = STIXClient(source=fido_url)
+
+        flare_lut_file = Path(CONFIG.get('Pipeline', 'flareid_sc_lut_file'))
+        SCFlareListManager.instance = SCFlareListManager(flare_lut_file, fido_client, update=True)
+
         flare_lut_file = Path(CONFIG.get('Pipeline', 'flareid_sdc_lut_file'))
         SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False)
 
@@ -218,11 +234,14 @@ def run_daily_pipeline(args):
 
         aspect_anc_processor = AspectANC(fits_in_dir, fits_out_dir)
 
-        flarelist_sdc = FlareListANC(SDCFlareListManager.instance, fits_out_dir)
+        flarelist_sdc = FlareListL3(SDCFlareListManager.instance, fits_out_dir)
+        flarelist_sc = FlareListL3(SCFlareListManager.instance, fits_out_dir)
         fl_to_fl = FLtoFL(fits_in_dir,
                           fits_out_dir,
-                          products_in_out=[  # (FlarelistSDC, FlarelistSDCLoc),
-                                           (FlarelistSDCLoc, FlarelistSDCLocImg)],
+                          products_in_out=[(FlarelistSDC, FlarelistSDCLoc),
+                                           (FlarelistSDCLoc, FlarelistSDCLocImg),
+                                           (FlarelistSC, FlarelistSCLoc),
+                                           (FlarelistSCLoc, FlarelistSCLocImg)],
                           cadence=timedelta(seconds=1))
 
         ll03ql = LL03QL(fits_in_dir, fits_out_dir, in_product=LightCurve, out_product=LightCurveL3,
@@ -233,23 +252,20 @@ def run_daily_pipeline(args):
         l3_fits_writer = FitsL3Processor(fits_out_dir)
         anc_fits_writer = FitsANCProcessor(fits_out_dir)
 
-        # should be done later: internally
+        # hk_in_files = aspect_anc_processor.get_processing_files(phs)
         hk_in_files = []
-        candidates = list()  # aspect_anc_processor.find_processing_candidates()
-        v_candidates = aspect_anc_processor.get_version(candidates, version="latest")
-        for fc in v_candidates:
-            tr = aspect_anc_processor.test_for_processing(fc, phs)
-            if tr == TestForProcessingResult.Suitable:
-                hk_in_files.append(fc)
 
-        fl_months = flarelist_sdc.find_processing_months(phs)
-        fl_months = []
+        fl_sdc_months = flarelist_sdc.find_processing_months(phs)
+        # fl_sdc_months = []
+
+        # fl_sc_months = flarelist_sc.find_processing_months(phs)
+        fl_sc_months = []
 
-        ll_candidates = ll03ql.get_processing_files(phs)
+        # ll_candidates = ll03ql.get_processing_files(phs)
         ll_candidates = []
 
         fl_to_fl_files = fl_to_fl.get_processing_files(phs)
-        # flsdc_to_flcdspos_files = []
+        # fl_to_fl_files = []
 
         # all processing files should be terminated before the next step as the different
         # processing steeps might create new candidates
@@ -262,10 +278,16 @@ def run_daily_pipeline(args):
                                         processor=l2_fits_writer,
                                         config=CONFIG))
 
-            jobs.append(executor.submit(flarelist_sdc.process_fits_files, fl_months,
+            jobs.append(executor.submit(flarelist_sdc.process_fits_files, fl_sdc_months,
+                                        soopmanager=SOOPManager.instance,
+                                        spice_kernel_path=Spice.instance.meta_kernel_path,
+                                        processor=l3_fits_writer,
+                                        config=CONFIG))
+
+            jobs.append(executor.submit(flarelist_sc.process_fits_files, fl_sc_months,
                                         soopmanager=SOOPManager.instance,
                                         spice_kernel_path=Spice.instance.meta_kernel_path,
-                                        processor=anc_fits_writer,
+                                        processor=l3_fits_writer,
                                         config=CONFIG))
 
             # TODO a owen processing step for each flarelist file?

stixcore/processing/FlareListANC.py stixcore/processing/FlareListL3.pystixcore/processing/FlareListANC.py renamed to stixcore/processing/FlareListL3.py

@@ -541,7 +541,8 @@ def publish_fits_to_esa(args):
 
     parser.add_argument("-l", "--include_levels",
                         help="what levels should be published", type=str,
-                        default=CONFIG.get('Publish', 'include_levels', fallback="L0, L1, L2, ANC"))
+                        default=CONFIG.get('Publish', 'include_levels',
+                                           fallback="L0, L1, L2, ANC, LL03"))
 
     parser.add_argument("-p", "--include_products",
                         help="what products should be published", type=str,