preperation for merge and release

Author: nicHoch

docs/code_ref/io.rst

@@ -6,8 +6,8 @@ to certain directories.
 
 .. automodapi:: stixcore.io
 
-.. automodapi:: stixcore.io.fits
-.. automodapi:: stixcore.io.fits.processors
+.. automodapi:: stixcore.io.product_processors.fits.processors
+.. automodapi:: stixcore.io.product_processors.plots.processors
 
 .. automodapi:: stixcore.io.soc
 .. automodapi:: stixcore.io.soc.manager

processed.sqlite

@@ -95,8 +95,7 @@ def read_flarelist(self):
         """
         if self.update or not self.file.exists():
             last_date = datetime(2020, 1, 1, 0, 0, 0)
-            last_date = datetime(2025, 4, 12, 0, 0, 0)
-            # only run flare detection for time oldet than 7 days
+            # only run flare detection for time older than 7 days
             today = datetime.now() - timedelta(days=7)
             flare_df_lists = []
             if self.file.exists():
@@ -141,6 +140,7 @@ def read_flarelist(self):
                         f"with {len(ql_lc_files)} lightcurve files and "
                         f"{len(ql_bg_files)} background files"
                     )
+                    # TODO re-enable flare-list creation
                     # flares = stixpy.detect_flares(start, end,
                     #                               ql_lc_files=ql_lc_files,
                     #                               ql_bg_files=ql_bg_files)

stixcore/io/FlareListManager.py

@@ -1,10 +1,8 @@
 """Module for the different processing levels."""
 
-from pathlib import Path
 from datetime import datetime
 
 import numpy as np
-from matplotlib import pyplot as plt
 
 import astropy.units as u
 from astropy.io import fits
@@ -27,7 +25,6 @@
     "FitsL0Processor",
     "FitsL1Processor",
     "FitsL2Processor",
-    "PlotProcessor",
 ]
 
 
@@ -1084,106 +1081,6 @@ def write_fits(self, prod, *, version=0):
         return [filetowrite]
 
 
-class PlotProcessor(FitsL2Processor):
-    """A file product processor for plot images"""
-
-    def __init__(self, archive_path):
-        """Creates a new PlotProcessor object.
-
-        Parameters
-        ----------
-        archive_path : Path
-            the output root path where the files should be created
-        """
-        super().__init__(archive_path)
-
-    def generate_filename(self, product, *, version=0, suffix=".svg"):
-        """Generates a SOAR conform filename based on product characteristics.
-
-        Parameters
-        ----------
-        product : Product
-            The data product the file name should be generated for
-        version : int, optional
-            The file version, by default 0 = detect from codebase
-        suffix : str, optional
-            file name suffix like svg, png, ..., by default '.svg'
-
-        Returns
-        -------
-        Path
-            a Path object with full name and path
-        """
-        p = Path(super().generate_filename(product=product, version=version, header=True, status="C"))
-        return p.with_suffix(suffix).name
-
-    def generate_primary_header(self, filename, product, *, version=0):
-        """Transforms the fits header into a more generic header dict
-        that might be used in other output file formats
-
-        Parameters
-        ----------
-        filename : Path
-            The envisoned file name and path for the product
-        product : Product
-            The products holding the data
-        version : int, optional
-            the processing version, by default 0 = detect from codebase
-
-        Returns
-        -------
-        dict
-            a dict of header keywords and values
-        """
-        l1, l2 = super().generate_primary_header(filename, product, version=version)
-        header = dict()
-        for k, v in l1:
-            header[k] = v
-        for kv in l2:
-            header[kv[0]] = kv[1]
-        return header
-
-    def write_plot(self, product, *, version=0):
-        """
-        Write products into a plot image file.
-
-        Parameters
-        ----------
-        product : `stixcore.product.level2`
-
-        version : `int`
-            the version modifier for the filename
-            default 0 = detect from codebase.
-
-        Returns
-        -------
-        list
-            of created file as `pathlib.Path`
-
-        """
-        if version == 0:
-            version = product.get_processing_version()
-
-        filename = self.generate_filename(product=product, version=version)
-
-        # headers = self.generate_primary_header(filename, product, version=version)
-        # headers['parent'] = get_complete_file_name(product.parent_file_path.name)
-
-        parts = [product.level, product.utc_timerange.center.strftime("%Y/%m/%d"), product.type.upper()]
-        # for science data use start date
-        if product.type in ["sci", "flarelist"]:
-            parts[1] = product.utc_timerange.start.strftime("%Y/%m/%d")
-        path = self.archive_path.joinpath(*[str(x) for x in parts])
-        path.mkdir(parents=True, exist_ok=True)
-
-        plot_path = path / filename
-
-        fig = product.get_plot()
-        fig.savefig(plot_path, format="svg")
-        plt.close(fig)
-        return plot_path
-
-
 class FitsL3Processor(FitsL2Processor):
     def __init__(self, archive_path):
         super().__init__(archive_path)

stixcore/io/fits/__init__.py …e/io/product_processors/fits/__init__.pystixcore/io/fits/__init__.py renamed to stixcore/io/product_processors/fits/__init__.py stixcore/io/fits/__init__.py renamed to stixcore/io/product_processors/fits/__init__.py

@@ -0,0 +1,115 @@
+"""Module for the different processing levels."""
+
+from pathlib import Path
+
+from matplotlib import pyplot as plt
+
+from stixcore.io.product_processors.fits.processors import FitsL2Processor
+from stixcore.util.logging import get_logger
+
+__all__ = [
+    "PlotProcessor",
+]
+
+
+logger = get_logger(__name__)
+
+
+class PlotProcessor(FitsL2Processor):
+    """A file product processor for plot images"""
+
+    def __init__(self, archive_path):
+        """Creates a new PlotProcessor object.
+
+        Parameters
+        ----------
+        archive_path : Path
+            the output root path where the files should be created
+        """
+        super().__init__(archive_path)
+
+    def generate_filename(self, product, *, version=0, suffix=".svg"):
+        """Generates a SOAR conform filename based on product characteristics.
+
+        Parameters
+        ----------
+        product : Product
+            The data product the file name should be generated for
+        version : int, optional
+            The file version, by default 0 = detect from codebase
+        suffix : str, optional
+            file name suffix like svg, png, ..., by default '.svg'
+
+        Returns
+        -------
+        Path
+            a Path object with full name and path
+        """
+        p = Path(super().generate_filename(product=product, version=version, header=True, status="C"))
+        return p.with_suffix(suffix).name
+
+    def generate_primary_header(self, filename, product, *, version=0):
+        """Transforms the fits header into a more generic header dict
+        that might be used in other output file formats
+
+        Parameters
+        ----------
+        filename : Path
+            The envisoned file name and path for the product
+        product : Product
+            The products holding the data
+        version : int, optional
+            the processing version, by default 0 = detect from codebase
+
+        Returns
+        -------
+        dict
+            a dict of header keywords and values
+        """
+        l1, l2 = super().generate_primary_header(filename, product, version=version)
+        header = dict()
+        for k, v in l1:
+            header[k] = v
+        for kv in l2:
+            header[kv[0]] = kv[1]
+        return header
+
+    def write_plot(self, product, *, version=0):
+        """
+        Write products into a plot image file.
+
+        Parameters
+        ----------
+        product : `stixcore.product.level2`
+
+        version : `int`
+            the version modifier for the filename
+            default 0 = detect from codebase.
+
+        Returns
+        -------
+        list
+            of created file as `pathlib.Path`
+
+        """
+        if version == 0:
+            version = product.get_processing_version()
+
+        filename = self.generate_filename(product=product, version=version)
+
+        # headers = self.generate_primary_header(filename, product, version=version)
+        # headers['parent'] = get_complete_file_name(product.parent_file_path.name)
+
+        parts = [product.level, product.utc_timerange.center.strftime("%Y/%m/%d"), product.type.upper()]
+        # for science data use start date
+        if product.type in ["sci", "flarelist"]:
+            parts[1] = product.utc_timerange.start.strftime("%Y/%m/%d")
+        path = self.archive_path.joinpath(*[str(x) for x in parts])
+        path.mkdir(parents=True, exist_ok=True)
+
+        plot_path = path / filename
+
+        fig = product.get_plot()
+        fig.savefig(plot_path, format="svg")
+        plt.close(fig)
+        return plot_path

stixcore/io/fits/processors.py …io/product_processors/fits/processors.pystixcore/io/fits/processors.py renamed to stixcore/io/product_processors/fits/processors.py stixcore/io/fits/processors.py renamed to stixcore/io/product_processors/fits/processors.py

@@ -6,7 +6,12 @@
 from astropy.table import QTable
 
 from stixcore.data.test import test_data
-from stixcore.io.fits.processors import FitsL0Processor, FitsL1Processor, FitsLBProcessor
+from stixcore.io.product_processors.fits.processors import (
+    FitsL0Processor,
+    FitsL1Processor,
+    FitsLBProcessor,
+)
+from stixcore.io.product_processors.plots.processors import PlotProcessor
 from stixcore.products.product import Product
 from stixcore.soop.manager import SOOPManager
 from stixcore.time import SCETime, SCETimeRange
@@ -51,7 +56,7 @@ def test_levelb_processor_generate_filename_without_rid():
 
 
 @patch("stixcore.products.level0.quicklookL0.QLProduct")
-@patch("stixcore.io.fits.processors.datetime")
+@patch("stixcore.io.product_processors.fits.processors.datetime")
 def test_levelb_processor_generate_primary_header(datetime, product):
     processor = FitsLBProcessor("some/path")
     datetime.now().isoformat.return_value = "1234-05-07T01:02:03.346"
@@ -136,7 +141,7 @@ def test_level0_processor_generate_filename():
 
 
 @patch("stixcore.products.level0.quicklookL0.QLProduct")
-@patch("stixcore.io.fits.processors.datetime")
+@patch("stixcore.io.product_processors.fits.processors.datetime")
 def test_level0_processor_generate_primary_header(datetime, product):
     processor = FitsL0Processor("some/path")
     datetime.now().isoformat.return_value = "1234-05-07T01:02:03.346"
@@ -290,3 +295,15 @@ def test_level1_processor_generate_primary_header(product, soop_manager):
                 assert np.allclose(test_data[name], value)
             else:
                 assert value == test_data[name]
+
+
+@patch("stixcore.products.lowlatency.quicklookLL.LightCurveL3")
+def test_plot_processor_generate_filename(product):
+    processor = PlotProcessor("some/path")
+    product.type = "ql"
+    product.scet_timerange = SCETimeRange(start=SCETime(0, 0), end=SCETime(coarse=0, fine=2**16 - 1))
+    product.utc_timerange = product.scet_timerange.to_timerange()
+    product.level = "LL03"
+    product.name = "lightcurve"
+    filename = processor.generate_filename(product, version=1)
+    assert filename == "solo_LL03_stix-ql-lightcurve_20000101_V01C.svg"

stixcore/io/product_processors/plots/processors.py

@@ -8,7 +8,7 @@
 
 from stixcore.config.config import CONFIG
 from stixcore.ephemeris.manager import Spice, SpiceKernelManager
-from stixcore.io.fits.processors import FitsL1Processor
+from stixcore.io.product_processors.fits.processors import FitsL1Processor
 from stixcore.products import Product
 from stixcore.products.level0.scienceL0 import NotCombineException
 from stixcore.soop.manager import SOOPManager

stixcore/io/fits/tests/__init__.py …/io/product_processors/tests/__init__.pystixcore/io/fits/tests/__init__.py renamed to stixcore/io/product_processors/tests/__init__.py stixcore/io/fits/tests/__init__.py renamed to stixcore/io/product_processors/tests/__init__.py

@@ -8,7 +8,7 @@
 
 from stixcore.config.config import CONFIG
 from stixcore.ephemeris.manager import Spice
-from stixcore.io.fits.processors import FitsL2Processor
+from stixcore.io.product_processors.fits.processors import FitsL2Processor
 from stixcore.processing.sswidl import SSWIDLProcessor
 from stixcore.products import Product
 from stixcore.products.level0.scienceL0 import NotCombineException