Flarelist anc (#431)

rewrite pipeline to cronjob based approach #422

* flare list, flare list with position, flare list with soop, flare list preview maps

* change flarelist to L3

* new naming for LL3 QL products

* fixes after python3.11 upgrade

* fix missing isnan from merge conflicts

fix tests

fix tests

fix tests

* add future migration logic to  ProcessingHistoryStorage

* preperation for merge and release

* fix for #437

* Update stixpy dependency to use GitHub source

Author: nicHoch

.gitignore

@@ -165,3 +165,4 @@ monitor_status.json
 stixcore/data/test/idb/v2.26.38/idb.sqlite
 .python-version
 sunpy/_version.py
+stixcore/_version.py

docs/code_ref/io.rst

@@ -6,8 +6,8 @@ to certain directories.
 
 .. automodapi:: stixcore.io
 
-.. automodapi:: stixcore.io.fits
-.. automodapi:: stixcore.io.fits.processors
+.. automodapi:: stixcore.io.product_processors.fits.processors
+.. automodapi:: stixcore.io.product_processors.plots.processors
 
 .. automodapi:: stixcore.io.soc
 .. automodapi:: stixcore.io.soc.manager

docs/pipelineconfiguration.rst

@@ -75,6 +75,25 @@ Sync FITS products with Platform (stix data center)
     # sync current fits to NAS (connection to Platform)
     0 */6 * * * sudo rsync -avLo --delete /var/www/data/fits/ /mnt/nas05/stix/stixcore_fits/
 
+Sync STIX-CONF repo in all used instances
+*****************************************
+
+```
+# sync the STIX-CONF to the latest release update version number
+# in the publix www folder
+10 */4 * * * cd /var/www/data/STIX-CONF && git fetch --tags && latestTag=$(git describe --tags `git rev-list --tags --max-count=1`) && git checkout $latestTag; echo $latestTag > VERSION.TXT
+# but also in the Pipeline setup once a day
+# had to be copied into the venv folder as well to get activated
+20 22 * * * cd /home/stixcore/STIXCore/stixcore/config/data/common/  && git fetch --tags && latestTag=$(git describe --tags `git rev-list --tags --max-count=1`) && git checkout $latestTag; echo $latestTag > VERSION.TXT && cp -r /home/stixcore/STIXCore/stixcore/config/data/common /home/stixcore/STIXCore/venv/lib/python3.9/site-packages/stixcore/config/data/
+```
+
+Sync FITS products with Platform (stix data center)
+***************************************************
+
+```
+# sync current fits to NAS (connection to Platform)
+0 */6 * * * sudo rsync -avLo --delete /var/www/data/fits/ /mnt/nas05/stix/stixcore_fits/
+```
 
 Sync SPICE kernels and SOOP data
 ********************************

pyproject.toml

@@ -15,7 +15,7 @@ license = "BSD-3-Clause"
 license-files = ["LICENSE.rst"]
 authors = [
     {name="Shane Maloney", email = "shane.maloney@dias.ie"},
-    {name="Nicky Maloney", email = "shane.maloney@dias.ie"}
+    {name="Nicky Hochmuth", email = "nicky.hochmuth@ateleris.ch"}
 ]
 dependencies = [
     "bitstring~=4.2",
@@ -27,9 +27,10 @@ dependencies = [
     "pyyaml",
     "scp",
     "spiceypy~=6.0",
+    "sunpy~=6.0.0",
     "watchdog~=6.0",
     "stixdcpy==3.0",
-    "stixpy>=0.1.3"
+    "stixpy@git+https://github.com/TCDSolar/stixpy.git@main"
 ]
 dynamic = ["version"]
 keywords = ["solar physics", "solar", "sun", "solarorbiter", "stix"]

stixcore/data/stixcore.ini

@@ -3,6 +3,7 @@ tm_archive =
 fits_archive = /data/stix/out/test/pipeline_auto_test
 spice_kernels =
 soop_files =
+fido_search_url = file:///data/stix/out/fits_v1.2.0
 [Logging]
 stop_on_error = False
 [IDLBridge]
@@ -13,6 +14,8 @@ status_server_port = 12345
 parallel_batchsize_L0 = 300
 parallel_batchsize_L1 = 300
 parallel_batchsize_L2 = 100
+flareid_sdc_lut_file = ./stixcore/data/publish/flarelist_sdc_lut.csv
+flareid_sc_lut_file = ./stixcore/data/publish/flarelist_sc_lut.csv
 [Publish]
 db_file = ./stixcore/data/publish/published.sqlite
 include_levels = L0
@@ -29,3 +32,5 @@ endpoint = https://solarorbiter.esac.esa.int/soopkitchen/api
 user = smaloney
 password = set_in_user_ini
 soop_files_download = ./stixcore/data/soop
+[Processing]
+flarelist_sdc_min_count = 1000

stixcore/data/test/stixcore.ini

@@ -20,6 +20,7 @@ error_mail_sender = test@test.de
 parallel_batchsize_L0 = 3
 parallel_batchsize_L1 = 3
 parallel_batchsize_L2 = 3
+status_server_port = 12345
 [SOOP]
 endpoint = https://solarorbiter.esac.esa.int/soopkitchen/api
 user = smaloney