fix end2end testing (#444)

* Add step to show installed Python packages

* fix back to typo needs to be addressed in its own issue

* fixes after review

Author: nicHoch

.github/workflows/end2end_run.yml

@@ -42,6 +42,8 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install -e .[dev]
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: show environment
+      run: python -m pip freeze 
     - name: end2end test with pytest
       run: |
         pytest -p no:warnings --doctest-rst -m "end2end" --pyargs stixcore --remote-data=any

docs/pipelineconfiguration.rst

@@ -44,7 +44,7 @@ SOLMOC TM data is synced from pub026 via rsync by cron every 15min.
 .. code-block::
 
     # sync TM data from GFTS to local "wait for processing" folder
-    */15 * * * * rsync -av stixcore@147.86.8.26:'/home/solmoc/from_edds/tm/*PktTmRaw*.xml' /data/stix/SOLSOC/from_edds/tm/incoming > /dev/null
+    */15 * * * * rsync -av stixcore@147.86.8.26:'/home/solmoc/from_edds/tm/*PktTmRaw*.xml' /data/stix/SOLSOC/from_edds/tm/incomming > /dev/null
 
 The very latest SPICE kernels are required for accurate pointing information of the satellite.
 We may receive TM files via GFTS that contain data for times that are not available in the latest SPICE kernels, as the latest SPICE kernels have not yet been updated and delivered.
@@ -53,7 +53,7 @@ Therefore, a stage and delay step is added via cron (every 30 min) that only pub
 .. code-block::
 
     # move the TM data from the wait into the processing folder - that will trigger the pipeline to "get started": the wait period is meanwhile short
-    */30 * * * * find /data/stix/SOLSOC/from_edds/tm/incoming/ -type f -mmin +2 -exec rsync -a {} /data/stix/SOLSOC/from_edds/tm/processing/ \;
+    */30 * * * * find /data/stix/SOLSOC/from_edds/tm/incomming/ -type f -mmin +2 -exec rsync -a {} /data/stix/SOLSOC/from_edds/tm/processing/ \;
 
 Sync STIX-CONF repo in all used instances
 *****************************************

stixcore/util/scripts/end2end_testing.py

@@ -188,7 +188,7 @@ def end2end_pipeline(indir, fitsdir):
             splits=1,
             outdir=datapath,
             socdir=Path(
-                "/data/stix/SOLSOC/from_edds/tm/incoming/",
+                "/data/stix/SOLSOC/from_edds/tm/incomming/",
             ),
         )
 

stixcore/util/scripts/find_unprocessed_tm.py

@@ -12,7 +12,7 @@
 TM_REGEX = re.compile(r".*PktTmRaw.*.xml$")
 logging_dir = Path(CONFIG.get("Pipeline", "log_dir"))
 tm_dir = Path(CONFIG.get("Paths", "tm_archive"))
-tm_dir = Path("/data/stix/SOLSOC/from_edds/tm/incoming")
+tm_dir = Path("/data/stix/SOLSOC/from_edds/tm/incomming")
 list_of_log_files = logging_dir.glob("*.xml.out")
 latest_log_file = (
     logging_dir / "TM_BatchRequest.PktTmRaw.SOL.0.2023.305.15.33.13.852.bBWz@2023.352.21.00.01.039.1.xml.out"