Merge pull request #828 from CroatianMeteorNetwork/prerelease

Prerelease update - February 2026

Author: dvida

.config

@@ -66,6 +66,11 @@ camera_group_name: none
 ; three arguments: captured_night_dir, archived_night_dir, config 
 external_script_run: false
 
+; Toggle logging of the external script. If true, logging  will be captured by RMS from the external script. 
+; If false, the script process will have all log handlers removed to prevent interference with its own logging.
+external_script_log: false
+
+
 ; Run the external script after auto reprocess.
 ; "auto_reprocess" needs to be "true" for this to work.
 auto_reprocess_external_script_run: false
@@ -365,6 +370,22 @@ win_pc_weave: -Wall
 ; Flag for enabling/disabling upload to meteor server
 upload_enabled: true
 
+; If upload_split is true, two archives will be created.
+; One archive, suffix _imgdata, contains the images. The
+; other archive, suffix _metadata will only contain the
+; data derived from the images, including the timelapse.
+;
+; The metadata archive is much smaller than the imagedata
+; archive, and can be extracted very quickly. This speeds up the
+; processing on the server, since the image data is generally not
+; required.
+
+; If upload_split is false a single archive will be uploaded
+; with the suffix _detected containing all images, and data derived.
+; These archives are large and slow down the server side processing.
+
+upload_split: true
+
 ; Delay upload for the given number of minutes
 upload_delay: 0
 
@@ -800,3 +821,8 @@ timelapse_generate_from_frames: true
 ; color palette to use for various charts -can be any matplotlib color scheme
 ; other options are gist_ncar, rainbow, gist_rainbow, inferno 
 shower_color_map: gist_ncar
+
+; color to use for plotting sporadics on the radiants map. Can be any matplotlib
+; named color: https://matplotlib.org/stable/gallery/color/named_colors.html
+; white stands out well but the default is gray. 
+sporadic_color: gray

README.md

@@ -2,16 +2,19 @@
 
 
 import os
+import shutil
 import sys
 import traceback
 
 
+
 from RMS.Formats.FFfile import validFFName
 from RMS.Logger import getLogger
 from RMS.Misc import archiveDir, tarWithProgress
 from RMS.Routines import MaskImage
 from Utils.GenerateThumbnails import generateThumbnails
 from Utils.StackFFs import stackFFs
+from Utils.LogArchiver import makeLogArchives
 
 
 # Get the logger from the main module
@@ -147,7 +150,13 @@ def archiveFieldsums(dir_path):
 
 
 def archiveDetections(captured_path, archived_path, ff_detected, config, extra_files=None):
-    """ Create thumbnails and compress all files with detections and the accompanying files in one archive.
+    """ Create thumbnails and compress all files with detections and the accompanying files
+        in one archive, suffix _detected
+
+        or
+
+        only the fr*.bin, ff*.fits and extra_files in one archive, suffix _imgdata,
+        and everything apart from fr*.bin and ff*.fits in another archive, suffix _metadata
 
     Arguments:
         captured_path: [str] Path where the captured files are located.
@@ -156,18 +165,19 @@ def archiveDetections(captured_path, archived_path, ff_detected, config, extra_f
         config: [conf object] Configuration.
 
     Keyword arguments:
-        extra_files: [list] A list of extra files (with fill paths) which will be be saved to the night 
+        extra_files: [list] A list of extra files (with full paths) which will be saved to the night
             archive.
 
     Return:
         archive_name: [str] Name of the archive where the files were compressed to.
+        imgdata_archive_name: [str] Name of the archive where the images were compressed to.
+        metadata_archive_name: [str] Name of the archive where the metadata was compressed to.
 
     """
 
     # Get the list of files to archive
     file_list = selectFiles(config, captured_path, ff_detected)
 
-    
     log.info('Generating thumbnails...')
 
     try:
@@ -249,21 +259,70 @@ def archiveDetections(captured_path, archived_path, ff_detected, config, extra_f
         log.error('Generating stack failed with error:' + repr(e))
         log.error("".join(traceback.format_exception(*sys.exc_info())))
 
+    log.info("Generating an archive file of most recent logs...")
+
+    try:
+
+
+        log_archive_path = makeLogArchives(config, captured_path)
+        log.info(f"Log archive saved to: {log_archive_path}")
+        file_list.append(os.path.basename(log_archive_path))
+
+    except Exception as e:
+        log.error('Generating log archives failed with error:' + repr(e))
+        log.error("".join(traceback.format_exception(*sys.exc_info())))
+
 
 
     if file_list:
 
         # Create the archive ZIP in the parent directory of the archive directory
-        archive_name = os.path.join(os.path.abspath(os.path.join(archived_path, os.pardir)), 
-            os.path.basename(captured_path) + '_detected')
+        archive_base = os.path.join(os.path.abspath(os.path.join(archived_path, os.pardir)),
+            os.path.basename(captured_path))
 
-        # Archive the files
-        archive_name = archiveDir(captured_path, file_list, archived_path, archive_name, \
-            extra_files=extra_files)
+        # Create the imgdata set which is the union of the sets of FF files and FR files
+        imgdata_set = (set([item for item in file_list if item.startswith("FF") and item.endswith(".fits")]) |
+                       set([item for item in file_list if item.startswith("FR") and item.endswith(".bin")]))
+
+        # Create the metadata set which is all the files from _detected excluding the files in imgdata_set,
+        # (*.fits and *.bin)
+
+        metadata_set = set([item for item in file_list if item not in imgdata_set])
+
+        # Create all the required archive names and paths
+        archive_name = f"{archive_base}_detected"
+        metadata_archived_path = f"{archived_path}_metadata"
+        metadata_archive_name = f"{archive_base}_metadata"
+        imgdata_archived_path = f"{archived_path}_imgdata"
+        imgdata_archive_name = f"{archive_base}_imgdata"
+
+        # In all cases, generate the _detected directory and archive as a record for the station
+
+        # Make the archive directory and compress into _detected.tar.bz2 if config.upload_split is False.
+
+        create_detected_tar_bz2 = not config.upload_split
+        archive_name = archiveDir(captured_path, file_list, archived_path, archive_name, extra_files=extra_files,
+                                  create_archive=create_detected_tar_bz2)
+
+        if config.upload_split:
+            # Create a directory to hold the metadata files, archive, then remove the directory
+            metadata_archive_name = archiveDir(captured_path, metadata_set, metadata_archived_path,
+                                               metadata_archive_name, extra_files=extra_files, delete_dest_dir=True)
+
+            # Create a directory to hold the imgdata files, archive, then remove the directory
+            imgdata_archive_name = archiveDir(captured_path, imgdata_set, imgdata_archived_path,
+                                              imgdata_archive_name, extra_files=extra_files, delete_dest_dir=True)
+
+            # Set to archive_name to None to prevent upload from this execution path
+            archive_name = None
+
+        else:
+            # Set to None, as these archives will not be generated in this path
+            imgdata_archive_name, metadata_archive_name = None, None
 
-        return archive_name
+        return archive_name, imgdata_archive_name, metadata_archive_name
 
-    return None
+    return None, None, None
 
 
 def archiveFrameTimelapse(frames_root,
@@ -355,7 +414,7 @@ def archiveFrameTimelapse(frames_root,
 
     ff_detected = ['FF_CA0001_20170905_094707_004_0000000.fits', 'FF_CA0001_20170905_094716_491_0000256.fits']
 
-    archive_name = archiveDetections(captured_path, archived_path, ff_detected, config)
+    archive_name, metadata_name, imgdata_name = archiveDetections(captured_path, archived_path, ff_detected, config)
 
-    print(archive_name)
+    print(archive_name, metadata_name, imgdata_name)