Make night product failures visible and improve tool discoverability

RMS/ConfigReader.py

@@ -16,6 +16,7 @@
 
 from __future__ import absolute_import, division, print_function
 
+import logging
 import math
 import os
 import sys
@@ -32,6 +33,11 @@
     from ConfigParser import NoOptionError, RawConfigParser
     FileNotFoundError = IOError  # Map FileNotFoundError to IOError in Python 2
 
+# Module logger - propagates to the root logger, so warnings end up in the night log once logging is
+# initialized, and on the console before that
+log = logging.getLogger(__name__)
+
+
 # Used to determine if ML filtering is available
 TFLITE_AVAILABLE = False
 
@@ -751,7 +757,7 @@ def parse(path, strict=True):
 
     # Disable upload if the default station name is used
     if config.stationID == "XX0001":
-        print("Disabled upload because the default station code is used!")
+        log.warning("Disabled upload because the default station code is used!")
         config.upload_enabled = False
 
 
@@ -1141,8 +1147,7 @@ def parseCapture(config, parser):
         # Limit the FPS to 1 million, as the time precision of datetime is 1 us
         if config.fps > 1000000:
             config.fps = 1000000
-            print()
-            print("WARNING! The FPS has been limited to 1,000,000!")
+            log.warning("The FPS has been limited to 1,000,000!")
 
     if parser.has_option(section, "camera_buffer"):
         config.camera_buffer = parser.getint(section, "camera_buffer")
@@ -1161,8 +1166,8 @@ def parseCapture(config, parser):
 
 
     if (config.fov_w <= 0) or (config.fov_h <= 0):
-        print('The field of view in the config file (fov_h and fov_w) have to be positive numbers!')
-        print('Make sure to set the approximate FOV size correctly!')
+        log.error('The field of view in the config file (fov_h and fov_w) have to be positive numbers!')
+        log.error('Make sure to set the approximate FOV size correctly!')
         sys.exit()
 
 
@@ -1196,7 +1201,7 @@ def parseCapture(config, parser):
     # If the option is absent from the config, fall back to the class default.
     save_requested = parser.getboolean(section, "save_frames", fallback=config.save_frames)
     if save_requested and not config.ffmpeg_binary:
-        print("save_frames requested but FFmpeg not available - disabling.")
+        log.warning("save_frames requested but FFmpeg not available - disabling.")
         config.save_frames = False
     else:
         config.save_frames = save_requested
@@ -1211,8 +1216,7 @@ def parseCapture(config, parser):
         # Must be an integer between 0 and 100
         if not 0 <= config.jpgs_quality <= 100:
             config.jpgs_quality = 90
-            print()
-            print("WARNING! The jpgs_quality must be between 0 and 100. It has been reset to 90!")
+            log.warning("The jpgs_quality must be between 0 and 100. It has been reset to 90!")
 
     # Load the PNG compression
     if parser.has_option(section, "png_compression"):
@@ -1221,8 +1225,7 @@ def parseCapture(config, parser):
         # Must be an integer between 0 and 9
         if not 0 <= config.png_compression <= 9:
             config.png_compression = 3
-            print()
-            print("WARNING! The png_compression must be between 0 and 9. It has been reset to 3!")
+            log.warning("The png_compression must be between 0 and 9. It has been reset to 3!")
 
 
     # Load the interval for saving video frame
@@ -1473,8 +1476,7 @@ def parseMeteorDetection(config, parser):
         # Check that the given bin size is a factor of 2
         if bin_factor > 1:
             if math.log(bin_factor, 2)/int(math.log(bin_factor, 2)) != 1:
-                print('Warning! The given binning factor is not a factor of 2!')
-                print('Defaulting to 1...')
+                log.warning('The given binning factor is not a factor of 2! Defaulting to 1...')
                 bin_factor = 1
 
         config.detection_binning_factor = bin_factor
@@ -1485,8 +1487,8 @@ def parseMeteorDetection(config, parser):
 
         bin_method_list = ['sum', 'avg']
         if bin_method not in bin_method_list:
-            print('Warning! The binning method {:s} is not an allowed binning method: ', bin_method_list)
-            print('Defaulting to avg...')
+            log.warning('The binning method {:s} is not an allowed binning method: {}. '
+                        'Defaulting to avg...'.format(bin_method, bin_method_list))
             bin_method = 'avg'
 
         config.detection_binning_method = bin_method
@@ -1708,7 +1710,7 @@ def parseCalibration(config, parser):
         # If they're all zero, use the V band
         if all([x == 0 for x in config.star_catalog_band_ratios]):
             config.star_catalog_band_ratios[1] = 1.0
-            print('Warning! All band ratios are zero! Using the V band as the default band ratio...')
+            log.warning('All band ratios are zero! Using the V band as the default band ratio...')
 
 
     if parser.has_option(section, "platepar_name"):

RMS/Reprocess.py

@@ -277,8 +277,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
                 generateCalibrationReport(config, night_data_dir, platepar=platepar)
 
             except Exception as e:
-                log.debug('Generating calibration report failed with the message:\n' + repr(e))
-                log.debug(repr(traceback.format_exception(*sys.exc_info())))
+                log.warning('Generating calibration report failed with the message:\n' + repr(e))
+                log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
 
 
@@ -290,8 +290,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
                     sporadic_color=config.sporadic_color)
 
             except Exception as e:
-                log.debug('Shower association failed with the message:\n' + repr(e))
-                log.debug(repr(traceback.format_exception(*sys.exc_info())))
+                log.warning('Shower association failed with the message:\n' + repr(e))
+                log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
 
 
@@ -339,8 +339,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
 
 
             except Exception as e:
-                log.debug("Generating a FOV KML file failed with the message:\n" + repr(e))
-                log.debug(repr(traceback.format_exception(*sys.exc_info())))
+                log.warning("Generating a FOV KML file failed with the message:\n" + repr(e))
+                log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
 
 
@@ -351,8 +351,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
                                  mask=mask, platepar=platepar)
 
             except Exception as e:
-                log.debug("Preparing flux files failed with the message:\n" + repr(e))
-                log.debug(repr(traceback.format_exception(*sys.exc_info())))
+                log.warning("Preparing flux files failed with the message:\n" + repr(e))
+                log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
 
     else:
@@ -369,8 +369,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
         plotFieldsums(night_data_dir, config)
 
     except Exception as e:
-        log.debug('Plotting field sums failed with message:\n' + repr(e))
-        log.debug(repr(traceback.format_exception(*sys.exc_info())))
+        log.warning('Plotting field sums failed with message:\n' + repr(e))
+        log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
 
 
@@ -429,8 +429,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
                     except Exception as e:
                         print("Error in archiving process: {}".format(e))
         except Exception as e:
-            log.debug('Archiving FT files failed with message:\n' + repr(e))
-            log.debug(repr(traceback.format_exception(*sys.exc_info())))
+            log.warning('Archiving FT files failed with message:\n' + repr(e))
+            log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
 
     log.info('Making a flat...')
@@ -440,8 +440,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
         flat_img = makeFlat(night_data_dir, config)
 
     except Exception as e:
-        log.debug('Making a flat failed with message:\n' + repr(e))
-        log.debug(repr(traceback.format_exception(*sys.exc_info())))
+        log.warning('Making a flat failed with message:\n' + repr(e))
+        log.warning(repr(traceback.format_exception(*sys.exc_info())))
         flat_img = None
 
 
@@ -479,8 +479,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
             extra_files.append(timelapse_path)
 
         except Exception as e:
-            log.debug('Generating a timelapse failed with message:\n' + repr(e))
-            log.debug(repr(traceback.format_exception(*sys.exc_info())))
+            log.warning('Generating a timelapse failed with message:\n' + repr(e))
+            log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
     log.info('Plotting timestamp intervals...')
 
@@ -504,8 +504,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
 
 
     except Exception as e:
-        log.debug('Plotting timestamp interval failed with message:\n' + repr(e))
-        log.debug(repr(traceback.format_exception(*sys.exc_info())))
+        log.warning('Plotting timestamp interval failed with message:\n' + repr(e))
+        log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
     # Generate a config audit report
     log.info('Generate config audit report')
@@ -528,8 +528,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
         extra_files.append(audit_file_path)
 
     except Exception as e:
-        log.debug('Generating config audit failed with message:\n' + repr(e))
-        log.debug(repr(traceback.format_exception(*sys.exc_info())))
+        log.warning('Generating config audit failed with message:\n' + repr(e))
+        log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
 
     ### Add extra files to archive
@@ -655,8 +655,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False)
 
 
     except Exception as e:
-        log.debug('Finalizing Observation Summary failed with message:\n' + repr(e))
-        log.debug(repr(traceback.format_exception(*sys.exc_info())))
+        log.warning('Finalizing Observation Summary failed with message:\n' + repr(e))
+        log.warning(repr(traceback.format_exception(*sys.exc_info())))
 
     obs_summary_to_log = serialize(config, night_directory=night_data_dir, final=True)
     log.info("\n\nObservation Summary\n===================\n\n" + obs_summary_to_log + "\n\n")

Utils/FRbinViewer.py

@@ -260,6 +260,11 @@ def destroyWindow(self):
             self._mpl_ready = False
 
 
+# On-screen key legend, toggled with the 'h' key while viewing
+KEY_LEGEND = "Keys: SPACE pause | 1 prev file | 2 next line | q quit | h hide keys"
+show_key_legend = True
+
+
 def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=None, hide=False,
         avg_background=False, split=False, add_timestamp=False, add_frame_number=False, append_ff_to_video=False,
          add_shower_name=False, associations={}):
@@ -284,6 +289,8 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N
 
     """
 
+    global show_key_legend
+
     if extract_format is None:
         extract_format = 'png'
 
@@ -459,6 +466,10 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N
 
                 # Resize large frames so they fit comfortably on screen.
                 display_image = resizeImageIfNeed(img)
+
+                # Draw the key legend
+                if show_key_legend:
+                    addTextToImage(display_image, KEY_LEGEND, 10, 40)
                 # Abort preview if the backend fails so we can continue processing.
                 if not display or not display.showFrame(display_image):
                     if display:
@@ -483,6 +494,7 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N
                 # Space key: pause display.
                 # 1: previous file.
                 # 2: next line.
+                # h: toggle the key legend.
                 # q: Quit.
                 key = display.waitKey(wait_time)
 
@@ -491,14 +503,19 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N
                         display.destroyWindow()
                     return -1
 
-                elif key == ord("2"): 
+                elif key == ord("2"):
                     break
 
-                elif key == ord(" "): 
-                    
+                elif key == ord(" "):
+
                     # Pause/unpause video
                     pause_flag = not pause_flag
 
+                elif key == ord("h"):
+
+                    # Toggle the on-screen key legend
+                    show_key_legend = not show_key_legend
+
                 elif key == ord("q"):
                     os._exit(0)
 
@@ -673,6 +690,7 @@ def loadShowerAssociations(folder, configuration):
             Space: pause display.
             1: previous file.
             2: next line.
+            h: toggle the on-screen key legend.
             q: Quit.
             """, formatter_class=argparse.RawTextHelpFormatter)
 

Utils/README.md

@@ -82,6 +82,25 @@ Detailed atmospheric and orbital analysis tools, particularly for computing mete
 
 ---
 
+## ⚙️ Runnable Modules in `RMS/`
+
+The main pipeline modules live in the `RMS/` package and are run the same way, using `python -m RMS.<ModuleName>`:
+
+- **`RMS.StartCapture`**: Starts the capture pipeline - recording, compression, detection, calibration, and upload. The main entry point for running a station.
+- **`RMS.Reprocess`**: Reprocesses a night directory from raw FF files - detection, calibration, archiving, and all night products.
+- **`RMS.DetectStarsAndMeteors`**: Runs star extraction and meteor detection on FF files in a given directory.
+- **`RMS.ExtractStars`**: Extracts stars from FF files and writes a CALSTARS file.
+- **`RMS.EventMonitor`**: Monitors the network event list and uploads footage if a fireball trajectory crossed the station's field of view.
+- **`RMS.DownloadMask`**: Downloads a new mask file for the station from the server.
+- **`RMS.CaptureDuration`**: Prints the capture start time and duration for the night (given the station coordinates).
+- **`RMS.DeleteOldObservations`**: Frees up disk space by deleting old observation data.
+- **`RMS.MLFilter`**: Filters detections using the machine-learning meteor classifier.
+- **`RMS.ClearSkyDetector`**: Estimates which parts of the night had clear skies.
+
+Other pipeline modules (e.g. `RMS.ArchiveDetections`, `RMS.UploadManager`, `RMS.DownloadPlatepar`, `RMS.CaptureModeSwitcher`) are internal - their `__main__` blocks are hardcoded test stubs, not user CLIs. They run as part of `RMS.StartCapture`/`RMS.Reprocess`.
+
+---
+
 ### General Usage
 To use a script, you can generally invoke it as a Python module to ensure that the main `RMS` dependency paths are resolved properly:
 ```bash