From 50500de2c0802dd8afb567b001c6a5a3d228698a Mon Sep 17 00:00:00 2001 From: Denis Vida Date: Tue, 7 Jul 2026 09:27:11 +0200 Subject: [PATCH 1/2] Make night product failures visible and improve tool discoverability Promote the exception handlers around night product generation in Reprocess (calibration report, observation summary, timelapse, flat, shower association, FOV KML, flux files, field sums, FT archiving, config audit, timestamp plot) from log.debug to log.warning so failures reach the uploaded night logs instead of being silently swallowed. Route ConfigReader validation warnings (FPS clamp, FOV check, jpg/png ranges, binning factor/method, band ratios, upload disabled on the default station code) through logging instead of print(), so they also land in the night logs with a level and timestamp. Add an on-screen key legend to FRbinViewer (toggled with 'h') - the keys were previously only documented in the --help epilog. Document the runnable RMS/ pipeline modules in Utils/README.md, which previously only cataloged the Utils/ scripts. Co-Authored-By: Claude Fable 5 --- RMS/ConfigReader.py | 32 +++++++++++++++++--------------- RMS/Reprocess.py | 44 ++++++++++++++++++++++---------------------- Utils/FRbinViewer.py | 24 +++++++++++++++++++++--- Utils/README.md | 21 +++++++++++++++++++++ 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/RMS/ConfigReader.py b/RMS/ConfigReader.py index bf695ddd4..bcf5eda0f 100644 --- a/RMS/ConfigReader.py +++ b/RMS/ConfigReader.py @@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function +import logging import math import os import sys @@ -32,6 +33,11 @@ from ConfigParser import NoOptionError, RawConfigParser FileNotFoundError = IOError # Map FileNotFoundError to IOError in Python 2 +# Module logger - propagates to the root logger, so warnings end up in the night log once logging is +# initialized, and on the console before that +log = logging.getLogger(__name__) + + # Used to determine if ML filtering is available TFLITE_AVAILABLE = False @@ -751,7 +757,7 @@ def parse(path, strict=True): # Disable upload if the default station name is used if config.stationID == "XX0001": - print("Disabled upload because the default station code is used!") + log.warning("Disabled upload because the default station code is used!") config.upload_enabled = False @@ -1141,8 +1147,7 @@ def parseCapture(config, parser): # Limit the FPS to 1 million, as the time precision of datetime is 1 us if config.fps > 1000000: config.fps = 1000000 - print() - print("WARNING! The FPS has been limited to 1,000,000!") + log.warning("The FPS has been limited to 1,000,000!") if parser.has_option(section, "camera_buffer"): config.camera_buffer = parser.getint(section, "camera_buffer") @@ -1161,8 +1166,8 @@ def parseCapture(config, parser): if (config.fov_w <= 0) or (config.fov_h <= 0): - print('The field of view in the config file (fov_h and fov_w) have to be positive numbers!') - print('Make sure to set the approximate FOV size correctly!') + log.error('The field of view in the config file (fov_h and fov_w) have to be positive numbers!') + log.error('Make sure to set the approximate FOV size correctly!') sys.exit() @@ -1196,7 +1201,7 @@ def parseCapture(config, parser): # If the option is absent from the config, fall back to the class default. save_requested = parser.getboolean(section, "save_frames", fallback=config.save_frames) if save_requested and not config.ffmpeg_binary: - print("save_frames requested but FFmpeg not available - disabling.") + log.warning("save_frames requested but FFmpeg not available - disabling.") config.save_frames = False else: config.save_frames = save_requested @@ -1211,8 +1216,7 @@ def parseCapture(config, parser): # Must be an integer between 0 and 100 if not 0 <= config.jpgs_quality <= 100: config.jpgs_quality = 90 - print() - print("WARNING! The jpgs_quality must be between 0 and 100. It has been reset to 90!") + log.warning("The jpgs_quality must be between 0 and 100. It has been reset to 90!") # Load the PNG compression if parser.has_option(section, "png_compression"): @@ -1221,8 +1225,7 @@ def parseCapture(config, parser): # Must be an integer between 0 and 9 if not 0 <= config.png_compression <= 9: config.png_compression = 3 - print() - print("WARNING! The png_compression must be between 0 and 9. It has been reset to 3!") + log.warning("The png_compression must be between 0 and 9. It has been reset to 3!") # Load the interval for saving video frame @@ -1473,8 +1476,7 @@ def parseMeteorDetection(config, parser): # Check that the given bin size is a factor of 2 if bin_factor > 1: if math.log(bin_factor, 2)/int(math.log(bin_factor, 2)) != 1: - print('Warning! The given binning factor is not a factor of 2!') - print('Defaulting to 1...') + log.warning('The given binning factor is not a factor of 2! Defaulting to 1...') bin_factor = 1 config.detection_binning_factor = bin_factor @@ -1485,8 +1487,8 @@ def parseMeteorDetection(config, parser): bin_method_list = ['sum', 'avg'] if bin_method not in bin_method_list: - print('Warning! The binning method {:s} is not an allowed binning method: ', bin_method_list) - print('Defaulting to avg...') + log.warning('The binning method {:s} is not an allowed binning method: {}. ' + 'Defaulting to avg...'.format(bin_method, bin_method_list)) bin_method = 'avg' config.detection_binning_method = bin_method @@ -1708,7 +1710,7 @@ def parseCalibration(config, parser): # If they're all zero, use the V band if all([x == 0 for x in config.star_catalog_band_ratios]): config.star_catalog_band_ratios[1] = 1.0 - print('Warning! All band ratios are zero! Using the V band as the default band ratio...') + log.warning('All band ratios are zero! Using the V band as the default band ratio...') if parser.has_option(section, "platepar_name"): diff --git a/RMS/Reprocess.py b/RMS/Reprocess.py index 4f034bb05..804cd74bd 100644 --- a/RMS/Reprocess.py +++ b/RMS/Reprocess.py @@ -277,8 +277,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) generateCalibrationReport(config, night_data_dir, platepar=platepar) except Exception as e: - log.debug('Generating calibration report failed with the message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Generating calibration report failed with the message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) @@ -290,8 +290,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) sporadic_color=config.sporadic_color) except Exception as e: - log.debug('Shower association failed with the message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Shower association failed with the message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) @@ -339,8 +339,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) except Exception as e: - log.debug("Generating a FOV KML file failed with the message:\n" + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning("Generating a FOV KML file failed with the message:\n" + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) @@ -351,8 +351,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) mask=mask, platepar=platepar) except Exception as e: - log.debug("Preparing flux files failed with the message:\n" + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning("Preparing flux files failed with the message:\n" + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) else: @@ -369,8 +369,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) plotFieldsums(night_data_dir, config) except Exception as e: - log.debug('Plotting field sums failed with message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Plotting field sums failed with message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) @@ -429,8 +429,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) except Exception as e: print("Error in archiving process: {}".format(e)) except Exception as e: - log.debug('Archiving FT files failed with message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Archiving FT files failed with message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) log.info('Making a flat...') @@ -440,8 +440,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) flat_img = makeFlat(night_data_dir, config) except Exception as e: - log.debug('Making a flat failed with message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Making a flat failed with message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) flat_img = None @@ -479,8 +479,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) extra_files.append(timelapse_path) except Exception as e: - log.debug('Generating a timelapse failed with message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Generating a timelapse failed with message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) log.info('Plotting timestamp intervals...') @@ -504,8 +504,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) except Exception as e: - log.debug('Plotting timestamp interval failed with message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Plotting timestamp interval failed with message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) # Generate a config audit report log.info('Generate config audit report') @@ -528,8 +528,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) extra_files.append(audit_file_path) except Exception as e: - log.debug('Generating config audit failed with message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Generating config audit failed with message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) ### Add extra files to archive @@ -655,8 +655,8 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) except Exception as e: - log.debug('Finalizing Observation Summary failed with message:\n' + repr(e)) - log.debug(repr(traceback.format_exception(*sys.exc_info()))) + log.warning('Finalizing Observation Summary failed with message:\n' + repr(e)) + log.warning(repr(traceback.format_exception(*sys.exc_info()))) obs_summary_to_log = serialize(config, night_directory=night_data_dir, final=True) log.info("\n\nObservation Summary\n===================\n\n" + obs_summary_to_log + "\n\n") diff --git a/Utils/FRbinViewer.py b/Utils/FRbinViewer.py index fdeb718ba..754dd93ab 100644 --- a/Utils/FRbinViewer.py +++ b/Utils/FRbinViewer.py @@ -260,6 +260,11 @@ def destroyWindow(self): self._mpl_ready = False +# On-screen key legend, toggled with the 'h' key while viewing +KEY_LEGEND = "Keys: SPACE pause | 1 prev file | 2 next line | q quit | h hide keys" +show_key_legend = True + + def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=None, hide=False, avg_background=False, split=False, add_timestamp=False, add_frame_number=False, append_ff_to_video=False, add_shower_name=False, associations={}): @@ -284,6 +289,8 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N """ + global show_key_legend + if extract_format is None: extract_format = 'png' @@ -459,6 +466,10 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N # Resize large frames so they fit comfortably on screen. display_image = resizeImageIfNeed(img) + + # Draw the key legend + if show_key_legend: + addTextToImage(display_image, KEY_LEGEND, 10, 40) # Abort preview if the backend fails so we can continue processing. if not display or not display.showFrame(display_image): if display: @@ -483,6 +494,7 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N # Space key: pause display. # 1: previous file. # 2: next line. + # h: toggle the key legend. # q: Quit. key = display.waitKey(wait_time) @@ -491,14 +503,19 @@ def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format=N display.destroyWindow() return -1 - elif key == ord("2"): + elif key == ord("2"): break - elif key == ord(" "): - + elif key == ord(" "): + # Pause/unpause video pause_flag = not pause_flag + elif key == ord("h"): + + # Toggle the on-screen key legend + show_key_legend = not show_key_legend + elif key == ord("q"): os._exit(0) @@ -673,6 +690,7 @@ def loadShowerAssociations(folder, configuration): Space: pause display. 1: previous file. 2: next line. + h: toggle the on-screen key legend. q: Quit. """, formatter_class=argparse.RawTextHelpFormatter) diff --git a/Utils/README.md b/Utils/README.md index ccd884c56..3de4f0342 100644 --- a/Utils/README.md +++ b/Utils/README.md @@ -82,6 +82,27 @@ Detailed atmospheric and orbital analysis tools, particularly for computing mete --- +## ⚙️ Runnable Modules in `RMS/` + +The main pipeline modules live in the `RMS/` package and are run the same way, using `python -m RMS.`: + +- **`RMS.StartCapture`**: Starts the capture pipeline - recording, compression, detection, calibration, and upload. The main entry point for running a station. +- **`RMS.Reprocess`**: Reprocesses a night directory from raw FF files - detection, calibration, archiving, and all night products. +- **`RMS.DetectStarsAndMeteors`**: Runs star extraction and meteor detection on FF files in a given directory. +- **`RMS.ExtractStars`**: Extracts stars from FF files and writes a CALSTARS file. +- **`RMS.ArchiveDetections`**: Archives detection results of a night into upload-ready archive files. +- **`RMS.UploadManager`**: Uploads archived night data to the server over SFTP. +- **`RMS.EventMonitor`**: Monitors the network event list and uploads footage if a fireball trajectory crossed the station's field of view. +- **`RMS.DownloadMask`**: Downloads a new mask file for the station from the server. +- **`RMS.DownloadPlatepar`**: Downloads an updated platepar calibration file from the server. +- **`RMS.CaptureDuration`**: Prints the capture start time and duration for the night (given the station coordinates). +- **`RMS.DeleteOldObservations`**: Frees up disk space by deleting old observation data. +- **`RMS.MLFilter`**: Filters detections using the machine-learning meteor classifier. +- **`RMS.ClearSkyDetector`**: Estimates which parts of the night had clear skies. +- **`RMS.CaptureModeSwitcher`**: Switches between day and night capture modes based on the Sun's altitude. + +--- + ### General Usage To use a script, you can generally invoke it as a Python module to ensure that the main `RMS` dependency paths are resolved properly: ```bash From b95a62b446674a468414b216f52d8089f1e2aed9 Mon Sep 17 00:00:00 2001 From: Denis Vida Date: Tue, 7 Jul 2026 09:51:04 +0200 Subject: [PATCH 2/2] Only list RMS/ modules with real CLIs as runnable in the README ArchiveDetections, UploadManager, DownloadPlatepar, and CaptureModeSwitcher have hardcoded test stubs in their __main__ blocks rather than user CLIs, so following the README could trigger unintended test file/network operations. Mark them as internal instead. Co-Authored-By: Claude Fable 5 --- Utils/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Utils/README.md b/Utils/README.md index 3de4f0342..6d168dc8c 100644 --- a/Utils/README.md +++ b/Utils/README.md @@ -90,16 +90,14 @@ The main pipeline modules live in the `RMS/` package and are run the same way, u - **`RMS.Reprocess`**: Reprocesses a night directory from raw FF files - detection, calibration, archiving, and all night products. - **`RMS.DetectStarsAndMeteors`**: Runs star extraction and meteor detection on FF files in a given directory. - **`RMS.ExtractStars`**: Extracts stars from FF files and writes a CALSTARS file. -- **`RMS.ArchiveDetections`**: Archives detection results of a night into upload-ready archive files. -- **`RMS.UploadManager`**: Uploads archived night data to the server over SFTP. - **`RMS.EventMonitor`**: Monitors the network event list and uploads footage if a fireball trajectory crossed the station's field of view. - **`RMS.DownloadMask`**: Downloads a new mask file for the station from the server. -- **`RMS.DownloadPlatepar`**: Downloads an updated platepar calibration file from the server. - **`RMS.CaptureDuration`**: Prints the capture start time and duration for the night (given the station coordinates). - **`RMS.DeleteOldObservations`**: Frees up disk space by deleting old observation data. - **`RMS.MLFilter`**: Filters detections using the machine-learning meteor classifier. - **`RMS.ClearSkyDetector`**: Estimates which parts of the night had clear skies. -- **`RMS.CaptureModeSwitcher`**: Switches between day and night capture modes based on the Sun's altitude. + +Other pipeline modules (e.g. `RMS.ArchiveDetections`, `RMS.UploadManager`, `RMS.DownloadPlatepar`, `RMS.CaptureModeSwitcher`) are internal - their `__main__` blocks are hardcoded test stubs, not user CLIs. They run as part of `RMS.StartCapture`/`RMS.Reprocess`. ---