diff --git a/README.md b/README.md index 43b2b1e8d..9f41014e1 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,15 @@ pip install . Once your system is fully set up and configured (via the `.config` file, as explained in the Wiki), you can run various RMS modules. +### Available tools +To see a list of the most commonly used tools with short descriptions, run: + +```bash +python -m RMS +``` + +Any of the listed commands can then be run as e.g. `python -m RMS skyfit .` or `python -m RMS status`. A full catalog of the utility scripts is in [Utils/README.md](Utils/README.md). + ### Capturing video and saving data To start the automated video capture, navigate to your base RMS folder in the terminal and run: diff --git a/RMS/CLITools.py b/RMS/CLITools.py new file mode 100644 index 000000000..5a4de69f2 --- /dev/null +++ b/RMS/CLITools.py @@ -0,0 +1,60 @@ +""" Shared helpers for command-line interfaces of RMS scripts. + +This module is intentionally lightweight (stdlib-only at import time) so that it can be imported at the +top of any script without pulling in heavy dependencies. +""" + +from __future__ import print_function, division, absolute_import + + +def addConfigArgument(parser, help_text=None): + """ Register the standard -c/--config argument on an argparse parser. + + Unlike the legacy nargs=1 pattern, the value is stored as a plain string, so scripts can pass + cml_args.config directly to loadConfig() without indexing. + + Arguments: + parser: [argparse.ArgumentParser] Parser to add the argument to. + + Keyword arguments: + help_text: [str] Custom help text. If None, a standard description is used. + + """ + + if help_text is None: + help_text = ("Path to a config file which will be used instead of the default one." + " To load the .config file in the given data directory, use '.' as the value.") + + parser.add_argument('-c', '--config', metavar='CONFIG_PATH', type=str, help=help_text) + + +def loadConfig(cml_args_config, dir_path=None): + """ Load the RMS config given the value of the --config argument. + + Accepts the value in any historical shape (None, a plain string, or the 1-element list produced by + the legacy nargs=1 pattern) and normalizes it before handing it to + RMS.ConfigReader.loadConfigFromDirectory, which expects a list. + + Arguments: + cml_args_config: [None/str/list] Value of cml_args.config from argparse. + + Keyword arguments: + dir_path: [str or list] Path to the working directory (or several). If None, the current + directory is used. + + Return: + config: [Config instance] Loaded config. + + """ + + # Deferred import so importing this module stays cheap + import RMS.ConfigReader as cr + + if dir_path is None: + dir_path = '.' + + # Normalize the config argument to the list shape loadConfigFromDirectory expects + if isinstance(cml_args_config, str): + cml_args_config = [cml_args_config] + + return cr.loadConfigFromDirectory(cml_args_config, dir_path) diff --git a/RMS/Formats/ObservationSummary.py b/RMS/Formats/ObservationSummary.py index 202988bd1..a25a7f0b7 100644 --- a/RMS/Formats/ObservationSummary.py +++ b/RMS/Formats/ObservationSummary.py @@ -207,6 +207,29 @@ def addRequiredColumns(conn, d): return set(getColumns(conn)) +def getLatestObservationRecord(conn): + """Get the most recent observation summary record from the database. + + Arguments: + conn: connection to database. + + Return: + [dict or None] The latest record as a column name -> value dict, or None if the table is empty. + """ + + cursor = conn.execute( + f"SELECT * FROM {OBSERVATIONS_TABLE_NAME} ORDER BY {NIGHT_DATA_DIR_COL} DESC LIMIT 1") + + row = cursor.fetchone() + + if row is None: + return None + + columns = [description[0] for description in cursor.description] + + return dict(zip(columns, row)) + + def storeDictInDB(conn, d, debug=False): """Store the dict d in the observation summary database, create new columns if needed. diff --git a/RMS/__main__.py b/RMS/__main__.py new file mode 100644 index 000000000..2c29853d7 --- /dev/null +++ b/RMS/__main__.py @@ -0,0 +1,106 @@ +""" Umbrella launcher for RMS tools. + +Run "python -m RMS" to list the available tools, and "python -m RMS [args...]" to run one. +Every command is a thin alias for a module which can also be run directly, e.g. +"python -m RMS skyfit ." is the same as "python -m Utils.SkyFit2 .". +""" + +from __future__ import print_function, division, absolute_import + +import runpy +import sys + + +# Registry of subcommands: (command, module, one-line description), grouped by topic. +# Keep the descriptions short - they are printed as a single line each. +COMMANDS = [ + ("Station operation", [ + ("start", "RMS.StartCapture", "Start the capture pipeline (recording, detection, upload)"), + ("reprocess", "RMS.Reprocess", "Reprocess a night directory from raw FF files"), + ("status", "Utils.StationStatus", "Show the station health report from the last night"), + ("capture-duration", "RMS.CaptureDuration", "Print the capture start time and duration for tonight"), + ("delete-old", "RMS.DeleteOldObservations", "Free up disk space by deleting old observations"), + ]), + ("Calibration & astrometry", [ + ("skyfit", "Utils.SkyFit2", "Astrometric calibration and manual reduction GUI"), + ("calibration-report", "Utils.CalibrationReport", "Generate a calibration quality report for a night"), + ("flat", "Utils.MakeFlat", "Make a flat field image from a night of data"), + ("shower-association", "Utils.ShowerAssociation", "Associate detected meteors with showers"), + ]), + ("Camera setup", [ + ("camera-control", "Utils.CameraControl", "Get/set IP camera parameters"), + ("camera-manager", "Utils.CamManager", "Find and configure cameras on the network (GUI)"), + ("camera-address", "Utils.SetCameraAddress", "Change the camera IP address"), + ("camera-params", "Utils.setAllCameraParams", "Apply the standard RMS camera settings"), + ("livestream", "Utils.ShowLiveStream", "Show the live video stream from the camera"), + ]), + ("Viewing & media", [ + ("liveview", "Utils.LiveViewer", "Slideshow of FF files as they are created"), + ("frbin", "Utils.FRbinViewer", "View fireball detections from FR bin files"), + ("checknight", "Utils.CheckNight", "Visually inspect the images of a night"), + ("stack", "Utils.StackFFs", "Stack FF files into a single image"), + ("trackstack", "Utils.TrackStack", "Star-aligned stack of a whole night"), + ("timelapse", "Utils.GenerateTimelapse", "Make a timelapse video of a night"), + ("mp4s", "Utils.GenerateMP4s", "Make MP4 clips of individual detections"), + ("thumbnails", "Utils.GenerateThumbnails", "Make a thumbnail grid of a night"), + ]), + ("Configuration", [ + ("audit-config", "Utils.AuditConfig", "Compare the station .config against the current template"), + ("migrate-config", "Utils.MigrateConfig", "Upgrade the .config to the latest template format"), + ]), + ("Analysis", [ + ("flux", "Utils.Flux", "Compute single-station meteor shower flux"), + ("fov-kml", "Utils.FOVKML", "Make a Google Earth KML of the camera field of view"), + ("fov-skymap", "Utils.FOVSkyMap", "Plot the camera field of view on a sky map"), + ]), +] + + +def printToolList(): + + print(__doc__) + + for group_name, entries in COMMANDS: + + print("{:s}:".format(group_name)) + + for command, module, description in entries: + print(" {:<20s} {:s}".format(command, description)) + + print() + + print("Run \"python -m RMS -h\" for the arguments of an individual tool.") + print("Tools not listed here can be run directly as modules - see Utils/README.md for a full catalog.") + + +def main(): + + args = sys.argv[1:] + + # No command, an explicit list request, or a help flag - print the tool list + if (not args) or (args[0] in ("tools", "list", "-h", "--help", "help")): + printToolList() + return + + command = args[0] + + # Find the command in the registry + module = None + for _, entries in COMMANDS: + for cmd_name, cmd_module, _ in entries: + if command == cmd_name: + module = cmd_module + break + + if module is None: + print("Unknown command: {:s}".format(command)) + print("Run \"python -m RMS\" to list the available commands.") + sys.exit(2) + + # Hand over to the target module as if it was run with python -m + sys.argv = [module] + args[1:] + runpy.run_module(module, run_name="__main__", alter_sys=True) + + +if __name__ == "__main__": + main() diff --git a/Utils/Grouping3DRunner.py b/Utils/Grouping3DRunner.py index 58aea7c69..15b123d3f 100644 --- a/Utils/Grouping3DRunner.py +++ b/Utils/Grouping3DRunner.py @@ -2,14 +2,14 @@ from __future__ import print_function, division, absolute_import +from RMS.CLITools import addConfigArgument, loadConfig from RMS.VideoExtraction import Extractor -import RMS.ConfigReader as cr import RMS.Formats.FFfile as FFfile from RMS.Routines.Grouping3D import find3DLines, getAllPoints import os -import sys +import argparse import time @@ -27,11 +27,20 @@ if __name__ == "__main__": - # Extract the directory name from the given argument - bin_dir = sys.argv[1] + arg_parser = argparse.ArgumentParser(description="""Run the fireball extractor on FF files in the \ +given directory and show the detected points and lines as 3D plots.""") + + arg_parser.add_argument('dir_path', metavar='DIR_PATH', type=str, + help='Path to the directory with FF files.') + + addConfigArgument(arg_parser) + + cml_args = arg_parser.parse_args() + + bin_dir = cml_args.dir_path # Load config file - config = cr.parse(".config") + config = loadConfig(cml_args.config, bin_dir) print('Directory:', bin_dir) diff --git a/Utils/PointsViewer.py b/Utils/PointsViewer.py index a80111626..a60ba4154 100644 --- a/Utils/PointsViewer.py +++ b/Utils/PointsViewer.py @@ -1,32 +1,41 @@ +""" Show points detected by the video extractor on a given FF file as a 3D plot. """ + +from __future__ import print_function, division, absolute_import + +import os +import argparse + import numpy as np from mpl_toolkits.mplot3d import Axes3D import matplotlib.pyplot as plt -import sys + +from RMS.CLITools import addConfigArgument, loadConfig from RMS.Formats import FFfile from RMS import VideoExtraction -import RMS.ConfigReader as cr -def view(ff): - config = cr.parse(".config") - + +def view(dir_path, file_name, config): + + ff = FFfile.read(dir_path, file_name, array=True) + ve = VideoExtraction.Extractor(config) ve.frames = np.empty((256, ff.nrows, ff.ncols)) ve.compressed = ff.array - + points = np.array(ve.findPoints()) - - plot(points, ff.nrows//config.f, ff.ncols//config.f) -def plot(points, y_dim, x_dim): + plot(points, ff.nrows//config.f, ff.ncols//config.f, file_name) + +def plot(points, y_dim, x_dim, name): fig = plt.figure() - + ax = fig.add_subplot(111, projection='3d') plt.title(name) - + y = points[:,0] x = points[:,1] z = points[:,2] - + # Plot points in 3D ax.scatter(x, y, z) @@ -34,15 +43,28 @@ def plot(points, y_dim, x_dim): ax.set_zlim(0, 255) plt.xlim([0, x_dim]) plt.ylim([0, y_dim]) - + ax.set_ylabel("Y") ax.set_xlabel("X") ax.set_zlabel("Time") - + plt.show() if __name__ == "__main__": - ff = FFfile.read(sys.argv[1], sys.argv[2], array=True) - - view(ff) \ No newline at end of file + + arg_parser = argparse.ArgumentParser(description="""Show points detected by the video extractor \ +on the given FF file as a 3D plot.""") + + arg_parser.add_argument('ff_path', metavar='FF_PATH', type=str, + help='Path to the FF file to inspect.') + + addConfigArgument(arg_parser) + + cml_args = arg_parser.parse_args() + + dir_path, file_name = os.path.split(os.path.abspath(cml_args.ff_path)) + + config = loadConfig(cml_args.config, dir_path) + + view(dir_path, file_name, config) diff --git a/Utils/SaturationSimulation.py b/Utils/SaturationSimulation.py index 236512831..fbaff2f34 100644 --- a/Utils/SaturationSimulation.py +++ b/Utils/SaturationSimulation.py @@ -3,6 +3,8 @@ from __future__ import print_function, division, absolute_import +import argparse + import numpy as np import matplotlib.pyplot as plt @@ -139,13 +141,30 @@ def _costFunc(mag_app_unsaturated, params): if __name__ == "__main__": + arg_parser = argparse.ArgumentParser(description="""Simulate the saturation of a moving meteor \ +modelled as a moving Gaussian, and plot the apparent vs actual magnitude for a range of PSF sizes.""") + + arg_parser.add_argument('-p', '--photomoffset', metavar='PHOTOM_OFFSET', type=float, default=10.7, + help='Photometric offset. Default: 10.7.') + + arg_parser.add_argument('-b', '--background', metavar='BACKGROUND_LVL', type=float, default=60, + help='Background level (0-255). Default: 60.') + + arg_parser.add_argument('-f', '--fps', metavar='FPS', type=float, default=25, + help='Frames per second of the simulated camera. Default: 25.') + + arg_parser.add_argument('-a', '--angvel', metavar='ANG_VEL', type=float, default=250, + help='Angular velocity of the meteor in px/s. Default: 250.') - photom_offset = 10.7 + arg_parser.add_argument('-s', '--stddevs', metavar='GAUSS_STDDEV', type=float, nargs='+', + default=[1.5, 1.3, 1.15], help='Gaussian PSF standard deviations to simulate. Default: 1.5 1.3 1.15.') + cml_args = arg_parser.parse_args() - background_lvl = 60 - fps = 25 - ang_vel = 250 # px/s + photom_offset = cml_args.photomoffset + background_lvl = cml_args.background + fps = cml_args.fps + ang_vel = cml_args.angvel # px/s #print(findUnsaturatedMagnitude(-0.5, photom_offset, 50, 25, 100, 1)) @@ -156,8 +175,8 @@ def _costFunc(mag_app_unsaturated, params): app_mag_range = np.append(np.linspace(-0.5, 2, 1000), np.linspace(2, 6, 10)) - # Generate a range of gaussian stddevs - gauss_stddevs = [1.5, 1.3, 1.15] + # Range of gaussian stddevs to simulate + gauss_stddevs = cml_args.stddevs diff --git a/Utils/SetCameraAddress.py b/Utils/SetCameraAddress.py index fd25a904e..ef8883cd2 100644 --- a/Utils/SetCameraAddress.py +++ b/Utils/SetCameraAddress.py @@ -1,3 +1,4 @@ +import argparse import struct import json import hashlib @@ -232,30 +233,35 @@ def set_command(self, command, data, code): if __name__ == '__main__': - if len(sys.argv) < 3: - print("This script allows you to set your Camera's IP address.") - print('') - print('To use the script, your camera must be attached to your home network') - print('and acessible via its default IP address. If you are not sure what') - print('what that is, you may be able to find out from your home router. Most routers') - print("have a page that displys connected clients. Look for one named 'HostName',") - print('and make a note of its address. Alternatively you can use free tools like ') - print('Advanced IP-Scanner to scan your network for the same name. ') - print('') - print('You will also need to know the address you want your camera to have.') - print('Normally this will be 192.168.42.10') - print('') - print('Once you have this information and the camera is on your network') - print('call this module again with two arguments, the CURRENT and DESIRED address, eg:') - print('') - print(' python -m Utils.SetCameraAddress 192.168.1.100 192.168.42.10') - print('') - print('replacing the two addresses as needed') - print('') - exit(0) - ipaddr = sys.argv[1] - newaddr = sys.argv[2] + arg_parser = argparse.ArgumentParser( + description="Set your camera's IP address.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""To use the script, your camera must be attached to your home network +and accessible via its default IP address. If you are not sure what +that is, you may be able to find out from your home router. Most routers +have a page that displays connected clients. Look for one named 'HostName', +and make a note of its address. Alternatively you can use free tools like +Advanced IP-Scanner to scan your network for the same name. + +You will also need to know the address you want your camera to have. +Normally this will be 192.168.42.10 + +Example: + + python -m Utils.SetCameraAddress 192.168.1.100 192.168.42.10 +""") + + arg_parser.add_argument('current_address', metavar='CURRENT_IP', type=str, + help='Current IP address of the camera.') + + arg_parser.add_argument('new_address', metavar='NEW_IP', type=str, + help='Desired new IP address of the camera, e.g. 192.168.42.10.') + + cml_args = arg_parser.parse_args() + + ipaddr = cml_args.current_address + newaddr = cml_args.new_address if not checkValidIPAddr(ipaddr) or not checkValidIPAddr(newaddr): print('') diff --git a/Utils/StationStatus.py b/Utils/StationStatus.py new file mode 100644 index 000000000..68431479f --- /dev/null +++ b/Utils/StationStatus.py @@ -0,0 +1,190 @@ +""" Headless station health report. + +Prints a summary of how the last observing night went (using the observation summary database) plus +the current state of the station (disk space, camera reachability, next capture time). Designed to be +run over SSH on stations without a display: + + python -m Utils.StationStatus + +or through the umbrella launcher: + + python -m RMS status +""" + +from __future__ import print_function, division, absolute_import + +import argparse +import contextlib +import json +import os +import re +import shutil +import sys + +from RMS.CLITools import addConfigArgument, loadConfig +from RMS.Formats.ObservationSummary import (OBSERVATION_DB_FILE_NAME, getObsDBConn, + getLatestObservationRecord, getObservationDuration, pingOnce) +from RMS.Misc import RmsDateTime + + +def getLastNightReport(config): + """ Load the most recent night's record from the observation summary database. + + Return: + [dict or None] Latest observation record, or None if no data is available yet. + """ + + db_path = os.path.join(config.data_dir, OBSERVATION_DB_FILE_NAME) + + # Don't create the database as a side effect of a read-only status check + if not os.path.isfile(db_path): + return None + + conn = getObsDBConn(config) + + if conn is None: + return None + + try: + record = getLatestObservationRecord(conn) + finally: + conn.close() + + return record + + +def getCurrentStatus(config): + """ Gather the live station state - disk space, camera reachability, next capture time. + + Return: + [dict] Current status values. Values which could not be determined are None. + """ + + status = {} + + # Free disk space in the data directory + data_dir = os.path.expanduser(config.data_dir) + try: + usage = shutil.disk_usage(data_dir) + status['disk_free_gb'] = usage.free/(1024**3) + status['disk_total_gb'] = usage.total/(1024**3) + except OSError: + status['disk_free_gb'] = None + status['disk_total_gb'] = None + + # Camera reachability - only for IP cameras where an IP can be extracted from the device string + camera_ip_match = re.search(r'(?:\d{1,3}\.){3}\d{1,3}', str(config.deviceID)) + if camera_ip_match is not None: + status['camera_ip'] = camera_ip_match.group() + status['camera_reachable'] = pingOnce(status['camera_ip']) + else: + status['camera_ip'] = None + status['camera_reachable'] = None + + # Next capture start and duration. The capture duration code works with naive UTC times, so a + # timezone-aware datetime must not be passed in. + try: + start_time, duration, end_time = getObservationDuration(config, RmsDateTime.utcnow()) + status['next_capture_start'] = str(start_time) + status['next_capture_duration_hrs'] = duration/3600 if duration is not None else None + except Exception: + status['next_capture_start'] = None + status['next_capture_duration_hrs'] = None + + return status + + +def formatReport(config, record, status): + """ Format the status report as human-readable text. """ + + def _get(key): + if record is None: + return 'n/a' + value = record.get(key) + return 'n/a' if value in (None, '') else str(value) + + lines = [] + lines.append("=" * 60) + lines.append("RMS station status - {:s}".format(config.stationID)) + lines.append("=" * 60) + + if record is None: + lines.append("") + lines.append("No observation data yet - the station hasn't completed a night.") + lines.append("") + + else: + lines.append("") + lines.append("Last night: {:s}".format(os.path.basename(_get('night_data_dir')))) + lines.append(" FF files captured: {:s} of {:s} expected".format( + _get('total_fits'), _get('total_expected_fits'))) + lines.append(" Lost capture time: {:s}".format(_get('fits_file_shortfall_as_time'))) + lines.append(" First / last FF: {:s} / {:s}".format( + _get('time_first_fits_file'), _get('time_last_fits_file'))) + lines.append(" First / last detection: {:s} / {:s}".format( + _get('time_first_detection'), _get('time_last_detection'))) + lines.append(" Days since detection: {:s}".format(_get('days_since_last_detection'))) + lines.append(" Tracebacks in log: {:s}".format(_get('traceback_count'))) + lines.append(" Time sync: {:s} (source: {:s}, offset {:s} ms)".format( + _get('clock_synchronized'), _get('clock_measurement_source'), _get('clock_ahead_ms'))) + lines.append(" Repo lag behind remote: {:s} days".format(_get('repository_lag_remote_days'))) + lines.append(" Camera pointing: az {:s}, alt {:s}, FOV {:s}x{:s} deg".format( + _get('camera_pointing_az'), _get('camera_pointing_alt'), + _get('camera_fov_h'), _get('camera_fov_v'))) + lines.append("") + + lines.append("Current state:") + + if status['disk_free_gb'] is not None: + lines.append(" Disk free: {:.1f} GB of {:.1f} GB ({:s})".format( + status['disk_free_gb'], status['disk_total_gb'], config.data_dir)) + else: + lines.append(" Disk free: n/a (data directory not found)") + + if status['camera_ip'] is not None: + lines.append(" Camera ({:s}): {:s}".format(status['camera_ip'], + "reachable" if status['camera_reachable'] else "NOT REACHABLE")) + else: + lines.append(" Camera: n/a (no IP camera configured)") + + if status['next_capture_start'] is not None: + lines.append(" Next capture: {:s} UTC, {:.1f} h".format( + status['next_capture_start'], status['next_capture_duration_hrs'])) + else: + lines.append(" Next capture: n/a") + + lines.append("=" * 60) + + return "\n".join(lines) + + +if __name__ == "__main__": + + arg_parser = argparse.ArgumentParser(description="""Print a headless station health report - how \ +the last night went (capture completeness, detections, errors) and the current station state (disk, \ +camera, next capture). Works over SSH, no display needed.""") + + addConfigArgument(arg_parser) + + arg_parser.add_argument('-j', '--json', action="store_true", + help="Output the report as JSON for use in scripts.") + + cml_args = arg_parser.parse_args() + + # With --json, keep stdout parseable - config loading prints its banner messages, so route them + # to stderr instead + if cml_args.json: + with contextlib.redirect_stdout(sys.stderr): + config = loadConfig(cml_args.config) + else: + config = loadConfig(cml_args.config) + + record = getLastNightReport(config) + status = getCurrentStatus(config) + + if cml_args.json: + print(json.dumps({'station_id': config.stationID, 'last_night': record, 'current': status}, + indent=4, sort_keys=True, default=str)) + + else: + print(formatReport(config, record, status)) diff --git a/Utils/setAllCameraParams.py b/Utils/setAllCameraParams.py index 2d81e9c2a..14177bedc 100644 --- a/Utils/setAllCameraParams.py +++ b/Utils/setAllCameraParams.py @@ -1,3 +1,4 @@ +import argparse import struct import json import hashlib @@ -279,7 +280,15 @@ def setCameraParam(cam, opts): if __name__ == '__main__': - ipaddr = sys.argv[1] + arg_parser = argparse.ArgumentParser(description="""Apply the standard set of RMS camera parameters \ +(video encoding, color, exposure, gain) to an IMX291-style IP camera over the DVRIP protocol.""") + + arg_parser.add_argument('ip_address', metavar='CAMERA_IP', type=str, + help='IP address of the camera, e.g. 192.168.42.10.') + + cml_args = arg_parser.parse_args() + + ipaddr = cml_args.ip_address if not checkValidIPAddr(ipaddr): print('ipaddress invalid')