From 6faa07e2f0e8ab4bf40d7578fc41991b1e9419b8 Mon Sep 17 00:00:00 2001 From: Cybis320 Date: Mon, 25 Mar 2024 12:52:40 -0700 Subject: [PATCH 01/13] Add lossless compression to FITS data units --- RMS/Formats/FFfits.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/RMS/Formats/FFfits.py b/RMS/Formats/FFfits.py index 17cc209f3..595e9999d 100644 --- a/RMS/Formats/FFfits.py +++ b/RMS/Formats/FFfits.py @@ -164,10 +164,10 @@ def write(ff, directory, filename): ff.stdpixel = ff.stdpixel[0] # Add the maxpixle to the list - maxpixel_hdu = fits.ImageHDU(ff.maxpixel, name='MAXPIXEL') - maxframe_hdu = fits.ImageHDU(ff.maxframe, name='MAXFRAME') - avepixel_hdu = fits.ImageHDU(ff.avepixel, name='AVEPIXEL') - stdpixel_hdu = fits.ImageHDU(ff.stdpixel, name='STDPIXEL') + maxpixel_hdu = fits.CompImageHDU(ff.maxpixel, name='MAXPIXEL', compression_type='RICE_1') + maxframe_hdu = fits.CompImageHDU(ff.maxframe, name='MAXFRAME', compression_type='RICE_1') + avepixel_hdu = fits.CompImageHDU(ff.avepixel, name='AVEPIXEL', compression_type='RICE_1') + stdpixel_hdu = fits.CompImageHDU(ff.stdpixel, name='STDPIXEL', compression_type='RICE_1') # Create the primary part prim = fits.PrimaryHDU(header=head) From 234e9385e6c4a8a7d55dc7c802c88d600ff842c8 Mon Sep 17 00:00:00 2001 From: Cybis320 Date: Mon, 25 Mar 2024 14:10:31 -0700 Subject: [PATCH 02/13] Edit comment --- RMS/Formats/FFfits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RMS/Formats/FFfits.py b/RMS/Formats/FFfits.py index 595e9999d..d8d6ea3b5 100644 --- a/RMS/Formats/FFfits.py +++ b/RMS/Formats/FFfits.py @@ -163,7 +163,7 @@ def write(ff, directory, filename): ff.avepixel = ff.avepixel[0] ff.stdpixel = ff.stdpixel[0] - # Add the maxpixle to the list + # Add the data arrays to the list and specify compression algorithm maxpixel_hdu = fits.CompImageHDU(ff.maxpixel, name='MAXPIXEL', compression_type='RICE_1') maxframe_hdu = fits.CompImageHDU(ff.maxframe, name='MAXFRAME', compression_type='RICE_1') avepixel_hdu = fits.CompImageHDU(ff.avepixel, name='AVEPIXEL', compression_type='RICE_1') From d4498aadd61c96c590c61a1494325ed6fbda7c86 Mon Sep 17 00:00:00 2001 From: Cybis320 Date: Wed, 27 Mar 2024 15:48:08 -0700 Subject: [PATCH 03/13] Add hdu_compress config flag --- .config | 5 ++++- RMS/Compression.py | 12 ++++++------ RMS/ConfigReader.py | 10 +++++++--- RMS/Formats/FFfile.py | 4 ++-- RMS/Formats/FFfits.py | 19 +++++++++++++------ 5 files changed, 32 insertions(+), 18 deletions(-) diff --git a/.config b/.config index a7532bc30..eb83c5dcb 100644 --- a/.config +++ b/.config @@ -237,7 +237,10 @@ event_monitor_check_interval_fast: 3 [Compression] - +; Specify whether the FITS file's data units (HDUs) should be losslessly compressed. +; When set to 'true', HDUs are losslessly compressed, potentially reducing FITS file size by 30 to 60%. +; Set to 'false' if compatibility with deprecated FITS software is a concern. +hdu_compress: true [FireballDetection] diff --git a/RMS/Compression.py b/RMS/Compression.py index e64a83455..ff8411c40 100644 --- a/RMS/Compression.py +++ b/RMS/Compression.py @@ -121,12 +121,11 @@ def saveFF(self, arr, startTime, N): micros = int((startTime - floor(startTime))*1000000) millis = int((startTime - floor(startTime))*1000) - - filename_millis = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(millis).zfill(3) \ - + "_" + str(N).zfill(7) + filename_millis = str(self.config.stationID).zfill(3) + "_" + date_string + "_" \ + + str(millis).zfill(3) + "_" + str(N).zfill(7) - filename_micros = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(micros).zfill(6) \ - + "_" + str(N).zfill(7) + filename_micros = str(self.config.stationID).zfill(3) + "_" + date_string + "_" \ + + str(micros).zfill(6) + "_" + str(N).zfill(7) ff = FFStruct.FFStruct() ff.array = arr @@ -140,7 +139,8 @@ def saveFF(self, arr, startTime, N): ff.starttime = date_string + "_" + str(micros).zfill(6) # Write the FF file - FFfile.write(ff, self.data_dir, filename_millis, fmt=self.config.ff_format) + FFfile.write(ff, self.data_dir, filename_millis, fmt=self.config.ff_format, + compress=self.config.hdu_compress) return filename_millis, filename_micros diff --git a/RMS/ConfigReader.py b/RMS/ConfigReader.py index 40f91fff0..387f03b18 100644 --- a/RMS/ConfigReader.py +++ b/RMS/ConfigReader.py @@ -382,8 +382,8 @@ def __init__(self): self.event_monitor_check_interval = 30 self.event_monitor_check_interval_fast = 5 - - + ##### Compression + self.hdu_compress = True ##### Weave compilation arguments self.extra_compile_args = ["-O3"] @@ -1121,7 +1121,11 @@ def parseBuildArgs(config, parser): def parseCompression(config, parser): section = "Compression" - pass + if not parser.has_section(section): + return + + if parser.has_option(section, "hdu_compress"): + config.hdu_compress = parser.getboolean(section, "hdu_compress") diff --git a/RMS/Formats/FFfile.py b/RMS/Formats/FFfile.py index 75c61df63..7152d4f9c 100644 --- a/RMS/Formats/FFfile.py +++ b/RMS/Formats/FFfile.py @@ -99,7 +99,7 @@ def read(directory, filename, fmt=None, array=False, full_filename=False, verbos -def write(ff, directory, filename, fmt=None): +def write(ff, directory, filename, fmt=None, compress=True): """ Write a FF structure to a FITS file in specified directory. Arguments: @@ -140,7 +140,7 @@ def write(ff, directory, filename, fmt=None): # RMS fits format elif fmt == 'fits': - writeFFfits(ff, directory, filename) + writeFFfits(ff, directory, filename, compress=compress) def reconstructFrame(ff, frame_no, avepixel=False): diff --git a/RMS/Formats/FFfits.py b/RMS/Formats/FFfits.py index d8d6ea3b5..a4a4ac5cc 100644 --- a/RMS/Formats/FFfits.py +++ b/RMS/Formats/FFfits.py @@ -122,7 +122,7 @@ def read(directory, filename, array=False, full_filename=False): -def write(ff, directory, filename): +def write(ff, directory, filename, compress=True): """ Write a FF structure to a FITS file in specified directory. Arguments: @@ -164,11 +164,18 @@ def write(ff, directory, filename): ff.stdpixel = ff.stdpixel[0] # Add the data arrays to the list and specify compression algorithm - maxpixel_hdu = fits.CompImageHDU(ff.maxpixel, name='MAXPIXEL', compression_type='RICE_1') - maxframe_hdu = fits.CompImageHDU(ff.maxframe, name='MAXFRAME', compression_type='RICE_1') - avepixel_hdu = fits.CompImageHDU(ff.avepixel, name='AVEPIXEL', compression_type='RICE_1') - stdpixel_hdu = fits.CompImageHDU(ff.stdpixel, name='STDPIXEL', compression_type='RICE_1') + if compress: + maxpixel_hdu = fits.CompImageHDU(ff.maxpixel, name='MAXPIXEL', compression_type='RICE_1') + maxframe_hdu = fits.CompImageHDU(ff.maxframe, name='MAXFRAME', compression_type='RICE_1') + avepixel_hdu = fits.CompImageHDU(ff.avepixel, name='AVEPIXEL', compression_type='RICE_1') + stdpixel_hdu = fits.CompImageHDU(ff.stdpixel, name='STDPIXEL', compression_type='RICE_1') + else: + maxpixel_hdu = fits.ImageHDU(ff.maxpixel, name='MAXPIXEL') + maxframe_hdu = fits.ImageHDU(ff.maxframe, name='MAXFRAME') + avepixel_hdu = fits.ImageHDU(ff.avepixel, name='AVEPIXEL') + stdpixel_hdu = fits.ImageHDU(ff.stdpixel, name='STDPIXEL') + # Create the primary part prim = fits.PrimaryHDU(header=head) @@ -176,7 +183,7 @@ def write(ff, directory, filename): hdulist = fits.HDUList([prim, maxpixel_hdu, maxframe_hdu, avepixel_hdu, stdpixel_hdu]) # Save the FITS - hdulist.writeto(file_path, overwrite=True) + hdulist.writeto(file_path, overwrite=overwrite) From fcc1474f1cd6094ad92d5795d4b328a7178be793 Mon Sep 17 00:00:00 2001 From: Cybis320 Date: Wed, 27 Mar 2024 15:48:41 -0700 Subject: [PATCH 04/13] Add utility to convert compressed FITS to uncompressed FITS --- Utils/ConvertCompressedFits.py | 92 ++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 Utils/ConvertCompressedFits.py diff --git a/Utils/ConvertCompressedFits.py b/Utils/ConvertCompressedFits.py new file mode 100644 index 000000000..bd5f12609 --- /dev/null +++ b/Utils/ConvertCompressedFits.py @@ -0,0 +1,92 @@ +""" +This script automates the conversion of compressed FITS files to uncompressed FITS files. +Designed to work recursively, it searches for all .fits files within a specified input directory +and its subdirectories, then converts each file to an uncompressed format, preserving the original +directory structure in a separate output directory. + +Usage: + python ConvertCompressedFits.py + +Where: + is the directory containing the compressed .fits files to convert. + is the directory where the uncompressed .fits files will be saved. + +Example: + python ConvertCompressedFits.py /path/to/compressed /path/to/uncompressed + +""" + +import argparse +import os +import sys +import glob + +from RMS.Formats import FFfits + + +def findFitsFiles(directory): + """Recursively find all FITS files in a directory and its subdirectories. + + Arguments: + directory: [str] The path to the folder containing the files to convert. + + """ + fits_files = [] + for root, dirs, files in os.walk(directory): + fits_files.extend(glob.glob(os.path.join(root, '*.fits'))) + return fits_files + + +def convertDirectoryFits(input_directory, output_directory): + """Convert all FITS files found in the input directory to uncompressed + files in the output directory. + + Arguments: + input_directory: [str] The path to the folder containing the files to convert. + output_directory: [str] The path to the folder where the converted files will be saved. + + """ + fits_files = findFitsFiles(input_directory) + for fits_file in fits_files: + try: + # Assuming the path structure should be preserved in the output directory + relative_path = os.path.relpath(fits_file, input_directory) + output_path = os.path.join(output_directory, relative_path) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + # Read the compressed FITS file + ff = FFfits.read('', fits_file, array=True, full_filename=True) + + # Write it uncompressed + FFfits.write(ff, os.path.dirname(output_path), os.path.basename(output_path), compress=False) + print(f"Converted {fits_file} to {output_path}") + except Exception as e: + print(f"Failed to convert {fits_file}: {e}", file=sys.stderr) + + +if __name__ == "__main__": + + ### COMMAND LINE ARGUMENTS + + # Init the command line arguments parser + + arg_parser = argparse.ArgumentParser(description="Convert all compressed FITS files in a directory (and " + "subdirectories) to uncompressed FITS files in a separate directory.") + + arg_parser.add_argument("input_directory", + help="The directory to search for FITS files.") + + arg_parser.add_argument("output_directory", + help="The directory where the uncompressed FITS files will be saved.") + + cml_args = arg_parser.parse_args() + + ######################### + + if not os.path.isdir(cml_args.input_directory): + print("The specified input directory does not exist.", file=sys.stderr) + sys.exit(1) + + os.makedirs(cml_args.output_directory, exist_ok=True) + + convertDirectoryFits(cml_args.input_directory, cml_args.output_directory) From 3bbd02b2feb1bea8238d644d580f4e3bb35f3d72 Mon Sep 17 00:00:00 2001 From: Cybis320 Date: Wed, 27 Mar 2024 15:55:15 -0700 Subject: [PATCH 05/13] Fix Typo --- RMS/Formats/FFfits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RMS/Formats/FFfits.py b/RMS/Formats/FFfits.py index a4a4ac5cc..5108ca16e 100644 --- a/RMS/Formats/FFfits.py +++ b/RMS/Formats/FFfits.py @@ -183,7 +183,7 @@ def write(ff, directory, filename, compress=True): hdulist = fits.HDUList([prim, maxpixel_hdu, maxframe_hdu, avepixel_hdu, stdpixel_hdu]) # Save the FITS - hdulist.writeto(file_path, overwrite=overwrite) + hdulist.writeto(file_path, overwrite=True) From 349e331520f81e6f507fb77ef8ef392fbf38b96c Mon Sep 17 00:00:00 2001 From: Cybis320 Date: Wed, 27 Mar 2024 16:32:34 -0700 Subject: [PATCH 06/13] Change import code --- Utils/ConvertCompressedFits.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Utils/ConvertCompressedFits.py b/Utils/ConvertCompressedFits.py index bd5f12609..213f9332b 100644 --- a/Utils/ConvertCompressedFits.py +++ b/Utils/ConvertCompressedFits.py @@ -21,7 +21,8 @@ import sys import glob -from RMS.Formats import FFfits +from RMS.Formats.FFfits import read as readFFfits +from RMS.Formats.FFfits import write as writeFFfits def findFitsFiles(directory): @@ -29,7 +30,10 @@ def findFitsFiles(directory): Arguments: directory: [str] The path to the folder containing the files to convert. - + + Return: + None + """ fits_files = [] for root, dirs, files in os.walk(directory): @@ -45,6 +49,9 @@ def convertDirectoryFits(input_directory, output_directory): input_directory: [str] The path to the folder containing the files to convert. output_directory: [str] The path to the folder where the converted files will be saved. + Return: + None + """ fits_files = findFitsFiles(input_directory) for fits_file in fits_files: @@ -55,10 +62,10 @@ def convertDirectoryFits(input_directory, output_directory): os.makedirs(os.path.dirname(output_path), exist_ok=True) # Read the compressed FITS file - ff = FFfits.read('', fits_file, array=True, full_filename=True) + ff = readFFfits('', fits_file, array=True, full_filename=True) # Write it uncompressed - FFfits.write(ff, os.path.dirname(output_path), os.path.basename(output_path), compress=False) + writeFFfits(ff, os.path.dirname(output_path), os.path.basename(output_path), compress=False) print(f"Converted {fits_file} to {output_path}") except Exception as e: print(f"Failed to convert {fits_file}: {e}", file=sys.stderr) From 6db8cf1a1fcc665bb5e8c5e3fc693cc51f61f62b Mon Sep 17 00:00:00 2001 From: Cybis320 Date: Wed, 27 Mar 2024 17:15:49 -0700 Subject: [PATCH 07/13] Default hdu_compress to False --- .config | 2 +- RMS/ConfigReader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.config b/.config index eb83c5dcb..1c65cea00 100644 --- a/.config +++ b/.config @@ -240,7 +240,7 @@ event_monitor_check_interval_fast: 3 ; Specify whether the FITS file's data units (HDUs) should be losslessly compressed. ; When set to 'true', HDUs are losslessly compressed, potentially reducing FITS file size by 30 to 60%. ; Set to 'false' if compatibility with deprecated FITS software is a concern. -hdu_compress: true +hdu_compress: false [FireballDetection] diff --git a/RMS/ConfigReader.py b/RMS/ConfigReader.py index 387f03b18..58d45b75e 100644 --- a/RMS/ConfigReader.py +++ b/RMS/ConfigReader.py @@ -383,7 +383,7 @@ def __init__(self): self.event_monitor_check_interval_fast = 5 ##### Compression - self.hdu_compress = True + self.hdu_compress = False ##### Weave compilation arguments self.extra_compile_args = ["-O3"] From cff79abc995273970a4a8dc98b98b21c9fa9fd25 Mon Sep 17 00:00:00 2001 From: Denis Vida Date: Mon, 25 Nov 2024 09:47:46 -0500 Subject: [PATCH 08/13] make sure config file name is preserved if different from .config --- RMS/Reprocess.py | 4 +++- RMS/StartCapture.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/RMS/Reprocess.py b/RMS/Reprocess.py index 1e69fe650..3ca485eb2 100644 --- a/RMS/Reprocess.py +++ b/RMS/Reprocess.py @@ -436,9 +436,11 @@ def processNight(night_data_dir, config, detection_results=None, nodetect=False) # Make the name of the audit file audit_file_name = night_data_dir_name.replace("_detected", "") + "_config_audit_report.txt" + config_file_name = os.path.basename(config.config_file_name) + # Construct the full path for files audit_file_path = os.path.join(night_data_dir, audit_file_name) - config_file_path = os.path.join(night_data_dir, ".config") + config_file_path = os.path.join(night_data_dir, config_file_name) with open(audit_file_path, 'w') as f: f.write(compareConfigs(config_file_path, diff --git a/RMS/StartCapture.py b/RMS/StartCapture.py index 4125a36f7..0776fafeb 100644 --- a/RMS/StartCapture.py +++ b/RMS/StartCapture.py @@ -243,7 +243,11 @@ def runCapture(config, duration=None, video_file=None, nodetect=False, detect_en # Copy the used config file to the capture directory if os.path.isfile(config.config_file_name): try: - shutil.copy2(config.config_file_name, os.path.join(night_data_dir, ".config")) + + # Get the name of the originating config file + config_file_name = os.path.basename(config.config_file_name) + + shutil.copy2(config.config_file_name, os.path.join(night_data_dir, config_file_name)) except: log.error("Cannot copy the config file to the capture directory!") From 307e6c3d44adbb716efaabe9cadb8a45b6e5681b Mon Sep 17 00:00:00 2001 From: Denis Vida Date: Fri, 29 Nov 2024 09:12:05 -0500 Subject: [PATCH 09/13] added a directory check --- RMS/DetectStarsAndMeteors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RMS/DetectStarsAndMeteors.py b/RMS/DetectStarsAndMeteors.py index f7c32c36a..a2fa67d7f 100644 --- a/RMS/DetectStarsAndMeteors.py +++ b/RMS/DetectStarsAndMeteors.py @@ -194,6 +194,10 @@ def saveDetections(detection_results, ff_dir, config): # Generate the name for the CALSTARS file calstars_name = 'CALSTARS_' + prefix + '.txt' + # Create the ff_dir if it does not exist + if not os.path.isdir(ff_dir): + os.makedirs(ff_dir) + # Write detected stars to the CALSTARS file CALSTARS.writeCALSTARS(star_list, ff_dir, calstars_name, config.stationID, config.height, config.width) From 2c7c1f1db61ecda938d137bef3aea55e530a8a14 Mon Sep 17 00:00:00 2001 From: Denis Vida Date: Fri, 29 Nov 2024 12:19:10 -0500 Subject: [PATCH 10/13] moved h264parse element --- RMS/BufferedCapture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RMS/BufferedCapture.py b/RMS/BufferedCapture.py index 10aa31370..c65d6219b 100644 --- a/RMS/BufferedCapture.py +++ b/RMS/BufferedCapture.py @@ -559,12 +559,12 @@ def createGstreamDevice(self, video_format, gst_decoder='decodebin', source_to_tee = ( "rtspsrc buffer-mode=1 {:s} " "location=\"{:s}\" ! " - "rtph264depay ! h264parse ! tee name=t" + "rtph264depay ! tee name=t" ).format(protocol_str, device_url) # Branch for processing processing_branch = ( - "t. ! queue ! {:s} ! " + "t. ! queue ! h264parse ! {:s} ! " "queue leaky=downstream max-size-buffers=100 max-size-bytes=0 max-size-time=0 ! " "videoconvert ! video/x-raw,format={:s} ! " "queue max-size-buffers=100 max-size-bytes=0 max-size-time=0 ! " @@ -576,7 +576,7 @@ def createGstreamDevice(self, video_format, gst_decoder='decodebin', video_location = os.path.join(video_file_dir, "video_%05d.mkv") storage_branch = ( - "t. ! queue max-size-buffers=0 max-size-bytes=0 max-size-time=0 ! " + "t. ! queue max-size-buffers=0 max-size-bytes=0 max-size-time=0 ! h264parse ! " "splitmuxsink location={:s} max-size-time={:d} muxer-factory=matroskamux" ).format(video_location, int(segment_duration_sec*1e9)) From f9db5fe6f4f5a9386a90ae2fa74b179915c0a134 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 30 Dec 2025 10:30:41 -0700 Subject: [PATCH 11/13] Cleanup merging issues --- .config | 7 +++++-- RMS/Compression.py | 9 +++++---- RMS/DetectStarsAndMeteors.py | 4 ---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.config b/.config index cf5554626..c78cd15f1 100644 --- a/.config +++ b/.config @@ -124,7 +124,7 @@ camera_settings_path: ./camera_settings.json ; If no "XX0001.camera_init_done" flag file exists in the RMS root dir, ; the commands in the "init" section are sent to the camera. After a ; successful run, the flag file is created so future launches skip the -; initialization step. Remove flag to perform initialization again. +; initialization step. Remove flag to perform initialization again. initialize_camera: false uyvy_pixelformat: false @@ -418,7 +418,10 @@ event_monitor_check_interval_fast: 3 [Compression] - +; Specify whether the FITS file's data units (HDUs) should be losslessly compressed. +; When set to 'true', HDUs are losslessly compressed, potentially reducing FITS file size by 30 to 60%. +; Set to 'false' if compatibility with deprecated FITS software is a concern. +hdu_compress: true [FireballDetection] diff --git a/RMS/Compression.py b/RMS/Compression.py index 9b42f2cac..a62a6c9ef 100644 --- a/RMS/Compression.py +++ b/RMS/Compression.py @@ -125,11 +125,12 @@ def saveFF(self, arr, startTime, N): micros = int((startTime - floor(startTime))*1000000) millis = int((startTime - floor(startTime))*1000) - filename_millis = str(self.config.stationID).zfill(3) + "_" + date_string + "_" \ - + str(millis).zfill(3) + "_" + str(N).zfill(7) + + filename_millis = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(millis).zfill(3) \ + + "_" + str(N).zfill(7) - filename_micros = str(self.config.stationID).zfill(3) + "_" + date_string + "_" \ - + str(micros).zfill(6) + "_" + str(N).zfill(7) + filename_micros = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(micros).zfill(6) \ + + "_" + str(N).zfill(7) ff = FFStruct.FFStruct() ff.array = arr diff --git a/RMS/DetectStarsAndMeteors.py b/RMS/DetectStarsAndMeteors.py index 554e1e35b..97f4cf047 100644 --- a/RMS/DetectStarsAndMeteors.py +++ b/RMS/DetectStarsAndMeteors.py @@ -359,10 +359,6 @@ def saveDetections(detection_results, ff_dir, config, output_suffix=''): if not os.path.exists(ff_dir): os.makedirs(ff_dir) - # Create the ff_dir if it does not exist - if not os.path.isdir(ff_dir): - os.makedirs(ff_dir) - # Write detected stars to the CALSTARS file CALSTARS.writeCALSTARS(star_list, ff_dir, calstars_name, config.stationID, config.height, config.width) From 87f19e287e3ab978952c1b8b9b30963f2a1c2b73 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 30 Dec 2025 10:33:42 -0700 Subject: [PATCH 12/13] Fix docstring --- Utils/ConvertCompressedFits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utils/ConvertCompressedFits.py b/Utils/ConvertCompressedFits.py index 213f9332b..7bdf6723d 100644 --- a/Utils/ConvertCompressedFits.py +++ b/Utils/ConvertCompressedFits.py @@ -29,10 +29,10 @@ def findFitsFiles(directory): """Recursively find all FITS files in a directory and its subdirectories. Arguments: - directory: [str] The path to the folder containing the files to convert. + directory: [str] The path to the directory to search. Return: - None + [list] List of paths to FITS files found. """ fits_files = [] From f3770be904f028b126cfb7f903a5bb9b256d1631 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 30 Dec 2025 10:37:52 -0700 Subject: [PATCH 13/13] Make convert script bi-directional --- Utils/ConvertCompressedFits.py | 57 ++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/Utils/ConvertCompressedFits.py b/Utils/ConvertCompressedFits.py index 7bdf6723d..e4e33b56a 100644 --- a/Utils/ConvertCompressedFits.py +++ b/Utils/ConvertCompressedFits.py @@ -1,18 +1,21 @@ """ -This script automates the conversion of compressed FITS files to uncompressed FITS files. -Designed to work recursively, it searches for all .fits files within a specified input directory -and its subdirectories, then converts each file to an uncompressed format, preserving the original -directory structure in a separate output directory. +This script converts FITS files between compressed and uncompressed formats. +Designed to work recursively, it searches for all .fits files within a specified input directory +and its subdirectories, then converts each file preserving the original directory structure +in a separate output directory. Usage: - python ConvertCompressedFits.py + python ConvertCompressedFits.py [--compress | --decompress] Where: - is the directory containing the compressed .fits files to convert. - is the directory where the uncompressed .fits files will be saved. + is the directory containing the .fits files to convert. + is the directory where the converted .fits files will be saved. + --compress: Convert uncompressed FITS to compressed (RICE_1) format. + --decompress: Convert compressed FITS to uncompressed format (default). -Example: - python ConvertCompressedFits.py /path/to/compressed /path/to/uncompressed +Examples: + python ConvertCompressedFits.py /path/to/uncompressed /path/to/compressed --compress + python ConvertCompressedFits.py /path/to/compressed /path/to/uncompressed --decompress """ @@ -41,32 +44,34 @@ def findFitsFiles(directory): return fits_files -def convertDirectoryFits(input_directory, output_directory): - """Convert all FITS files found in the input directory to uncompressed - files in the output directory. +def convertDirectoryFits(input_directory, output_directory, compress=False): + """Convert all FITS files found in the input directory. Arguments: input_directory: [str] The path to the folder containing the files to convert. output_directory: [str] The path to the folder where the converted files will be saved. + compress: [bool] If True, compress the output files. If False, decompress them. Return: None """ fits_files = findFitsFiles(input_directory) + action = "Compressed" if compress else "Decompressed" + for fits_file in fits_files: try: - # Assuming the path structure should be preserved in the output directory + # Preserve the path structure in the output directory relative_path = os.path.relpath(fits_file, input_directory) output_path = os.path.join(output_directory, relative_path) os.makedirs(os.path.dirname(output_path), exist_ok=True) - # Read the compressed FITS file + # Read the FITS file ff = readFFfits('', fits_file, array=True, full_filename=True) - # Write it uncompressed - writeFFfits(ff, os.path.dirname(output_path), os.path.basename(output_path), compress=False) - print(f"Converted {fits_file} to {output_path}") + # Write with specified compression setting + writeFFfits(ff, os.path.dirname(output_path), os.path.basename(output_path), compress=compress) + print(f"{action} {fits_file} to {output_path}") except Exception as e: print(f"Failed to convert {fits_file}: {e}", file=sys.stderr) @@ -77,14 +82,20 @@ def convertDirectoryFits(input_directory, output_directory): # Init the command line arguments parser - arg_parser = argparse.ArgumentParser(description="Convert all compressed FITS files in a directory (and " - "subdirectories) to uncompressed FITS files in a separate directory.") - + arg_parser = argparse.ArgumentParser(description="Convert FITS files between compressed and uncompressed " + "formats in a directory (and subdirectories).") + arg_parser.add_argument("input_directory", help="The directory to search for FITS files.") - + arg_parser.add_argument("output_directory", - help="The directory where the uncompressed FITS files will be saved.") + help="The directory where the converted FITS files will be saved.") + + mode_group = arg_parser.add_mutually_exclusive_group() + mode_group.add_argument("--compress", action="store_true", + help="Compress FITS files using RICE_1 algorithm.") + mode_group.add_argument("--decompress", action="store_true", default=True, + help="Decompress FITS files (default).") cml_args = arg_parser.parse_args() @@ -96,4 +107,4 @@ def convertDirectoryFits(input_directory, output_directory): os.makedirs(cml_args.output_directory, exist_ok=True) - convertDirectoryFits(cml_args.input_directory, cml_args.output_directory) + convertDirectoryFits(cml_args.input_directory, cml_args.output_directory, compress=cml_args.compress)