diff --git a/.config b/.config index 2d99f803d..c879ff043 100644 --- a/.config +++ b/.config @@ -437,7 +437,10 @@ event_monitor_check_interval_fast: 3 [Compression] - +; Specify whether the FITS file's data units (HDUs) should be losslessly compressed. +; When set to 'true', HDUs are losslessly compressed, potentially reducing FITS file size by 30 to 60%. +; Set to 'false' if compatibility with deprecated FITS software is a concern. +hdu_compress: true [FireballDetection] diff --git a/RMS/Compression.py b/RMS/Compression.py index 735e62827..a62a6c9ef 100644 --- a/RMS/Compression.py +++ b/RMS/Compression.py @@ -153,7 +153,8 @@ def saveFF(self, arr, startTime, N): ff.starttime = dt.isoformat(timespec='microseconds') # Write the FF file - FFfile.write(ff, self.data_dir, filename_millis, fmt=self.config.ff_format) + FFfile.write(ff, self.data_dir, filename_millis, fmt=self.config.ff_format, + compress=self.config.hdu_compress) return filename_millis, filename_micros diff --git a/RMS/ConfigReader.py b/RMS/ConfigReader.py index aed8acc63..4fab4746b 100644 --- a/RMS/ConfigReader.py +++ b/RMS/ConfigReader.py @@ -519,8 +519,8 @@ def __init__(self): self.event_monitor_check_interval = 30 self.event_monitor_check_interval_fast = 5 - - + ##### Compression + self.hdu_compress = False ##### Weave compilation arguments self.extra_compile_args = ["-O3"] @@ -1427,7 +1427,11 @@ def parseBuildArgs(config, parser): def parseCompression(config, parser): section = "Compression" - pass + if not parser.has_section(section): + return + + if parser.has_option(section, "hdu_compress"): + config.hdu_compress = parser.getboolean(section, "hdu_compress") diff --git a/RMS/Formats/FFfile.py b/RMS/Formats/FFfile.py index 480572bd9..189ac8939 100644 --- a/RMS/Formats/FFfile.py +++ b/RMS/Formats/FFfile.py @@ -167,7 +167,7 @@ def read(directory, filename, fmt=None, array=False, full_filename=False, verbos -def write(ff, directory, filename, fmt=None): +def write(ff, directory, filename, fmt=None, compress=True): """ Write a FF structure to a FITS file in specified directory. Arguments: @@ -208,7 +208,7 @@ def write(ff, directory, filename, fmt=None): # RMS fits format elif fmt == 'fits': - writeFFfits(ff, directory, filename) + writeFFfits(ff, directory, filename, compress=compress) def reconstructFrame(ff, frame_no, avepixel=False): diff --git a/RMS/Formats/FFfits.py b/RMS/Formats/FFfits.py index 6495b8c64..890f49660 100644 --- a/RMS/Formats/FFfits.py +++ b/RMS/Formats/FFfits.py @@ -131,7 +131,7 @@ def read(directory, filename, array=False, full_filename=False, memmap=True): -def write(ff, directory, filename): +def write(ff, directory, filename, compress=True): """ Write a FF structure to a FITS file in specified directory. Arguments: @@ -172,12 +172,19 @@ def write(ff, directory, filename): ff.avepixel = ff.avepixel[0] ff.stdpixel = ff.stdpixel[0] - # Add the maxpixel to the list - maxpixel_hdu = fits.ImageHDU(ff.maxpixel, name='MAXPIXEL') - maxframe_hdu = fits.ImageHDU(ff.maxframe, name='MAXFRAME') - avepixel_hdu = fits.ImageHDU(ff.avepixel, name='AVEPIXEL') - stdpixel_hdu = fits.ImageHDU(ff.stdpixel, name='STDPIXEL') + # Add the data arrays to the list and specify compression algorithm + if compress: + maxpixel_hdu = fits.CompImageHDU(ff.maxpixel, name='MAXPIXEL', compression_type='RICE_1') + maxframe_hdu = fits.CompImageHDU(ff.maxframe, name='MAXFRAME', compression_type='RICE_1') + avepixel_hdu = fits.CompImageHDU(ff.avepixel, name='AVEPIXEL', compression_type='RICE_1') + stdpixel_hdu = fits.CompImageHDU(ff.stdpixel, name='STDPIXEL', compression_type='RICE_1') + else: + maxpixel_hdu = fits.ImageHDU(ff.maxpixel, name='MAXPIXEL') + maxframe_hdu = fits.ImageHDU(ff.maxframe, name='MAXFRAME') + avepixel_hdu = fits.ImageHDU(ff.avepixel, name='AVEPIXEL') + stdpixel_hdu = fits.ImageHDU(ff.stdpixel, name='STDPIXEL') + # Create the primary part prim = fits.PrimaryHDU(header=head) diff --git a/Utils/ConvertCompressedFits.py b/Utils/ConvertCompressedFits.py new file mode 100644 index 000000000..e4e33b56a --- /dev/null +++ b/Utils/ConvertCompressedFits.py @@ -0,0 +1,110 @@ +""" +This script converts FITS files between compressed and uncompressed formats. +Designed to work recursively, it searches for all .fits files within a specified input directory +and its subdirectories, then converts each file preserving the original directory structure +in a separate output directory. + +Usage: + python ConvertCompressedFits.py [--compress | --decompress] + +Where: + is the directory containing the .fits files to convert. + is the directory where the converted .fits files will be saved. + --compress: Convert uncompressed FITS to compressed (RICE_1) format. + --decompress: Convert compressed FITS to uncompressed format (default). + +Examples: + python ConvertCompressedFits.py /path/to/uncompressed /path/to/compressed --compress + python ConvertCompressedFits.py /path/to/compressed /path/to/uncompressed --decompress + +""" + +import argparse +import os +import sys +import glob + +from RMS.Formats.FFfits import read as readFFfits +from RMS.Formats.FFfits import write as writeFFfits + + +def findFitsFiles(directory): + """Recursively find all FITS files in a directory and its subdirectories. + + Arguments: + directory: [str] The path to the directory to search. + + Return: + [list] List of paths to FITS files found. + + """ + fits_files = [] + for root, dirs, files in os.walk(directory): + fits_files.extend(glob.glob(os.path.join(root, '*.fits'))) + return fits_files + + +def convertDirectoryFits(input_directory, output_directory, compress=False): + """Convert all FITS files found in the input directory. + + Arguments: + input_directory: [str] The path to the folder containing the files to convert. + output_directory: [str] The path to the folder where the converted files will be saved. + compress: [bool] If True, compress the output files. If False, decompress them. + + Return: + None + + """ + fits_files = findFitsFiles(input_directory) + action = "Compressed" if compress else "Decompressed" + + for fits_file in fits_files: + try: + # Preserve the path structure in the output directory + relative_path = os.path.relpath(fits_file, input_directory) + output_path = os.path.join(output_directory, relative_path) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + # Read the FITS file + ff = readFFfits('', fits_file, array=True, full_filename=True) + + # Write with specified compression setting + writeFFfits(ff, os.path.dirname(output_path), os.path.basename(output_path), compress=compress) + print(f"{action} {fits_file} to {output_path}") + except Exception as e: + print(f"Failed to convert {fits_file}: {e}", file=sys.stderr) + + +if __name__ == "__main__": + + ### COMMAND LINE ARGUMENTS + + # Init the command line arguments parser + + arg_parser = argparse.ArgumentParser(description="Convert FITS files between compressed and uncompressed " + "formats in a directory (and subdirectories).") + + arg_parser.add_argument("input_directory", + help="The directory to search for FITS files.") + + arg_parser.add_argument("output_directory", + help="The directory where the converted FITS files will be saved.") + + mode_group = arg_parser.add_mutually_exclusive_group() + mode_group.add_argument("--compress", action="store_true", + help="Compress FITS files using RICE_1 algorithm.") + mode_group.add_argument("--decompress", action="store_true", default=True, + help="Decompress FITS files (default).") + + cml_args = arg_parser.parse_args() + + ######################### + + if not os.path.isdir(cml_args.input_directory): + print("The specified input directory does not exist.", file=sys.stderr) + sys.exit(1) + + os.makedirs(cml_args.output_directory, exist_ok=True) + + convertDirectoryFits(cml_args.input_directory, cml_args.output_directory, compress=cml_args.compress)