Harmonise metadata creation (#140)

* Use sys.argv for metadata and standardise input files metadata

Author: anton-seaice

chlorophyll/chl_climatology_and_fill.py

@@ -70,18 +70,6 @@ def main():
     input_directory = os.path.abspath(args.input_directory)
     output_filename = os.path.abspath(args.output_filename)
 
-    this_file = os.path.normpath(__file__)
-
-    # Add some info about how the file was generated
-    runcmd = (
-        f"python3 {os.path.basename(this_file)} --input-directory={input_directory} "
-        f"--output-filename={output_filename}"
-    )
-
-    history_attrs = {
-        "history": get_provenance_metadata(this_file, runcmd),
-    }
-
     # Load the input data and compute the monthly climatology
     input_files = sorted(glob.glob(f"{input_directory}/*.nc"))
 
@@ -160,7 +148,7 @@ def main():
     chl.time.attrs["long_name"] = "Time"
     chl.time.attrs["standard_name"] = "time"
     chl.time.attrs["axis"] = "T"
-    chl.attrs |= history_attrs
+    chl.attrs |= get_provenance_metadata(input_files)
     comp = dict(zlib=True, complevel=4)
     encoding = {var: comp for var in chl.data_vars}
     # Time coords should be double type according for CF conventions

data_stream_xml_generation/generate_xml_datm_era5.py

@@ -85,17 +85,15 @@
 root = Element("file", id="stream", version="2.0")
 
 # Obtain metadata
-this_file = sys.argv[0]
-runcmd = " ".join(sys.argv)
-metadata_info = get_provenance_metadata(this_file, runcmd)
+metadata_info = get_provenance_metadata()
 
 # Add metadata
 metadata = SubElement(root, "metadata")
 SubElement(metadata, "File_type").text = "DATM xml file provides forcing data"
 SubElement(metadata, "date_generated").text = datetime.now().strftime(
     "%Y-%m-%d %H:%M:%S"
 )
-SubElement(metadata, "history").text = metadata_info
+SubElement(metadata, "history").text = metadata_info["history"]
 
 # Generate stream info elements with changing years
 for stream_name, era5_prefix, datavar_pairs, mapalgo, offset_seconds in STREAM_SPECS:

data_stream_xml_generation/generate_xml_datm_jra55.py

@@ -61,17 +61,15 @@
 root = Element("file", id="stream", version="2.0")
 
 # Obtain metadata
-this_file = sys.argv[0]
-runcmd = " ".join(sys.argv)
-metadata_info = get_provenance_metadata(this_file, runcmd)
+metadata_info = get_provenance_metadata()
 
 # Add metadata
 metadata = SubElement(root, "metadata")
 SubElement(metadata, "File_type").text = "DATM xml file provides forcing data"
 SubElement(metadata, "date_generated").text = datetime.now().strftime(
     "%Y-%m-%d %H:%M:%S"
 )
-SubElement(metadata, "history").text = metadata_info
+SubElement(metadata, "history").text = metadata_info["history"]
 
 # Define the stream info names and corresponding var names
 stream_info_names = [*STREAMS_AVE, *STREAMS_PT]

data_stream_xml_generation/generate_xml_drof_jra55.py

@@ -42,17 +42,15 @@
 root = Element("file", id="stream", version="2.0")
 
 # Obtain metadata
-this_file = sys.argv[0]
-runcmd = " ".join(sys.argv)
-metadata_info = get_provenance_metadata(this_file, runcmd)
+metadata_info = get_provenance_metadata()
 
 # Add metadata
 metadata = SubElement(root, "metadata")
 SubElement(metadata, "File_type").text = "DROF xml file provides river runoff data"
 SubElement(metadata, "date_generated").text = datetime.now().strftime(
     "%Y-%m-%d %H:%M:%S"
 )
-SubElement(metadata, "history").text = metadata_info
+SubElement(metadata, "history").text = metadata_info["history"]
 
 # Define the stream info names and corresponding var names
 stream_info_data = [

external_tidal_generation/generate_bottom_roughness_intermediate_woa.py

@@ -135,7 +135,7 @@
 
 path_root = Path(__file__).parents[1]
 sys.path.append(str(path_root))
-from scripts_common import get_provenance_metadata, md5sum
+from scripts_common import get_provenance_metadata
 
 
 def coriolis_f(lat: xr.DataArray) -> xr.DataArray:
@@ -838,30 +838,14 @@ def main():
         )
 
         # Add provenance metadata and MD5 hashes for input files.
-        this_file = os.path.normpath(__file__)
-        runcmd = (
-            f"mpirun -n $PBS_NCPUS python3 {os.path.basename(this_file)} "
-            f"--woa_temp_file={args.woa_temp_file} "
-            f"--woa_salt_file={args.woa_salt_file} "
-            f"--synbath_file={args.synbath_file} "
-            f"--chunk-lat={args.chunk_lat} "
-            f"--chunk-lon={args.chunk_lon} "
-            f"--nradial={args.nradial} "
-            f"--ntheta={args.ntheta} "
-            f"--earth-radius={args.earth_radius} "
-            f"--omega={args.omega} "
-            f"--print-every={args.print_every} "
-            f"--woa-intermediate-file={args.woa_intermediate_file} "
-        )
-
-        history = get_provenance_metadata(this_file, runcmd)
-        global_attrs = {"history": history}
-        file_hashes = [
-            f"{args.woa_temp_file} (md5 hash: {md5sum(args.woa_temp_file)})",
-            f"{args.woa_salt_file} (md5 hash: {md5sum(args.woa_salt_file)})",
-            f"{args.synbath_file} (md5 hash: {md5sum(args.synbath_file)})",
+        runcmd = f"mpirun -n $PBS_NCPUS python3 {' '.join(sys.argv)} "
+        input_files = [
+            args.woa_temp_file,
+            args.woa_salt_file,
+            args.synbath_file,
         ]
-        global_attrs["inputFile"] = ", ".join(file_hashes)
+        global_attrs = get_provenance_metadata(input_files, runcmd)
+
         ds_woa_output.attrs.update(global_attrs)
 
         ds_woa_output.to_netcdf(args.woa_intermediate_file)

external_tidal_generation/generate_bottom_roughness_polyfit.py

@@ -39,7 +39,7 @@
 path_root = Path(__file__).parents[1]
 sys.path.append(str(path_root))
 
-from scripts_common import get_provenance_metadata, md5sum
+from scripts_common import get_provenance_metadata
 from mesh_generation.generate_mesh import mom6_mask_detection
 
 
@@ -383,25 +383,10 @@ def main():
         )
 
         # Add provenance metadata and MD5 hashes for input files.
-        this_file = os.path.normpath(__file__)
-        runcmd = (
-            f"mpirun -n $PBS_NCPUS python3 {os.path.basename(this_file)} "
-            f"--high-res-topo-file={args.high_res_topo_file} "
-            f"--hgrid-file={args.hgrid_file} "
-            f"--topog-file={args.topog_file} "
-            f"--chunk-lat={args.chunk_lat} "
-            f"--chunk-lon={args.chunk_lon} "
-            f"--output={args.output}"
-        )
+        runcmd = f"mpirun -n $PBS_NCPUS python3 {' '.join(sys.argv)} "
+        input_files = [args.high_res_topo_file, args.hgrid_file, args.topog_file]
+        global_attrs = get_provenance_metadata(input_files, runcmd)
 
-        history = get_provenance_metadata(this_file, runcmd)
-        global_attrs = {"history": history}
-        file_hashes = [
-            f"{args.high_res_topo_file} (md5 hash: {md5sum(args.high_res_topo_file)})",
-            f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})",
-            f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})",
-        ]
-        global_attrs["inputFile"] = ", ".join(file_hashes)
         h2_out.attrs.update(global_attrs)
 
         h2_out.to_netcdf(args.output)

external_tidal_generation/generate_bottom_roughness_regrid.py

@@ -53,7 +53,7 @@
 
 path_root = Path(__file__).parents[1]
 sys.path.append(str(path_root))
-from scripts_common import get_provenance_metadata, md5sum
+from scripts_common import get_provenance_metadata
 from mesh_generation.generate_mesh import mom6_mask_detection
 
 
@@ -396,26 +396,8 @@ def main():
     print("Regridding done!")
 
     # Add provenance metadata and MD5 hashes for input files.
-    this_file = os.path.normpath(__file__)
-    runcmd = (
-        f"python3 {os.path.basename(this_file)} "
-        f"--topog_file={args.topog_file} "
-        f"--hgrid_file={args.hgrid_file} "
-        f"--woa_intermediate_file={args.woa_intermediate_file} "
-        f"--output_file={args.output_file} "
-        f"--method={args.method} "
-        f"--periodic_regrid={args.periodic_regrid} "
-        f"--periodic_lon_laplace={args.periodic_lon_laplace}"
-    )
-
-    history = get_provenance_metadata(this_file, runcmd)
-    global_attrs = {"history": history}
-    file_hashes = [
-        f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})",
-        f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})",
-        f"{args.woa_intermediate_file} (md5 hash: {md5sum(args.woa_intermediate_file)})",
-    ]
-    global_attrs["inputFile"] = ", ".join(file_hashes)
+    input_files = [args.hgrid_file, args.topog_file, args.woa_intermediate_file]
+    global_attrs = get_provenance_metadata(input_files)
     regrid_depth_var.attrs.update(global_attrs)
 
     output_path = Path(args.output_file)

external_tidal_generation/generate_tide_amplitude.py

@@ -38,7 +38,7 @@
 path_root = Path(__file__).parents[1]
 sys.path.append(str(path_root))
 
-from scripts_common import get_provenance_metadata, md5sum
+from scripts_common import get_provenance_metadata
 from mesh_generation.generate_mesh import mom6_mask_detection
 
 PRIMARY_CONSTITUENTS = ["m2", "s2", "n2", "k2", "k1", "o1", "p1", "q1"]
@@ -277,25 +277,8 @@ def main():
     tideamp = tideamp.rename({"x": "xh", "y": "yh"})
 
     # Add provenance metadata and MD5 hashes for input files.
-    this_file = os.path.normpath(__file__)
-    runcmd = (
-        f"python3 {os.path.basename(this_file)} "
-        f"--hgrid-file={args.hgrid_file} "
-        f"--topog-file={args.topog_file} "
-        f"--method={args.method} "
-        f"--data-path={args.data_path} "
-        f"--output={args.output} "
-    )
-
-    history = get_provenance_metadata(this_file, runcmd)
-    global_attrs = {"history": history}
-
-    # add md5 hashes for input files
-    file_hashes = [
-        f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})",
-        f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})",
-    ]
-    global_attrs["inputFile"] = ", ".join(file_hashes)
+    input_files = [args.hgrid_file, args.topog_file]
+    global_attrs = get_provenance_metadata(input_files)
     tideamp.attrs.update(global_attrs)
 
     tideamp.to_netcdf(args.output, unlimited_dims=["time"])

grid_generation/generate_vertical_grid.py

@@ -38,7 +38,7 @@
 path_root = Path(__file__).parents[1]
 sys.path.append(str(path_root))
 
-from scripts_common import get_provenance_metadata, md5sum
+from scripts_common import get_provenance_metadata
 
 # Define a small constant to initialize the iteration and prevent numerical issues
 epsilon = 0.001
@@ -91,18 +91,8 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename):
         : len(real_prop_z)
     ]  # Trim the spacing values to match the adjusted depth levels
 
-    this_file = os.path.normpath(__file__)
-
-    # Add some info about how the file was generated
-    runcmd = (
-        f"python3 {os.path.basename(this_file)} --H={H} --depfac={depfac} "
-        f"--dzd={dzd} "
-        f"--min_dz={min_dz} "
-        f"--output={output_filename} "
-    )
-
     # Write to NetCDF file
-    write_netcdf_file(output_filename, real_prop_z, this_file, runcmd)
+    write_netcdf_file(output_filename, real_prop_z)
 
     print(
         f"SUCCESS! A vertical grid with {len(real_prop_z) - 1} levels has been generated. "
@@ -111,7 +101,7 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename):
     )
 
 
-def write_netcdf_file(output_filename, real_prop_z, this_file, runcmd):
+def write_netcdf_file(output_filename, real_prop_z):
     """Function to write vertical grid data to a NetCDF file."""
     # Convert to float32 (single precision) to ensure values are exactly representable in single precision,
     # then convert back to float64 (double precision) for storage in NetCDF.
@@ -125,7 +115,7 @@ def write_netcdf_file(output_filename, real_prop_z, this_file, runcmd):
     zeta.standard_name = "depth"
     zeta.long_name = "vertical grid depth at top and bottom of each cell"
     eddyfile.variables["zeta"][:] = real_prop_z_float64
-    eddyfile.setncatts({"history": get_provenance_metadata(this_file, runcmd)})
+    eddyfile.setncatts(get_provenance_metadata())  # Obtain metadata
     eddyfile.close()
 
 

make_ryf/make_ryf.py

@@ -38,7 +38,7 @@
 path_root = Path(__file__).parents[1]
 sys.path.append(str(path_root))
 
-from scripts_common import get_provenance_metadata, md5sum
+from scripts_common import get_provenance_metadata
 
 FILLVALUE = 1e20
 # compression settings to use
@@ -113,7 +113,7 @@
     ds = {}
 
     for var in variables:
-        ryf_files = str()
+        input_files = []
         print(var)
         for y in (year1, year2):
             if source_data == "jra55v1p4" or source_data == "jra55v1p6":
@@ -147,7 +147,7 @@
             print("Loading {} for {}".format(files[0], y))
             ds[y] = xarray.open_dataset(files[0], decode_coords=False)
             # save info for metadata
-            ryf_files += f"{files[0]} (md5 hash: {md5sum(files[0])}, )"
+            input_files.append(files[0])
         # Make a copy of the second year without time_bnds
         ryf = ds[baseyear].drop_vars("time_bnds")
         ryf.encoding = ds[baseyear].encoding
@@ -205,10 +205,7 @@
         )
 
         # Add some info about how the file was generated
-        this_file = os.path.normpath(__file__)
-        runcmd = f"python3 {os.path.basename(this_file)}"
-        ryf.attrs |= {"RYF_creation": get_provenance_metadata(this_file, runcmd)}
-        ryf.attrs |= {"RYF_inputFiles": ryf_files}
+        ryf.attrs |= get_provenance_metadata(input_files)
 
         outfile = "RYF.{}.{}_{}.nc".format(var, year1, year2)
         print("Writing ", outfile)