Merge pull request #57 from smoia/feat/io

Improve extension handling, and make `.tsv.gz` the default extension

Author: smoia

nigsp/blocks.py

@@ -72,15 +72,15 @@ def export_metric(scgraph, outext, outprefix):
         except ImportError:
             LGR.warning(
                 "The necessary library for nifti export (nibabel) "
-                "was not found. Exporting metrics in CSV format instead."
+                "was not found. Exporting metrics in TSV format instead."
             )
-            outext = ".csv"
+            outext = ".tsv.gz"
         if scgraph.img is None:
             LGR.warning(
                 "A necessary atlas nifti image was not found. "
-                "Exporting metrics in CSV format instead."
+                "Exporting metrics in TSV format instead."
             )
-            outext = ".csv"
+            outext = ".tsv.gz"
 
     if scgraph.sdi is not None:
         if outext in io.EXT_NIFTI:
@@ -127,7 +127,7 @@ def plot_metric(scgraph, outprefix, atlas=None, thr=None):
                 atlas_plot = atlas
         except AttributeError:
             LGR.warning(
-                "The provided atlas is not in a format supported for " "markerplots."
+                "The provided atlas is not in a format supported for markerplots."
             )
             atlas_plot = None
 

nigsp/io.py

@@ -12,8 +12,12 @@
     List of supported nifti file extensions, in lower case.
 EXT_XLS : list
     List of supported XLS-like file extensions, in lower case.
-LGR
-    Logger
+EXT_ALL : list
+    All supported file extensions, in lower case.
+EXT_DICT : dictionary
+    Dictionary associating values to extension lists
+LOADMAT_DICT : dictionary
+    Dictionary assocting the same values in EXT_DICT to loading functions.
 """
 
 import logging
@@ -25,9 +29,9 @@
 from nigsp.utils import change_var_type
 
 EXT_1D = [".txt", ".csv", ".tsv", ".1d", ".par", ".tsv.gz", ".csv.gz"]
-EXT_XLS = [".xls"]
 EXT_MAT = [".mat"]
 EXT_NIFTI = [".nii", ".nii.gz"]
+EXT_XLS = [".xls"]
 EXT_ALL = EXT_1D + EXT_XLS + EXT_MAT + EXT_NIFTI
 
 EXT_DICT = {"1D": EXT_1D, "xls": EXT_XLS, "mat": EXT_MAT, "nifti": EXT_NIFTI}
@@ -84,7 +88,7 @@ def check_ext(all_ext, fname, scan=False, remove=False):
         if has_ext:
             obj_return += [fname[: -len(ext)], ext]  # case insensitive solution
         else:
-            obj_return += [fname, ""]
+            obj_return += [fname, None]
     else:
         obj_return += [fname]
 
@@ -404,12 +408,9 @@ def export_nifti(data, img, fname):
             "Please see install instructions."
         )
 
-    for e in EXT_NIFTI:
-        has_ext, fname, ext = check_ext(e, fname, remove=True)
-        if has_ext:
-            break
+    has_ext, fname, ext = check_ext(EXT_NIFTI, fname, remove=True)
 
-    if ext == "":
+    if ext is None:
         ext = ".nii.gz"
 
     LGR.info(f"Exporting nifti data into {fname}{ext}.")
@@ -437,6 +438,21 @@ def export_txt(data, fname, ext=None):
     0
         On a successful run
     """
+    has_ext, fname, ext_check = check_ext(EXT_ALL, fname, remove=True)
+
+    if has_ext:
+        if ext is not None:
+            LGR.warning(
+                f"Specified filename {fname}{ext_check} has an extension, but the "
+                f"extension {ext} was specified. Forcing specified extension."
+            )
+        else:
+            ext = ext_check
+    else:
+        if ext is None:
+            LGR.warning("Extension not specified. Forcing export in TSV.GZ format.")
+            ext = ".tsv.gz"
+
     if ext.lower() in [".csv", ".csv.gz", "", None]:
         delimiter = ","
     elif ext.lower() in [".tsv", ".tsv.gz"]:
@@ -494,22 +510,23 @@ def export_mtx(data, fname, ext=None):
     0
         On a successful run
     """
-    if ext is None:
-        # Check if extension was provided in fname.
-        for e in EXT_ALL:
-            has_ext, fname, ext = check_ext(e, fname, remove=True)
-            if has_ext:
-                break
-    elif ext.lower() not in EXT_ALL:
-        # Check if extension is supported.
-        ext = None
+    has_ext, fname, ext_check = check_ext(EXT_ALL, fname, remove=True)
+
+    if has_ext:
+        if ext is None:
+            ext = ext_check
+        else:
+            LGR.warning(
+                f"Specified filename {fname}{ext_check} has an extension, but the "
+                f"extension {ext} was specified. Forcing specified extension."
+            )
 
     if ext in [None, ""]:
         LGR.warning(
             "Extension not specified, or specified extension not "
-            "supported. Forcing export in CSV format."
+            "supported. Forcing export in TSV.GZ format."
         )
-        ext = ".csv"
+        ext = ".tsv.gz"
     elif ext.lower() in EXT_NIFTI:
         LGR.warning("Found nifti extension, exporting data in .1D instead")
         ext = ".1D"
@@ -520,7 +537,7 @@ def export_mtx(data, fname, ext=None):
             import scipy
         except ImportError:
             raise ImportError(
-                "To export .mat files, scipy is required. " "Please install it."
+                "To export .mat files, scipy is required. Please install it."
             )
         scipy.io.savemat(f"{fname}{ext}", {"data": data})
     elif ext.lower() in EXT_XLS:
@@ -529,13 +546,18 @@ def export_mtx(data, fname, ext=None):
         export_txt(data, fname, ext)
     else:
         raise BrokenPipeError(
-            f"This should not have happened: {ext} was the " "selected extension."
+            f"This should not have happened: {ext} was the selected extension."
         )
 
     return 0
 
 
-LOADMAT_DICT = {"1D": load_txt, "xls": load_xls, "mat": load_mat}
+LOADMAT_DICT = {
+    "1D": load_txt,
+    "xls": load_xls,
+    "mat": load_mat,
+    "nifti": load_nifti_get_mask,
+}
 
 
 """

nigsp/operations/nifti.py

@@ -64,9 +64,7 @@ def mat_to_vol(data, shape=None, asdata=None):
             )
         shape = asdata.shape
     elif shape is None:
-        raise ValueError(
-            "Both shape and asdata are empty. " "Must specify at least one"
-        )
+        raise ValueError("Both shape and asdata are empty. Must specify at least one")
 
     LGR.info(f"Reshape {data.ndim}D matrix into volume with shape {shape}.")
     return data.reshape(shape, order="F")
@@ -147,9 +145,7 @@ def unmask(data, mask, shape=None, asdata=None):
             )
         shape = asdata.shape
     elif shape is None:
-        raise ValueError(
-            "Both shape and asdata are empty. " "Must specify at least one."
-        )
+        raise ValueError("Both shape and asdata are empty. Must specify at least one.")
 
     if shape[: mask.ndim] != mask.shape:
         raise ValueError(
@@ -205,7 +201,7 @@ def apply_atlas(data, atlas, mask=None):
 
     if atlas.ndim > 3:
         raise NotImplementedError(
-            f"Files with {atlas.ndim} dimensions are not " "supported as atlases."
+            f"Files with {atlas.ndim} dimensions are not supported as atlases."
         )
     if data.shape[: mask.ndim] != mask.shape:
         raise ValueError(

nigsp/operations/timeseries.py

@@ -41,7 +41,7 @@ def normalise_ts(timeseries, globally=False):
     """
     if timeseries.ndim < 2 or (timeseries.ndim == 2 and timeseries.shape[1] == 1):
         LGR.warning(
-            "Given timeseries seems to be a single timepoint. " "Returning it as is."
+            "Given timeseries seems to be a single timepoint. Returning it as is."
         )
         return timeseries
 
@@ -81,7 +81,7 @@ def spc_ts(timeseries, globally=False):
     """
     if timeseries.ndim < 2 or (timeseries.ndim == 2 and timeseries.shape[1] == 1):
         LGR.warning(
-            "Given timeseries seems to be a single timepoint. " "Returning it as is."
+            "Given timeseries seems to be a single timepoint. Returning it as is."
         )
         return timeseries
 
@@ -119,7 +119,7 @@ def demean_ts(timeseries, globally=False):
     """
     if timeseries.ndim < 2 or (timeseries.ndim == 2 and timeseries.shape[1] == 1):
         LGR.warning(
-            "Given timeseries seems to be a single timepoint. " "Returning it as is."
+            "Given timeseries seems to be a single timepoint. Returning it as is."
         )
         return timeseries
 
@@ -157,7 +157,7 @@ def rescale_ts(timeseries, vmin=0, vmax=1, globally=False):
     """
     if timeseries.ndim < 2 or (timeseries.ndim == 2 and timeseries.shape[1] == 1):
         LGR.warning(
-            "Given timeseries seems to be a single timepoint. " "Returning it as is."
+            "Given timeseries seems to be a single timepoint. Returning it as is."
         )
         return timeseries
 

nigsp/tests/test_blocks.py

@@ -70,14 +70,14 @@ def test_export_metrics_nifti(sc_mtx, atlas, sdi, testdir):
     scgraph = SCGraph(mtx, ts, atlas=atlas_in, sdi=sdi_in)
     blocks.export_metric(scgraph, ".nii.gz", join(testdir, "molly_"))
 
-    assert isfile(join(testdir, "molly_sdi.csv"))
+    assert isfile(join(testdir, "molly_sdi.tsv.gz"))
 
     sys.modules["nibabel"] = None
     scgraph = SCGraph(mtx, ts, atlas=atlas_in, img=img, sdi=sdi_in)
     blocks.export_metric(scgraph, ".nii.gz", join(testdir, "molly_"))
     sys.modules["nibabel"] = nibabel
 
-    assert isfile(join(testdir, "molly_sdi.csv"))
+    assert isfile(join(testdir, "molly_sdi.tsv.gz"))
     shutil.rmtree(testdir)
     remove(sc_mtx)
     remove(atlas)

nigsp/tests/test_io.py

@@ -48,7 +48,7 @@ def test_check_ext():
 
         assert has_ext is False
         assert fname_out == fname
-        assert ext_out == ""
+        assert ext_out == None
 
 
 @mark.parametrize("data", [(rand(3, 4)), (rand(3, 4, 1)), (rand(3, 1, 4))])
@@ -135,7 +135,7 @@ def test_export_nifti(atlas):
         (".1D", ".1D"),
         (".csv", ".csv"),
         (".tsv", ".tsv"),
-        ("", ".csv"),
+        ("", ".tsv.gz"),
         (".mat", ".mat"),
     ],
 )
@@ -147,7 +147,7 @@ def test_export_mtx(ext_in, ext_out):
 
     if ext_out in [".csv"]:
         data_in = genfromtxt(f"serenity{ext_out}", delimiter=",")
-    if ext_out in [".tsv", ".1D"]:
+    if ext_out in [".tsv", ".tsv.gz", ".1D"]:
         data_in = genfromtxt(f"serenity{ext_out}")
 
     if ext_out in [".mat"]:

nigsp/workflow.py

@@ -309,7 +309,7 @@ def nigsp(
             atlas_is["1D"] or atlas_is["mat"] or atlas_is["xls"] or atlas_is["nifti"]
         ) is False:
             raise NotImplementedError(
-                f"Input file {atlasname} is not of a " "supported type."
+                f"Input file {atlasname} is not of a supported type."
             )
         elif atlas_is["1D"]:
             atlas = io.load_txt(atlasname)