Cleaner code and improved documentation

Author: OmFrederic

README.md

@@ -9,10 +9,11 @@ Key features
 
 - **Templates with different levels of features and complexity** 
   - Several templates ready to use: base, advanced and retro-compatible.
-  - Multiple configuration possibilities: basic constants, command line arguments or YAML configuration files.
+  - Multiple configuration possibilities: basic constants, command line arguments or YAML configuration files (all optional).
   - Simple logging mechanism.
   - File iteration embedded functionality.
-  - Template pipeline coordination (advanced).
+  - Multi-CPU parallelisation with progress bar (advanced).
+  - Thread-safe compiled CSV result with automatic header (advanced).
 - **Examples based on these templates with real functionality**
   - Readily available code to perform several requested tasks by the lab members.
   - Simple installation and usage.

examples/cell_cropper/README.md

@@ -32,7 +32,7 @@ The code expects you to perform certain steps to run with your data:
 
 To run HPA cell cropper you have to gather the information about the sets of images you want to process. HPA cell cropper reads `path_list.csv` to locate each set of images, in the following .csv format: 
 
-`r_image,y_image,b_image,g_image,segmentation_mask,crop_folder,output_prefix`
+`r_image,y_image,b_image,g_image,cell_mask,nuclei_mask,crop_folder,output_prefix`
 
 - `r_image`: the microtubules targeting marker FOV image. 
 - `y_image`: the ER targeting marker FOV image.
@@ -71,6 +71,8 @@ Output
 ------ 
 
 Cell cropper generates in the chosen crop_folder the following files:
-- `[output_prefix]_cell[X]_crop_[red|yellow|blue|green].png`: a cropped cell from the FOV.
-- `[output_prefix]_cell[X]_crop_masked_[red|yellow|blue|green].png`: a cropped and masked cell from the FOV (if the mask_cell option was selected).
-- Additionally, a `crop_info.csv` file will be created containing all generated cell crops bboxes for convenience.
+- `[output_prefix]cell[X]_crop_[red|yellow|blue|green].png`: a cropped cell from the FOV.
+- `[output_prefix]cell[X]_crop_masked_[red|yellow|blue|green].png`: a cropped and masked cell from the FOV (if the `mask_cell` option was selected).
+- `[output_prefix]cell[X]_cellmask.png`: the binary cell mask for that crop (if the `crop_mask` option was selected).
+- `[output_prefix]cell[X]_nucleimask.png`: the binary nuclei mask for that crop (if the `crop_mask` option was selected and a nuclei mask was provided).
+- Additionally, a `crop_info.csv` file will be created in the base folder containing all generated cell crops bounding boxes for convenience.

examples/cell_cropper/cell_cropper.py

@@ -1,10 +1,10 @@
 import warnings
 import numpy as np
-from imageio.v2 import imsave
 from skimage.measure import regionprops
 from scipy.ndimage.morphology import grey_dilation
 import pandas as pd
 import image_utils
+import cv2
 
 
 warnings.simplefilter(action="ignore", category=FutureWarning)
@@ -83,28 +83,28 @@ def generate_crops(image_stack, cell_mask, nuclei_mask, crop_size, crop_bitdepth
             this_cell_mask[this_cell_mask == region.label] = 1
             this_cell_mask = grey_dilation(this_cell_mask, size=7)
             curr_cell_mask, _ = safe_crop(this_cell_mask, fixed_bbox)
-            imsave(f"{output_folder}/{output_prefix}cell{region.label}_cellmask.png", np.uint8(curr_cell_mask * 255))
+            cv2.imwrite(f"{output_folder}/{output_prefix}cell{region.label}_cellmask.png", np.uint8(curr_cell_mask * 255))
             if nuclei_mask is not None:
                 this_nuclei_mask = nuclei_mask.copy()
                 this_nuclei_mask[this_nuclei_mask != region.label] = 0
                 this_nuclei_mask[this_nuclei_mask == region.label] = 1
                 this_nuclei_mask = grey_dilation(this_nuclei_mask, size=7)
                 curr_nuclei_mask, _ = safe_crop(this_nuclei_mask, fixed_bbox)
-                imsave(f"{output_folder}/{output_prefix}cell{region.label}_nucleimask.png", np.uint8(curr_nuclei_mask * 255))
+                cv2.imwrite(f"{output_folder}/{output_prefix}cell{region.label}_nucleimask.png", np.uint8(curr_nuclei_mask * 255))
 
         for curr_img_index in range(len(image_stack)):
             if curr_img_index != 0:
                 image_cp = image_stack[curr_img_index][0].copy()
 
             cell_crop, _ = safe_crop(image_cp, fixed_bbox)
-            imsave(f"{output_folder}/{output_prefix}cell{region.label}_crop_" + colors[curr_img_index] + ".png", image_utils.convert_bitdepth(cell_crop, crop_bitdepth))
+            cv2.imwrite(f"{output_folder}/{output_prefix}cell{region.label}_crop_" + colors[curr_img_index] + ".png", image_utils.convert_bitdepth(cell_crop, crop_bitdepth))
 
             if mask_cell:
                 this_cell_mask = cell_mask == region.label
                 this_cell_mask = grey_dilation(this_cell_mask, size=7)
                 image_cp[this_cell_mask == 0] = 0
                 cell_mask_crop, _ = safe_crop(image_cp, fixed_bbox)
-                imsave(f"{output_folder}/{output_prefix}cell{region.label}_crop_masked_" + colors[curr_img_index] + ".png", image_utils.convert_bitdepth(cell_mask_crop, crop_bitdepth))
+                cv2.imwrite(f"{output_folder}/{output_prefix}cell{region.label}_crop_masked_" + colors[curr_img_index] + ".png", image_utils.convert_bitdepth(cell_mask_crop, crop_bitdepth))
 
         new_center = (crop_size // 2, crop_size // 2)
         new_bbox = (

examples/cell_cropper/process.py

@@ -17,10 +17,10 @@
 config = { "log": logger}
 
 # If you want to use constants with your script, add them here
-config["crop_size"] = 1024
-config["crop_bitdepth"] = 8
+config["crop_size"] = 640
+config["crop_bitdepth"] = 16
 config["crop_mask"] = True
-config["mask_cell"] = False
+config["mask_cell"] = True
 
 # Log the start time and the final configuration so you can keep track of what you did
 config["log"].info('Start: ' + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
@@ -31,7 +31,8 @@
 
 # If we provide a "path_list.csv" file, we run our code for each pair of input/output sub-folders
 if os.path.exists("./path_list.csv"):
-    path_list = open("./path_list.csv", 'r')
+    with open("./path_list.csv", 'r') as path_list:
+        path_list = path_list.readlines()
 
     df = pd.DataFrame(columns=['id', 'cell', 'x1', 'y1', 'x2', 'y2'])
 

examples/cellpose_segmentation/README.md

@@ -43,7 +43,7 @@ All images can be relative or absolute paths, or directly URLs. You can also ski
 Check the following `path_list.csv` content as an example:
 
 ```
-#nuclei_image,cyto_image,output_folder,output_prefix
+#nuclei_image,cyto_image1,cyto_image2,output_folder,output_prefix
 input/nuc1.tif,input/mt1.tif,,output,example1_
 #input/nuc2.tif,input/mt2.tif,,output,example2_
 input/nuc3.tif,input/mt3.tif,,output,example3_

examples/cellpose_segmentation/process.py

@@ -34,7 +34,8 @@
 
 # If we provide a "path_list.csv" file, we run our code for each pair of input/output sub-folders
 if os.path.exists("./path_list.csv"):
-    path_list = open("./path_list.csv", 'r')
+    with open("./path_list.csv", 'r') as f:
+        path_list = f.readlines()
 
     for curr_set in path_list:
         if curr_set.strip() != "" and not curr_set.startswith("#"):

examples/micronuclei_segmentation/README.md

@@ -31,6 +31,7 @@ The code expects you to perform certain steps to run with your data:
   - Locate the line `config["eccentricity_tolerance"] = 0.9` and change the value of your maximum allowed eccentricity shape of the micronuclei (0 - circle, 1 - flattened ellipse). 
   - Locate the line `config["solidity_tolerance"] = 0.1` and change the value of your minimum allowed solidity shape of the micronuclei (0 - totally fragmented, 1 - totally convex hull shape). 
   - Locate the line `config["intensity_ratio_tolerance"] = 0.2` and change the value of your maximum allowed variance between the micronuclei average intensity and it's related nuclei average intensity (0 - no variance, 1 totally different)
+  - Locate the line `config["overlapping_acceptance_ratio"] = 0.2` and change the value of the maximum allowed overlap ratio between a micronuclei candidate and its parent nucleus (0 - no overlap accepted, 1 - full overlap accepted).
 
 To run micronuclei segmentation you have to gather the information about the sets of images you want to process. Micronuclei segmentation reads `path_list.csv` to locate each set of images, in the following .csv format: 
 

examples/micronuclei_segmentation/process.py

@@ -34,7 +34,8 @@
 
 # If we provide a "path_list.csv" file, we run our code for each pair of input/output sub-folders
 if os.path.exists("./path_list.csv"):
-    path_list = open("./path_list.csv", 'r')
+    with open("./path_list.csv", 'r') as f:
+        path_list = f.readlines()
 
     df = pd.DataFrame(columns=["id", "micnuc_label", "nuclei_label", "micnuc_area", "micnuc_centroid_xy", "micnuc_eccentricity", "micnuc_solidity", "micnuc_intensity", "nuc_area", "nuc_centroid_xy", "nuc_eccentricity", "nuc_solidity", "nuc_intensity"])
 

examples/otsu_explorer/README.md

@@ -25,8 +25,8 @@ Setup
 The code expects you to perform certain steps to run with your data:
 
 - Edit and modify `process.py` file:
-  - Locate the line `config["input_path"] = "./input"` and change the path value with your desired input directory. This directory should contain all your images OR all the subfolders present in `path_list.csv` (see below). 
-  - Locate the line `config["output_path"] = "./output"` and change the path value with your desired output directory. This directory will contain the resulting generated images OR the resulting subfolders stated `path_list.csv` (see below). OBS: it will be created if it does not exists
+  - Locate the line `config["input_directory"] = "./input"` and change the path value with your desired input directory. This directory should contain all your images OR all the subfolders present in `path_list.csv` (see below). 
+  - Locate the line `config["output_directory"] = "./output"` and change the path value with your desired output directory. This directory will contain the resulting generated images OR the resulting subfolders stated `path_list.csv` (see below). OBS: it will be created if it does not exists
 - You can copy all your images in the stated input folder OR use a detailed `path_list.csv` listing:
   - If you want to copy all your images in the input folder, just delete the `path_list.csv` file. OBS: don't choose an output folder inside the stated input folder if you are going this way! 
   - If you want to use a detailed `path_list.csv` listing:
@@ -47,3 +47,13 @@ Running the code
 
 Once you have activated your virtual environment and modified all the desired parameters (see section **Setup**, just run the code with `python process.py`. OBS: you have to run the `process.py` file, which will on its turn call the rest of the code.
 
+
+
+Output
+------
+
+Otsu explorer generates in the chosen output_directory the following files, for each input image:
+- `[image_name]_otsu_3.png`: the image thresholded using multi-Otsu with 3 bins.
+- `[image_name]_otsu_4.png`: the image thresholded using multi-Otsu with 4 bins.
+- `[image_name]_otsu_5.png`: the image thresholded using multi-Otsu with 5 bins.
+

examples/otsu_explorer/process.py

@@ -15,8 +15,8 @@
 config = { "log": logger}
 
 # If you want to use constants with your script, add them here
-config["input_path"] = "./input"
-config["output_path"] = "./output"
+config["input_directory"] = "./input"
+config["output_directory"] = "./output"
 
 # Log the start time and the final configuration so you can keep track of what you did
 config["log"].info('Start: ' + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
@@ -25,21 +25,20 @@
 config["log"].info('----------')
 
 
-# We create the output folder
-os.makedirs(config["output_path"], exist_ok=True)
 # If we provide a "path_list.csv" file, we run our code for each pair of input/output sub-folders
 if os.path.exists("./path_list.csv"):
-    path_list = open("./path_list.csv", 'r')
-    for curr_path in path_list:
-        if curr_path.strip() != "" and not curr_path.startswith("#"):
-            curr_input_path = config["input_path"] + "/" + curr_path.strip().split(",")[0]
-            curr_output_path = config["output_path"] + "/" + curr_path.strip().split(",")[1]
-            os.makedirs(curr_output_path, exist_ok=True)
-
-            otsu_explorer.otsu_explorer(config, curr_input_path, curr_output_path)
+    with open("./path_list.csv", 'r') as path_list:
+        for curr_path in path_list:
+            if curr_path.strip() != "" and not curr_path.startswith("#"):
+                curr_input_path = os.path.join(config["input_directory"], curr_path.strip().split(",")[0])
+                curr_output_path = os.path.join(config["output_directory"], curr_path.strip().split(",")[1])
+                os.makedirs(curr_output_path, exist_ok=True)
+
+                otsu_explorer.otsu_explorer(config, curr_input_path, curr_output_path)
 # If we DON'T provide a "path_list.csv" file, we run our code once directly over the base input/output folder
 else:
-    otsu_explorer.otsu_explorer(config, config["input_path"], config["output_path"])
+    os.makedirs(config["output_directory"], exist_ok=True)
+    otsu_explorer.otsu_explorer(config, config["input_directory"], config["output_directory"])
 
 config["log"].info('----------')
-config["log"].info('End: ' + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
+config["log"].info('End: ' + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"))