Skip to content

Commit 8ff6230

Browse files
IvanHCenalmorIvanHCenalmor
committed
Avoid forlder removing and allow smart stack processing
1 parent e679630 commit 8ff6230

2 files changed

Lines changed: 88 additions & 73 deletions

File tree

‎Colab_notebooks/Latest_Notebook_versions.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ Noise2Void (2D),1.16.2
2121
Noise2Void (3D),1.16.2
2222
pix2pix,1.18.1
2323
SplineDist (2D),1.14.1
24-
StarDist 2D,1.20.3
24+
StarDist 2D,1.20.4
2525
StarDist 3D,1.15.3
2626
YOLOv2,1.13
2727
Quality_control,1.13

‎Colab_notebooks/StarDist_2D_ZeroCostDL4Mic.ipynb‎

Lines changed: 87 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,8 @@
156156
"cell_type": "code",
157157
"execution_count": null,
158158
"metadata": {
159-
"id": "fq21zJVFNASx",
160-
"cellView": "form"
159+
"cellView": "form",
160+
"id": "fq21zJVFNASx"
161161
},
162162
"outputs": [],
163163
"source": [
@@ -238,14 +238,14 @@
238238
"cell_type": "code",
239239
"execution_count": null,
240240
"metadata": {
241-
"id": "HMAdm-Mc9HFz",
242-
"cellView": "form"
241+
"cellView": "form",
242+
"id": "HMAdm-Mc9HFz"
243243
},
244244
"outputs": [],
245245
"source": [
246246
"from __future__ import print_function, unicode_literals, absolute_import, division\n",
247247
"#@markdown ##Load key dependencies\n",
248-
"Notebook_version = '1.20.3'\n",
248+
"Notebook_version = '1.20.4'\n",
249249
"Network = 'StarDist 2D'\n",
250250
"\n",
251251
"from builtins import any as b_any\n",
@@ -909,8 +909,8 @@
909909
"cell_type": "code",
910910
"execution_count": null,
911911
"metadata": {
912-
"id": "zCvebubeSaGY",
913-
"cellView": "form"
912+
"cellView": "form",
913+
"id": "zCvebubeSaGY"
914914
},
915915
"outputs": [],
916916
"source": [
@@ -941,8 +941,8 @@
941941
"cell_type": "code",
942942
"execution_count": null,
943943
"metadata": {
944-
"id": "01Djr8v-5pPk",
945-
"cellView": "form"
944+
"cellView": "form",
945+
"id": "01Djr8v-5pPk"
946946
},
947947
"outputs": [],
948948
"source": [
@@ -989,8 +989,8 @@
989989
"cell_type": "code",
990990
"execution_count": null,
991991
"metadata": {
992-
"id": "P1f_hZ_ApuNZ",
993-
"cellView": "form"
992+
"cellView": "form",
993+
"id": "P1f_hZ_ApuNZ"
994994
},
995995
"outputs": [],
996996
"source": [
@@ -1070,8 +1070,8 @@
10701070
"cell_type": "code",
10711071
"execution_count": null,
10721072
"metadata": {
1073-
"id": "ewpNJ_I0Mv47",
1074-
"cellView": "form"
1073+
"cellView": "form",
1074+
"id": "ewpNJ_I0Mv47"
10751075
},
10761076
"outputs": [],
10771077
"source": [
@@ -1219,8 +1219,8 @@
12191219
"cell_type": "code",
12201220
"execution_count": null,
12211221
"metadata": {
1222-
"id": "DMqWq5-AxnFU",
1223-
"cellView": "form"
1222+
"cellView": "form",
1223+
"id": "DMqWq5-AxnFU"
12241224
},
12251225
"outputs": [],
12261226
"source": [
@@ -1296,8 +1296,8 @@
12961296
"cell_type": "code",
12971297
"execution_count": null,
12981298
"metadata": {
1299-
"id": "9vC2n-HeLdiJ",
1300-
"cellView": "form"
1299+
"cellView": "form",
1300+
"id": "9vC2n-HeLdiJ"
13011301
},
13021302
"outputs": [],
13031303
"source": [
@@ -1336,9 +1336,7 @@
13361336
" pretrained_model_name = \"2D_Demo\"\n",
13371337
" pretrained_model_path = base_path + \"/\" + pretrained_model_name\n",
13381338
" print(\"Downloading the 2D_Demo_Model_from_Stardist_Github\")\n",
1339-
" if os.path.exists(pretrained_model_path):\n",
1340-
" shutil.rmtree(pretrained_model_path)\n",
1341-
" os.makedirs(pretrained_model_path)\n",
1339+
" os.makedirs(pretrained_model_path, exist_ok=True)\n",
13421340
" wget.download(\"https://github.com/mpicbg-csbd/stardist/raw/master/models/examples/2D_demo/config.json\", pretrained_model_path)\n",
13431341
" wget.download(\"https://github.com/mpicbg-csbd/stardist/raw/master/models/examples/2D_demo/thresholds.json\", pretrained_model_path)\n",
13441342
" wget.download(\"https://github.com/mpicbg-csbd/stardist/blob/master/models/examples/2D_demo/weights_best.h5?raw=true\", pretrained_model_path)\n",
@@ -1352,9 +1350,7 @@
13521350
" pretrained_model_name = \"2D_versatile_fluo\"\n",
13531351
" pretrained_model_path = base_path + \"/\" + pretrained_model_name\n",
13541352
"\n",
1355-
" if os.path.exists(pretrained_model_path):\n",
1356-
" shutil.rmtree(pretrained_model_path)\n",
1357-
" os.makedirs(pretrained_model_path)\n",
1353+
" os.makedirs(pretrained_model_path, exist_ok=True)\n",
13581354
"\n",
13591355
" wget.download(\"https://cloud.mpi-cbg.de/index.php/s/1k5Zcy7PpFWRb0Q/download?path=/versatile&files=2D_versatile_fluo.zip\", pretrained_model_path)\n",
13601356
"\n",
@@ -1370,9 +1366,7 @@
13701366
" pretrained_model_name = \"2D_versatile_he\"\n",
13711367
" pretrained_model_path = base_path + \"/\" + pretrained_model_name\n",
13721368
"\n",
1373-
" if os.path.exists(pretrained_model_path):\n",
1374-
" shutil.rmtree(pretrained_model_path)\n",
1375-
" os.makedirs(pretrained_model_path)\n",
1369+
" os.makedirs(pretrained_model_path, exist_ok=True)\n",
13761370
"\n",
13771371
" wget.download(\"https://cloud.mpi-cbg.de/index.php/s/1k5Zcy7PpFWRb0Q/download?path=/versatile&files=2D_versatile_he.zip\", pretrained_model_path)\n",
13781372
"\n",
@@ -1465,8 +1459,8 @@
14651459
"cell_type": "code",
14661460
"execution_count": null,
14671461
"metadata": {
1468-
"id": "lIUAOJ_LMv5E",
1469-
"cellView": "form"
1462+
"cellView": "form",
1463+
"id": "lIUAOJ_LMv5E"
14701464
},
14711465
"outputs": [],
14721466
"source": [
@@ -1623,9 +1617,9 @@
16231617
"cell_type": "code",
16241618
"execution_count": null,
16251619
"metadata": {
1620+
"cellView": "form",
16261621
"id": "iwNmp1PUzRDQ",
1627-
"scrolled": true,
1628-
"cellView": "form"
1622+
"scrolled": true
16291623
},
16301624
"outputs": [],
16311625
"source": [
@@ -1650,10 +1644,7 @@
16501644
"# convert the history.history dict to a pandas DataFrame:\n",
16511645
"lossData = pd.DataFrame(history.history)\n",
16521646
"\n",
1653-
"if os.path.exists(model_path+\"/\"+model_name+\"/Quality Control\"):\n",
1654-
" shutil.rmtree(model_path+\"/\"+model_name+\"/Quality Control\")\n",
1655-
"\n",
1656-
"os.makedirs(model_path+\"/\"+model_name+\"/Quality Control\")\n",
1647+
"os.makedirs(model_path+\"/\"+model_name+\"/Quality Control\", exist_ok=True)\n",
16571648
"\n",
16581649
"# The training evaluation.csv is saved (overwrites the Files if needed).\n",
16591650
"lossDataCSVpath = model_path+'/'+model_name+'/Quality Control/training_evaluation.csv'\n",
@@ -1703,8 +1694,8 @@
17031694
"cell_type": "code",
17041695
"execution_count": null,
17051696
"metadata": {
1706-
"id": "eAJzMwPA6tlH",
1707-
"cellView": "form"
1697+
"cellView": "form",
1698+
"id": "eAJzMwPA6tlH"
17081699
},
17091700
"outputs": [],
17101701
"source": [
@@ -1762,8 +1753,8 @@
17621753
"cell_type": "code",
17631754
"execution_count": null,
17641755
"metadata": {
1765-
"id": "vMzSP50kMv5p",
1766-
"cellView": "form"
1756+
"cellView": "form",
1757+
"id": "vMzSP50kMv5p"
17671758
},
17681759
"outputs": [],
17691760
"source": [
@@ -1831,8 +1822,8 @@
18311822
"cell_type": "code",
18321823
"execution_count": null,
18331824
"metadata": {
1834-
"id": "w90MdriMxhjD",
1835-
"cellView": "form"
1825+
"cellView": "form",
1826+
"id": "w90MdriMxhjD"
18361827
},
18371828
"outputs": [],
18381829
"source": [
@@ -1850,13 +1841,7 @@
18501841
"Target_QC_folder = check_base_path(base_path, Target_QC_folder)\n",
18511842
"\n",
18521843
"#Create a quality control Folder and check if the folder already exist\n",
1853-
"if os.path.exists(QC_model_path+\"/\"+QC_model_name+\"/Quality Control\") == False:\n",
1854-
" os.makedirs(QC_model_path+\"/\"+QC_model_name+\"/Quality Control\")\n",
1855-
"\n",
1856-
"if os.path.exists(QC_model_path+\"/\"+QC_model_name+\"/Quality Control/Prediction\"):\n",
1857-
" shutil.rmtree(QC_model_path+\"/\"+QC_model_name+\"/Quality Control/Prediction\")\n",
1858-
"\n",
1859-
"os.makedirs(QC_model_path+\"/\"+QC_model_name+\"/Quality Control/Prediction\")\n",
1844+
"os.makedirs(QC_model_path+\"/\"+QC_model_name+\"/Quality Control/Prediction\", exist_ok=True)\n",
18601845
"\n",
18611846
"# Check if the sourcce and target folders contain base_path, if not add it\n",
18621847
"Source_QC_folder = check_base_path(base_path, Source_QC_folder)\n",
@@ -2032,8 +2017,8 @@
20322017
"cell_type": "code",
20332018
"execution_count": null,
20342019
"metadata": {
2035-
"id": "v5F2VWnQ_fL-",
2036-
"cellView": "form"
2020+
"cellView": "form",
2021+
"id": "v5F2VWnQ_fL-"
20372022
},
20382023
"outputs": [],
20392024
"source": [
@@ -2159,8 +2144,8 @@
21592144
"cell_type": "code",
21602145
"execution_count": null,
21612146
"metadata": {
2162-
"id": "y2TD5p7MZrEb",
2163-
"cellView": "form"
2147+
"cellView": "form",
2148+
"id": "y2TD5p7MZrEb"
21642149
},
21652150
"outputs": [],
21662151
"source": [
@@ -2177,9 +2162,7 @@
21772162
"# Check that the base_path is on the path and otherwise add it\n",
21782163
"Results_folder = check_base_path(base_path, Results_folder)\n",
21792164
"\n",
2180-
"if os.path.exists(Results_folder):\n",
2181-
" shutil.rmtree(Results_folder)\n",
2182-
"os.makedirs(Results_folder)\n",
2165+
"os.makedirs(Results_folder, exist_ok=True)\n",
21832166
"\n",
21842167
"#@markdown ###Are your data single images or stacks?\n",
21852168
"\n",
@@ -2311,6 +2294,18 @@
23112294
" short_name = os.path.splitext(image)\n",
23122295
" print(f\"{image} correctly loaded!\")\n",
23132296
"\n",
2297+
" # Check if the image has already been processed\n",
2298+
" output_frame_path = os.path.join(Results_folder, f\"{short_name[0]}\")\n",
2299+
" if os.path.exists(output_frame_path):\n",
2300+
" if len(os.listdir(output_frame_path)) == timelapse.shape[0]:\n",
2301+
" print(bcolors.WARNING + \"The stack \"+image+\" has already been processed. Please remove the folder \"+output_frame_path+\" if you want to reprocess the stack.\"+bcolors.ENDC)\n",
2302+
" continue\n",
2303+
" else:\n",
2304+
" print(bcolors.WARNING + \"The stack \"+image+\" is partially processed. Continuing with the unprocessed frames.\"+bcolors.ENDC)\n",
2305+
" num_processed_frames = len(os.listdir(output_frame_path))\n",
2306+
" else:\n",
2307+
" num_processed_frames = 0\n",
2308+
"\n",
23142309
" short_name = os.path.splitext(image)\n",
23152310
"\n",
23162311
" mi = np.percentile(timelapse,1,axis=(0,)+tuple(1+np.array(axis_norm)),keepdims=True)[0]\n",
@@ -2330,7 +2325,7 @@
23302325
" n_timepoint = timelapse.shape[0]\n",
23312326
" # prediction_stack = np.zeros((n_timepoint, timelapse.shape[1], timelapse.shape[2]))\n",
23322327
"\n",
2333-
" for t in range(n_timepoint):\n",
2328+
" for t in range(num_processed_frames, n_timepoint):\n",
23342329
" print(f\"Analyzing timepoint {t}/{n_timepoint}\")\n",
23352330
" img_t = timelapse[t]\n",
23362331
" # Normalize the image with the percentiles from the whole stack\n",
@@ -2409,8 +2404,8 @@
24092404
"cell_type": "code",
24102405
"execution_count": null,
24112406
"metadata": {
2412-
"id": "jxjHeOFFleSV",
2413-
"cellView": "form"
2407+
"cellView": "form",
2408+
"id": "jxjHeOFFleSV"
24142409
},
24152410
"outputs": [],
24162411
"source": [
@@ -2426,9 +2421,7 @@
24262421
"# Check that the base_path is on the path and otherwise add it\n",
24272422
"Results_folder = check_base_path(base_path, Results_folder)\n",
24282423
"\n",
2429-
"if os.path.exists(Results_folder):\n",
2430-
" shutil.rmtree(Results_folder)\n",
2431-
"os.makedirs(Results_folder)\n",
2424+
"os.makedirs(Results_folder, exist_ok=True)\n",
24322425
"\n",
24332426
"# model name and path\n",
24342427
"#@markdown ###Do you want to use the current trained model?\n",
@@ -2534,18 +2527,14 @@
25342527
"#Create a temp folder to save Zarr files\n",
25352528
"\n",
25362529
"Temp_folder = base_path + \"/Temp_folder\"\n",
2537-
"\n",
2538-
"if os.path.exists(Temp_folder):\n",
2539-
" shutil.rmtree(Temp_folder)\n",
2540-
"os.makedirs(Temp_folder)\n",
2530+
"os.makedirs(Temp_folder, exist_ok=True)\n",
25412531
"\n",
25422532
"\n",
25432533
"# mi, ma = np.percentile(img[::8], [1,99.8]) # compute percentiles from low-resolution image\n",
25442534
"# mi, ma = np.percentile(img[13000:16000,13000:16000], [1,99.8]) # compute percentiles from smaller crop\n",
25452535
"mi, ma = 0, 255 # use min and max dtype values (suitable here)\n",
25462536
"normalizer = MyNormalizer(mi, ma)\n",
25472537
"\n",
2548-
"\n",
25492538
"np.random.seed(16)\n",
25502539
"lbl_cmap = random_label_cmap()\n",
25512540
"\n",
@@ -2561,12 +2550,31 @@
25612550
"\n",
25622551
" print(\"Image dimension \"+str(X.shape))\n",
25632552
"\n",
2564-
" if is_stack:\n",
2553+
" if Is_stack:\n",
2554+
" stack_result_path = os.path.join(Results_folder, short_name[0])\n",
2555+
" polygon_stack_result_path = os.path.join(Results_folder, f\"{short_name[0]}_labels_roi\")\n",
2556+
"\n",
2557+
" # Check if the stack has been processed or is half processed\n",
2558+
" if os.path.exists(stack_result_path):\n",
2559+
" if len(os.listdir(stack_result_path)) == X.shape[0]:\n",
2560+
" print(bcolors.WARNING + \"The stack \"+image+\" has already been processed. Please remove the folder \"+stack_result_path+\" if you want to reprocess the stack.\"+bcolors.ENDC)\n",
2561+
" continue\n",
2562+
" else:\n",
2563+
" print(bcolors.WARNING + \"The stack \"+image+\" is partially processed. Continuing with the unprocessed frames.\"+bcolors.ENDC)\n",
2564+
" num_processed_frames = len(os.listdir(stack_result_path))\n",
2565+
" elif os.path.exists(polygon_stack_result_path):\n",
2566+
" if len(os.listdir(polygon_stack_result_path)) == X.shape[0]:\n",
2567+
" print(bcolors.WARNING + \"The stack \"+image+\" has already been processed. Please remove the folder \"+polygon_stack_result_path+\" if you want to reprocess the stack.\"+bcolors.ENDC)\n",
2568+
" continue\n",
2569+
" else:\n",
2570+
" print(bcolors.WARNING + \"The stack \"+image+\" is partially processed. Continuing with the unprocessed frames.\"+bcolors.ENDC)\n",
2571+
" num_processed_frames = len(os.listdir(polygon_stack_result_path))\n",
2572+
" else:\n",
2573+
" num_processed_frames = 0\n",
2574+
"\n",
25652575
" if Mask_images:\n",
2566-
" stack_result_path = os.path.join(Results_folder, short_name[0])\n",
25672576
" os.makedirs(stack_result_path, exist_ok=True)\n",
25682577
" if Region_of_interests:\n",
2569-
" polygon_stack_result_path = os.path.join(Results_folder, f\"{short_name[0]}_labels_roi\")\n",
25702578
" os.makedirs(polygon_stack_result_path, exist_ok=True)\n",
25712579
"\n",
25722580
" num_frames = X.shape[0]\n",
@@ -2641,6 +2649,13 @@
26412649
"# **7. Version log**\n",
26422650
"---\n",
26432651
"\n",
2652+
"<font size = 4>**v1.20.4**: \n",
2653+
"\n",
2654+
"* Modify the section `6. Using the trained model notebook`:\n",
2655+
" * On the stack prediction of section 6.1., to avoid out of memory error, the normalization of tha data is not done on the whole stack. Instead the minimum and maximum percentiles are taken from the full stack and then each frame is individually normalised.\n",
2656+
" * Also to avoid out of memory errors on section 6.1. each frame is individually saved in a folder. This is not efficient in terms of storage, but release the RAM memory required to store the predicted stack.\n",
2657+
" * On section 6.2., I included the 'Is_stack' option and included previous features.\n",
2658+
"\n",
26442659
"<font size = 4>**v1.20.3**: \n",
26452660
"\n",
26462661
"* Change the TensorFlow and Keras versions to Keras < 3. This has fixed the BioImage.IO model exportation.\n",
@@ -2703,9 +2718,10 @@
27032718
}
27042719
],
27052720
"metadata": {
2721+
"accelerator": "GPU",
27062722
"colab": {
2707-
"provenance": [],
2708-
"gpuType": "T4"
2723+
"gpuType": "T4",
2724+
"provenance": []
27092725
},
27102726
"kernelspec": {
27112727
"display_name": "Python 3",
@@ -2722,8 +2738,7 @@
27222738
"nbconvert_exporter": "python",
27232739
"pygments_lexer": "ipython3",
27242740
"version": "3.6.12"
2725-
},
2726-
"accelerator": "GPU"
2741+
}
27272742
},
27282743
"nbformat": 4,
27292744
"nbformat_minor": 0

0 commit comments

Comments
 (0)