Skip to content

Commit 6d41021

Browse files
author
Sofiene Boutaj
committed
Code cleaning
1 parent 02414b2 commit 6d41021

49 files changed

Lines changed: 218 additions & 363 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/thunder/__init__.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
from .datasets import download_datasets
2-
from .datasets import generate_splits
3-
from .models import download_models
41
from .benchmark import benchmark
2+
from .datasets import download_datasets, generate_splits
3+
from .models import download_models
54
from .utils import gather_results

‎src/thunder/benchmark.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1+
import logging
12
import os
3+
import shutil
4+
from typing import Callable
25

36
import hydra
4-
import logging
57
from omegaconf import DictConfig
6-
import shutil
7-
from typing import Callable
88

99
from .datasets.utils import is_dataset_available
1010
from .models.utils import is_model_available, load_custom_model_from_file
1111
from .utils.utils import print_task_hyperparams
1212

13+
1314
def benchmark(
1415
model: str | Callable,
1516
dataset: str,
@@ -46,7 +47,7 @@ def benchmark(
4647
from omegaconf import OmegaConf
4748

4849
from .utils.config import get_config
49-
50+
5051
wandb_mode = "online" if online_wandb else "offline"
5152
adaptation_type = "lora" if lora else "frozen"
5253
ckpt_saving = "save_ckpts_all_epochs" if ckpt_save_all else "save_best_ckpt_only"
@@ -110,7 +111,8 @@ def run_benchmark(cfg: DictConfig, model_cls: Callable = None) -> None:
110111
from .tasks.alignment_scoring import alignment_scoring
111112
from .tasks.image_retrieval import image_retrieval
112113
from .tasks.knn_classification import knn
113-
from .tasks.pre_computing_patch_embeddings import pre_computing_patch_embeddings
114+
from .tasks.pre_computing_patch_embeddings import \
115+
pre_computing_patch_embeddings
114116
from .tasks.simple_shot import simple_shot
115117
from .tasks.train_eval_probe import eval_probe, train_probe
116118
from .tasks.transformation_invariance import transformation_invariance

‎src/thunder/datasets/__init__.py‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,6 @@
1-
from .download import download_datasets
21
from .data_splits import generate_splits
3-
4-
from .dataset import bach
5-
from .dataset import bracs
6-
from .dataset import break_his
7-
from .dataset import ccrcc
8-
from .dataset import crc
9-
from .dataset import esca
10-
from .dataset import mhist
11-
from .dataset import ocelot
12-
from .dataset import pannuke
13-
from .dataset import patch_camelyon
14-
from .dataset import segpath_epithelial
15-
from .dataset import segpath_lymphocytes
16-
from .dataset import tcga_crc_msi
17-
from .dataset import tcga_tils
18-
from .dataset import tcga_uniform
19-
from .dataset import wilds
2+
from .dataset import (bach, bracs, break_his, ccrcc, crc, esca, mhist, ocelot,
3+
pannuke, patch_camelyon, segpath_epithelial,
4+
segpath_lymphocytes, tcga_crc_msi, tcga_tils,
5+
tcga_uniform, wilds)
6+
from .download import download_datasets

‎src/thunder/datasets/data_splits.py‎

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
from collections import defaultdict
2-
from collections.abc import Callable
31
import logging
42
import os
53
import random
4+
from collections import defaultdict
5+
from collections.abc import Callable
66
from pathlib import Path
77
from typing import List, Union
88

@@ -79,25 +79,17 @@ def generate_splits_for_dataset(dataset_name: str) -> None:
7979
dataset_name (str): The name of the dataset to generate splits for.
8080
"""
8181
from omegaconf import OmegaConf
82+
8283
from ..utils.constants import DatasetConstants
83-
from .dataset import (
84-
create_splits_bach,
85-
create_splits_bracs,
86-
create_splits_break_his,
87-
create_splits_ccrcc,
88-
create_splits_crc,
89-
create_splits_esca,
90-
create_splits_mhist,
91-
create_splits_patch_camelyon,
92-
create_splits_tcga_crc_msi,
93-
create_splits_tcga_tils,
94-
create_splits_tcga_uniform,
95-
create_splits_wilds,
96-
create_splits_ocelot,
97-
create_splits_pannuke,
98-
create_splits_segpath_epithelial,
99-
create_splits_segpath_lymphocytes,
100-
)
84+
from .dataset import (create_splits_bach, create_splits_bracs,
85+
create_splits_break_his, create_splits_ccrcc,
86+
create_splits_crc, create_splits_esca,
87+
create_splits_mhist, create_splits_ocelot,
88+
create_splits_pannuke, create_splits_patch_camelyon,
89+
create_splits_segpath_epithelial,
90+
create_splits_segpath_lymphocytes,
91+
create_splits_tcga_crc_msi, create_splits_tcga_tils,
92+
create_splits_tcga_uniform, create_splits_wilds)
10193

10294
DATASET_TO_FUNCTION = {
10395
# Classification
@@ -237,8 +229,8 @@ def check_dataset_md5(data_splits: dict, dataset_cfg: dict) -> None:
237229
:param data_splits: data splits dictionary.
238230
:param dataset_cfg: dataset-specific config.
239231
"""
240-
import json
241232
import hashlib
233+
import json
242234

243235
if hasattr(dataset_cfg, "md5sum"):
244236
expected_md5 = dataset_cfg["md5sum"]

‎src/thunder/datasets/dataset/__init__.py‎

Lines changed: 19 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,21 @@
11
# import downloaders
2-
from .bach import download_bach
3-
from .bracs import download_bracs
4-
from .break_his import download_break_his
5-
from .ccrcc import download_ccrcc
6-
from .crc import download_crc
7-
from .esca import download_esca
8-
from .mhist import download_mhist
9-
from .patch_camelyon import download_patch_camelyon
10-
from .tcga_crc_msi import download_tcga_crc_msi
11-
from .tcga_tils import download_tcga_tils
12-
from .tcga_uniform import download_tcga_uniform
13-
from .wilds import download_wilds
14-
from .ocelot import download_ocelot
15-
from .pannuke import download_pannuke
16-
from .segpath_epithelial import download_segpath_epithelial
17-
from .segpath_lymphocytes import download_segpath_lymphocytes
18-
192
# import split generators
20-
from .bach import create_splits_bach
21-
from .bracs import create_splits_bracs
22-
from .break_his import create_splits_break_his
23-
from .ccrcc import create_splits_ccrcc
24-
from .crc import create_splits_crc
25-
from .esca import create_splits_esca
26-
from .mhist import create_splits_mhist
27-
from .patch_camelyon import create_splits_patch_camelyon
28-
from .tcga_crc_msi import create_splits_tcga_crc_msi
29-
from .tcga_tils import create_splits_tcga_tils
30-
from .tcga_uniform import create_splits_tcga_uniform
31-
from .wilds import create_splits_wilds
32-
from .ocelot import create_splits_ocelot
33-
from .pannuke import create_splits_pannuke
34-
from .segpath_epithelial import create_splits_segpath_epithelial
35-
from .segpath_lymphocytes import create_splits_segpath_lymphocytes
3+
from .bach import create_splits_bach, download_bach
4+
from .bracs import create_splits_bracs, download_bracs
5+
from .break_his import create_splits_break_his, download_break_his
6+
from .ccrcc import create_splits_ccrcc, download_ccrcc
7+
from .crc import create_splits_crc, download_crc
8+
from .esca import create_splits_esca, download_esca
9+
from .mhist import create_splits_mhist, download_mhist
10+
from .ocelot import create_splits_ocelot, download_ocelot
11+
from .pannuke import create_splits_pannuke, download_pannuke
12+
from .patch_camelyon import (create_splits_patch_camelyon,
13+
download_patch_camelyon)
14+
from .segpath_epithelial import (create_splits_segpath_epithelial,
15+
download_segpath_epithelial)
16+
from .segpath_lymphocytes import (create_splits_segpath_lymphocytes,
17+
download_segpath_lymphocytes)
18+
from .tcga_crc_msi import create_splits_tcga_crc_msi, download_tcga_crc_msi
19+
from .tcga_tils import create_splits_tcga_tils, download_tcga_tils
20+
from .tcga_uniform import create_splits_tcga_uniform, download_tcga_uniform
21+
from .wilds import create_splits_wilds, download_wilds

‎src/thunder/datasets/dataset/bach.py‎

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
11
def download_bach(root_folder: str):
2-
from ..utils import (
3-
download_from_url,
4-
unzip_file,
5-
untar_file,
6-
ungzip_file,
7-
remove_symlinks,
8-
)
2+
from ..utils import (download_from_url, remove_symlinks, ungzip_file,
3+
untar_file, unzip_file)
94

105
urls = [
116
"https://zenodo.org/api/records/3632035/files/ICIAR2018_BACH_Challenge.zip/content",
@@ -24,21 +19,17 @@ def create_splits_bach(base_folder: str, dataset_cfg: dict) -> None:
2419
:param base_folder: path to the main folder storing datasets.
2520
:param dataset_cfg: dataset-specific config.
2621
"""
27-
from collections import defaultdict
2822
import math
2923
import os
30-
import pandas as pd
24+
from collections import defaultdict
3125
from pathlib import Path
3226

33-
from ..data_splits import (
34-
check_dataset,
35-
create_few_shot_training_data,
36-
init_dict,
37-
save_dict,
38-
sorted_listdir,
39-
)
27+
import pandas as pd
28+
4029
from ...utils.constants import UtilsConstants
4130
from ...utils.utils import set_seed
31+
from ..data_splits import (check_dataset, create_few_shot_training_data,
32+
init_dict, save_dict, sorted_listdir)
4233

4334
# Patient details are derived from ICIAR2018_BACH_dataset_micro_patient.xlsx
4435
# downloaded from https://www.dropbox.com/scl/fo/efjnzcjydcjc4ebz1py7n/AIQyydtZTQcAFlLRxsEQhCU?dl=0&e=5&preview=ICIAR2018_BACH_dataset_micro_patient.xlsx&rlkey=w5ebtg9dlwrk57663chvodzy6&st=k0dfgjck

‎src/thunder/datasets/dataset/bracs.py‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,10 @@ def create_splits_bracs(base_folder: str, dataset_cfg: dict) -> None:
2727
"""
2828
import os
2929

30-
from ..data_splits import (
31-
check_dataset,
32-
create_few_shot_training_data,
33-
get_data_from_set,
34-
init_dict,
35-
save_dict,
36-
)
3730
from ...utils.constants import UtilsConstants
3831
from ...utils.utils import set_seed
32+
from ..data_splits import (check_dataset, create_few_shot_training_data,
33+
get_data_from_set, init_dict, save_dict)
3934

4035
# Setting the random seed
4136
set_seed(UtilsConstants.DEFAULT_SEED.value)

‎src/thunder/datasets/dataset/break_his.py‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
def download_break_his(root_folder: str):
2-
from ..utils import (
3-
download_from_url,
4-
untar_file,
5-
)
2+
from ..utils import download_from_url, untar_file
63

74
download_from_url(
85
"http://www.inf.ufpr.br/vri/databases/BreaKHis_v1.tar.gz",
@@ -20,15 +17,10 @@ def create_splits_break_his(base_folder: str, dataset_cfg: dict) -> None:
2017
"""
2118
import os
2219

23-
from ..data_splits import (
24-
check_dataset,
25-
create_few_shot_training_data,
26-
init_dict,
27-
save_dict,
28-
sorted_listdir,
29-
)
3020
from ...utils.constants import UtilsConstants
3121
from ...utils.utils import set_seed
22+
from ..data_splits import (check_dataset, create_few_shot_training_data,
23+
init_dict, save_dict, sorted_listdir)
3224

3325
# Setting the random seed
3426
set_seed(UtilsConstants.DEFAULT_SEED.value)

‎src/thunder/datasets/dataset/ccrcc.py‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
def download_ccrcc(root_folder: str):
2-
from ..utils import (
3-
download_from_url,
4-
unzip_file,
5-
)
2+
from ..utils import download_from_url, unzip_file
63

74
download_from_url(
85
"https://zenodo.org/api/records/7898308/files/tissue_classification.zip/content",
@@ -18,19 +15,14 @@ def create_splits_ccrcc(base_folder: str, dataset_cfg: dict) -> None:
1815
:param base_folder: path to the main folder storing datasets.
1916
:param dataset_cfg: dataset-specific config.
2017
"""
21-
from collections import defaultdict
2218
import os
2319
import random
20+
from collections import defaultdict
2421

25-
from ..data_splits import (
26-
check_dataset,
27-
create_few_shot_training_data,
28-
init_dict,
29-
save_dict,
30-
sorted_listdir,
31-
)
3222
from ...utils.constants import UtilsConstants
3323
from ...utils.utils import set_seed
24+
from ..data_splits import (check_dataset, create_few_shot_training_data,
25+
init_dict, save_dict, sorted_listdir)
3426

3527
# Setting the random seed
3628
set_seed(UtilsConstants.DEFAULT_SEED.value)

‎src/thunder/datasets/dataset/crc.py‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
def download_crc(root_folder: str):
2-
from ..utils import (
3-
download_from_url,
4-
unzip_file,
5-
)
2+
from ..utils import download_from_url, unzip_file
63

74
download_from_url(
85
"https://zenodo.org/api/records/1214456/files/NCT-CRC-HE-100K.zip/content",
@@ -26,19 +23,15 @@ def create_splits_crc(base_folder: str, dataset_cfg: dict) -> None:
2623
:param dataset_cfg: dataset-specific config.
2724
"""
2825

29-
import numpy as np
3026
import os
3127
import random
3228

33-
from ..data_splits import (
34-
check_dataset,
35-
create_few_shot_training_data,
36-
get_data_from_set,
37-
init_dict,
38-
save_dict,
39-
)
29+
import numpy as np
30+
4031
from ...utils.constants import UtilsConstants
4132
from ...utils.utils import set_seed
33+
from ..data_splits import (check_dataset, create_few_shot_training_data,
34+
get_data_from_set, init_dict, save_dict)
4235

4336
# Setting the random seed
4437
set_seed(UtilsConstants.DEFAULT_SEED.value)

0 commit comments

Comments
 (0)