|
| 1 | +from typing import Dict, List, Tuple |
| 2 | + |
| 3 | +CLASS_TO_ID = { |
| 4 | + "ADI": 0, |
| 5 | + "LYM": 1, |
| 6 | + "MUC": 2, |
| 7 | + "MUS": 3, |
| 8 | + "NCS": 4, |
| 9 | + "NOR": 5, |
| 10 | + "BLD": 6, |
| 11 | + "FCT": 7, |
| 12 | + "TUM": 8, |
| 13 | +} |
| 14 | + |
| 15 | +VALID_EXTS = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"} |
| 16 | + |
| 17 | + |
| 18 | +def download_starc9(root_folder: str) -> None: |
| 19 | + """ |
| 20 | + Download the STARC-9 dataset from Hugging Face and extract all zip files. |
| 21 | +
|
| 22 | + Final split mapping: |
| 23 | + - train: Training_data_normalized |
| 24 | + - val: Validation_data/STANFORD-CRC-HE-VAL-SMALL |
| 25 | + - test: Validation_data/STANFORD-CRC-HE-VAL-LARGE |
| 26 | +
|
| 27 | + CURATED-TCGA is intentionally ignored here. |
| 28 | + """ |
| 29 | + from huggingface_hub import snapshot_download |
| 30 | + |
| 31 | + snapshot_download( |
| 32 | + repo_id="Path2AI/STARC-9", |
| 33 | + repo_type="dataset", |
| 34 | + local_dir=root_folder, |
| 35 | + local_dir_use_symlinks=False, |
| 36 | + ) |
| 37 | + |
| 38 | + extract_all_zips(root_folder) |
| 39 | + |
| 40 | + |
| 41 | +def extract_all_zips(root_dir: str) -> None: |
| 42 | + """ |
| 43 | + Recursively extract every .zip under root_dir into a folder with the same stem. |
| 44 | + """ |
| 45 | + import os |
| 46 | + from pathlib import Path |
| 47 | + |
| 48 | + from ..utils import unzip_file |
| 49 | + |
| 50 | + for current_root, _, files in os.walk(root_dir): |
| 51 | + for file_name in files: |
| 52 | + if not file_name.lower().endswith(".zip"): |
| 53 | + continue |
| 54 | + |
| 55 | + unzip_file( |
| 56 | + os.path.join(current_root, file_name), |
| 57 | + current_root, |
| 58 | + ) |
| 59 | + |
| 60 | + # Renaming folder extracted from STANFORD-CRC-HE-VAL-LARGE-NORMALIZED.zip |
| 61 | + if file_name == "STANFORD-CRC-HE-VAL-LARGE-NORMALIZED.zip": |
| 62 | + os.rename( |
| 63 | + os.path.join(current_root, "NORMALIZED"), |
| 64 | + os.path.join(current_root, "STANFORD-CRC-HE-VAL-LARGE"), |
| 65 | + ) |
| 66 | + |
| 67 | + |
| 68 | +def collect_images_from_class_root( |
| 69 | + class_root: str, |
| 70 | +) -> Tuple[List[str], List[int], Dict[str, int]]: |
| 71 | + """ |
| 72 | + Read all images from a directory structured like: |
| 73 | + class_root/ |
| 74 | + ADI/ |
| 75 | + LYM/ |
| 76 | + ... |
| 77 | + """ |
| 78 | + from pathlib import Path |
| 79 | + |
| 80 | + images: List[str] = [] |
| 81 | + labels: List[int] = [] |
| 82 | + |
| 83 | + class_root_path = Path(class_root) |
| 84 | + if not class_root_path.exists(): |
| 85 | + raise FileNotFoundError(f"Class root does not exist: {class_root}") |
| 86 | + |
| 87 | + missing_classes = [c for c in CLASS_TO_ID if not (class_root_path / c).exists()] |
| 88 | + if missing_classes: |
| 89 | + raise FileNotFoundError( |
| 90 | + f"Missing expected class folders under {class_root}: {missing_classes}" |
| 91 | + ) |
| 92 | + |
| 93 | + for class_name, class_id in CLASS_TO_ID.items(): |
| 94 | + class_dir = class_root_path / class_name |
| 95 | + for img_path in sorted(class_dir.rglob("*")): |
| 96 | + if img_path.is_file() and img_path.suffix.lower() in VALID_EXTS: |
| 97 | + images.append(str(img_path.resolve())) |
| 98 | + labels.append(class_id) |
| 99 | + |
| 100 | + return images, labels |
| 101 | + |
| 102 | + |
| 103 | +def create_splits_starc9(base_folder: str, dataset_cfg: dict) -> None: |
| 104 | + """ |
| 105 | + Generating data splits for the STARC-9 dataset. |
| 106 | +
|
| 107 | + :param base_folder: path to the main folder storing datasets. |
| 108 | + :param dataset_cfg: dataset-specific config. |
| 109 | + """ |
| 110 | + import os |
| 111 | + |
| 112 | + from ...utils.constants import UtilsConstants |
| 113 | + from ...utils.utils import set_seed |
| 114 | + from ..data_splits import ( |
| 115 | + check_dataset, |
| 116 | + create_few_shot_training_data, |
| 117 | + init_dict, |
| 118 | + save_dict, |
| 119 | + ) |
| 120 | + |
| 121 | + # Setting the random seed |
| 122 | + set_seed(UtilsConstants.DEFAULT_SEED.value) |
| 123 | + |
| 124 | + # Initializing dict |
| 125 | + starc9_data_splits = init_dict() |
| 126 | + |
| 127 | + # Getting folder paths |
| 128 | + dataset_root = os.path.join(base_folder, "starc9") |
| 129 | + train_root = os.path.join(dataset_root, "Training_data_normalized") |
| 130 | + val_root = os.path.join( |
| 131 | + dataset_root, |
| 132 | + "Validation_data", |
| 133 | + "STANFORD-CRC-HE-VAL-SMALL", |
| 134 | + ) |
| 135 | + test_root = os.path.join( |
| 136 | + dataset_root, |
| 137 | + "Validation_data", |
| 138 | + "STANFORD-CRC-HE-VAL-LARGE", |
| 139 | + ) |
| 140 | + |
| 141 | + # Collecting data |
| 142 | + train_images, train_labels = collect_images_from_class_root(train_root) |
| 143 | + val_images, val_labels = collect_images_from_class_root(val_root) |
| 144 | + test_images, test_labels = collect_images_from_class_root(test_root) |
| 145 | + |
| 146 | + # Updating dict |
| 147 | + starc9_data_splits["train"]["images"] = train_images |
| 148 | + starc9_data_splits["train"]["labels"] = train_labels |
| 149 | + starc9_data_splits["val"]["images"] = val_images |
| 150 | + starc9_data_splits["val"]["labels"] = val_labels |
| 151 | + starc9_data_splits["test"]["images"] = test_images |
| 152 | + starc9_data_splits["test"]["labels"] = test_labels |
| 153 | + |
| 154 | + # Few-shot training data |
| 155 | + starc9_data_splits = create_few_shot_training_data(starc9_data_splits) |
| 156 | + |
| 157 | + # Checking dataset characteristics |
| 158 | + check_dataset( |
| 159 | + starc9_data_splits, |
| 160 | + dataset_cfg, |
| 161 | + base_folder, |
| 162 | + ) |
| 163 | + |
| 164 | + # Saving dict |
| 165 | + save_dict( |
| 166 | + starc9_data_splits, os.path.join(base_folder, "data_splits", "starc9.json") |
| 167 | + ) |
0 commit comments