diff --git a/.github/mypy_changed.ini b/.github/mypy_changed.ini new file mode 100644 index 00000000..a60a5de0 --- /dev/null +++ b/.github/mypy_changed.ini @@ -0,0 +1,4 @@ +[mypy] +python_version = 3.11 +ignore_missing_imports = True +follow_imports = skip diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 59096e2a..df972046 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v4 @@ -54,14 +56,89 @@ jobs: python -m pip install --upgrade pip pip install black flake8 isort mypy + - name: Collect changed Python files + id: changed-py + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" + files=$(git diff --name-only --diff-filter=ACMRT "origin/${{ github.base_ref }}...HEAD" -- 'connectomics/**/*.py') + else + if git rev-parse --verify HEAD~1 >/dev/null 2>&1; then + files=$(git diff --name-only --diff-filter=ACMRT HEAD~1..HEAD -- 'connectomics/**/*.py') + else + files=$(git ls-files 'connectomics/**/*.py') + fi + fi + + echo "files<> "$GITHUB_OUTPUT" + echo "$files" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + - name: Run black - run: black --check connectomics/ + env: + CHANGED_FILES: ${{ steps.changed-py.outputs.files }} + run: | + python - <<'PY' + import os + import subprocess + import sys + + files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f] + if not files: + print("No changed Python files under connectomics/. Skipping black.") + sys.exit(0) + + subprocess.check_call(["black", "--check", *files]) + PY - name: Run flake8 - run: flake8 connectomics/ --max-line-length=100 + env: + CHANGED_FILES: ${{ steps.changed-py.outputs.files }} + run: | + python - <<'PY' + import os + import subprocess + import sys + + files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f] + if not files: + print("No changed Python files under connectomics/. Skipping flake8.") + sys.exit(0) + + subprocess.check_call(["flake8", "--max-line-length=100", *files]) + PY - name: Run isort - run: isort --check connectomics/ + env: + CHANGED_FILES: ${{ steps.changed-py.outputs.files }} + run: | + python - <<'PY' + import os + import subprocess + import sys + + files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f] + if not files: + print("No changed Python files under connectomics/. Skipping isort.") + sys.exit(0) + + subprocess.check_call(["isort", "--check", *files]) + PY - name: Run mypy - run: mypy connectomics/ --ignore-missing-imports + env: + CHANGED_FILES: ${{ steps.changed-py.outputs.files }} + run: | + python - <<'PY' + import os + import subprocess + import sys + + files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f] + if not files: + print("No changed Python files under connectomics/. Skipping mypy.") + sys.exit(0) + + subprocess.check_call(["mypy", "--config-file", ".github/mypy_changed.ini", *files]) + PY diff --git a/connectomics/data/dataset/__init__.py b/connectomics/data/dataset/__init__.py index cd165d1f..d56e162a 100755 --- a/connectomics/data/dataset/__init__.py +++ b/connectomics/data/dataset/__init__.py @@ -10,40 +10,44 @@ .claude/INFERENCE_DESIGN.md for details. """ +# Dataset factory functions (builder pattern) +from .build import ( + create_connectomics_dataset, + create_tile_data_dicts_from_json, + create_tile_dataset, + create_volume_dataset, +) + +# Shared data-dict helpers +from .data_dicts import ( + create_data_dicts_from_paths, + create_volume_data_dicts, +) + # MONAI base datasets from .dataset_base import ( - MonaiConnectomicsDataset, MonaiCachedConnectomicsDataset, + MonaiConnectomicsDataset, MonaiPersistentConnectomicsDataset, ) -# Volume datasets -from .dataset_volume import ( - MonaiVolumeDataset, - MonaiCachedVolumeDataset, +# Multi-dataset utilities +from .dataset_multi import ( + StratifiedConcatDataset, + UniformConcatDataset, + WeightedConcatDataset, ) # Tile datasets from .dataset_tile import ( - MonaiTileDataset, MonaiCachedTileDataset, + MonaiTileDataset, ) -# Multi-dataset utilities -from .dataset_multi import ( - WeightedConcatDataset, - StratifiedConcatDataset, - UniformConcatDataset, -) - -# Dataset factory functions (builder pattern) -from .build import ( - create_data_dicts_from_paths, - create_volume_data_dicts, - create_tile_data_dicts_from_json, - create_connectomics_dataset, - create_volume_dataset, - create_tile_dataset, +# Volume datasets +from .dataset_volume import ( + MonaiCachedVolumeDataset, + MonaiVolumeDataset, ) __all__ = [ diff --git a/connectomics/data/dataset/build.py b/connectomics/data/dataset/build.py index 6c6f3e2f..f1bb8d18 100644 --- a/connectomics/data/dataset/build.py +++ b/connectomics/data/dataset/build.py @@ -11,21 +11,26 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence, Tuple, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union + from monai.transforms import Compose +from .data_dicts import ( + create_data_dicts_from_paths, + create_volume_data_dicts, +) from .dataset_base import ( - MonaiConnectomicsDataset, MonaiCachedConnectomicsDataset, + MonaiConnectomicsDataset, MonaiPersistentConnectomicsDataset, ) from .dataset_tile import ( - MonaiTileDataset, MonaiCachedTileDataset, + MonaiTileDataset, ) if TYPE_CHECKING: - from .dataset_volume import MonaiVolumeDataset, MonaiCachedVolumeDataset + from .dataset_volume import MonaiCachedVolumeDataset, MonaiVolumeDataset __all__ = [ @@ -40,78 +45,6 @@ ] -# ============================================================================ -# Data Dictionary Creation -# ============================================================================ - - -def create_data_dicts_from_paths( - image_paths: List[str], - label_paths: Optional[List[str]] = None, - mask_paths: Optional[List[str]] = None, -) -> List[Dict[str, str]]: - """ - Create MONAI-style data dictionaries from file paths. - - Args: - image_paths: List of image file paths - label_paths: Optional list of label file paths - mask_paths: Optional list of mask file paths - - Returns: - List of dictionaries with 'image', 'label', and/or 'mask' keys - - Examples: - >>> image_paths = ['img1.h5', 'img2.h5'] - >>> label_paths = ['lbl1.h5', 'lbl2.h5'] - >>> data_dicts = create_data_dicts_from_paths(image_paths, label_paths) - >>> # [{'image': 'img1.h5', 'label': 'lbl1.h5'}, ...] - """ - data_dicts = [] - - for i, image_path in enumerate(image_paths): - data_dict = {"image": image_path} - - if label_paths is not None: - data_dict["label"] = label_paths[i] - - if mask_paths is not None: - data_dict["mask"] = mask_paths[i] - - data_dicts.append(data_dict) - - return data_dicts - - -def create_volume_data_dicts( - image_paths: List[str], - label_paths: Optional[List[str]] = None, - mask_paths: Optional[List[str]] = None, -) -> List[Dict[str, str]]: - """ - Create MONAI data dictionaries for volume datasets. - - This is a convenience wrapper around create_data_dicts_from_paths - for volume-specific use cases. - - Args: - image_paths: List of image volume file paths - label_paths: Optional list of label volume file paths - mask_paths: Optional list of valid mask file paths - - Returns: - List of MONAI-style data dictionaries - - Examples: - >>> data_dicts = create_volume_data_dicts(['vol1.tif'], ['lbl1.tif']) - """ - return create_data_dicts_from_paths( - image_paths=image_paths, - label_paths=label_paths, - mask_paths=mask_paths, - ) - - def create_tile_data_dicts_from_json( volume_json: str, label_json: Optional[str] = None, @@ -403,7 +336,7 @@ def create_volume_dataset( ... ) """ # Lazy import to avoid circular dependency during module import - from .dataset_volume import MonaiVolumeDataset, MonaiCachedVolumeDataset + from .dataset_volume import MonaiCachedVolumeDataset, MonaiVolumeDataset if dataset_type == "cached": return MonaiCachedVolumeDataset( diff --git a/connectomics/data/dataset/data_dicts.py b/connectomics/data/dataset/data_dicts.py new file mode 100644 index 00000000..d6ff0a71 --- /dev/null +++ b/connectomics/data/dataset/data_dicts.py @@ -0,0 +1,60 @@ +"""Shared helpers for constructing MONAI-style dataset dictionaries.""" + +from __future__ import annotations + +from typing import Dict, List, Optional + +__all__ = [ + "create_data_dicts_from_paths", + "create_volume_data_dicts", +] + + +def create_data_dicts_from_paths( + image_paths: List[str], + label_paths: Optional[List[str]] = None, + mask_paths: Optional[List[str]] = None, +) -> List[Dict[str, object]]: + """ + Create MONAI-style data dictionaries from file paths. + + Args: + image_paths: List of image file paths + label_paths: Optional list of label file paths + mask_paths: Optional list of mask file paths + + Returns: + List of dictionaries with 'image', 'label', and/or 'mask' keys + """ + data_dicts: List[Dict[str, object]] = [] + + for i, image_path in enumerate(image_paths): + data_dict: Dict[str, object] = {"image": image_path} + + if label_paths is not None: + data_dict["label"] = label_paths[i] + + if mask_paths is not None: + data_dict["mask"] = mask_paths[i] + + data_dicts.append(data_dict) + + return data_dicts + + +def create_volume_data_dicts( + image_paths: List[str], + label_paths: Optional[List[str]] = None, + mask_paths: Optional[List[str]] = None, +) -> List[Dict[str, object]]: + """ + Create MONAI data dictionaries for volume datasets. + + This is a convenience wrapper around ``create_data_dicts_from_paths`` + for volume-specific use cases. + """ + return create_data_dicts_from_paths( + image_paths=image_paths, + label_paths=label_paths, + mask_paths=mask_paths, + ) diff --git a/connectomics/data/dataset/dataset_volume.py b/connectomics/data/dataset/dataset_volume.py index 6d5d1bb6..b6d1043b 100755 --- a/connectomics/data/dataset/dataset_volume.py +++ b/connectomics/data/dataset/dataset_volume.py @@ -6,15 +6,16 @@ """ from __future__ import annotations + from typing import List, Optional, Tuple from monai.data import CacheDataset -from monai.transforms import Compose, RandSpatialCropd, CenterSpatialCropd +from monai.transforms import CenterSpatialCropd, Compose, RandSpatialCropd from monai.utils import ensure_tuple_rep -from .dataset_base import MonaiConnectomicsDataset -from .build import create_data_dicts_from_paths from ..io.monai_transforms import LoadVolumed +from .data_dicts import create_data_dicts_from_paths +from .dataset_base import MonaiConnectomicsDataset class MonaiVolumeDataset(MonaiConnectomicsDataset): diff --git a/connectomics/training/lit/__init__.py b/connectomics/training/lit/__init__.py index 619ae8a0..76cab4f8 100644 --- a/connectomics/training/lit/__init__.py +++ b/connectomics/training/lit/__init__.py @@ -4,27 +4,25 @@ Exports Lightning-specific modules (model, data, trainer, callbacks, config/utils). """ -from .model import ConnectomicsModule -from .data import ConnectomicsDataModule, VolumeDataModule, TileDataModule -from .trainer import create_trainer from .callbacks import ( - VisualizationCallback, - NaNDetectionCallback, EMAWeightsCallback, + NaNDetectionCallback, + VisualizationCallback, create_callbacks, ) from .config import ( - create_datamodule, expand_file_paths, setup_seed_everything, - setup_run_directory, - cleanup_run_directory, - modify_checkpoint_state, ) +from .data import ConnectomicsDataModule, TileDataModule, VolumeDataModule +from .data_factory import create_datamodule +from .model import ConnectomicsModule +from .runtime import cleanup_run_directory, modify_checkpoint_state, setup_run_directory +from .trainer import create_trainer from .utils import ( + extract_best_score_from_checkpoint, parse_args, setup_config, - extract_best_score_from_checkpoint, ) __all__ = [ diff --git a/connectomics/training/lit/config.py b/connectomics/training/lit/config.py index e4df2651..04e4293b 100644 --- a/connectomics/training/lit/config.py +++ b/connectomics/training/lit/config.py @@ -8,22 +8,17 @@ """ from __future__ import annotations -from glob import glob + from pathlib import Path from typing import List, Optional -import numpy as np import torch -# Import data and model utilities -from ...config import Config -from ...data.dataset import create_data_dicts_from_paths -from ...data.augment.build import ( - build_train_transforms, - build_val_transforms, - build_test_transforms, -) -from .data import ConnectomicsDataModule +from .data_factory import create_datamodule +from .path_utils import expand_file_paths as _expand_file_paths +from .runtime import cleanup_run_directory as _cleanup_run_directory +from .runtime import modify_checkpoint_state as _modify_checkpoint_state +from .runtime import setup_run_directory as _setup_run_directory def setup_seed_everything(): @@ -46,6 +41,7 @@ def setup_seed_everything(): # Fallback for older versions def seed_everything(seed, workers=True): import random + import numpy as np random.seed(seed) @@ -57,7 +53,7 @@ def seed_everything(seed, workers=True): def expand_file_paths(path_or_pattern) -> List[str]: """ - Expand glob patterns to list of file paths. + Backward-compatible wrapper for shared path expansion helper. Args: path_or_pattern: Single file path, glob pattern, or list of paths/patterns @@ -65,897 +61,17 @@ def expand_file_paths(path_or_pattern) -> List[str]: Returns: List of expanded file paths, sorted alphabetically """ - # If already a list, return it (may have been expanded by resolve_data_paths) - if isinstance(path_or_pattern, list): - return path_or_pattern - - # Check if pattern contains wildcards - if "*" in path_or_pattern or "?" in path_or_pattern: - # Expand glob pattern - paths = sorted(glob(path_or_pattern)) - if not paths: - raise FileNotFoundError(f"No files found matching pattern: {path_or_pattern}") - return paths - else: - # Single file path - return [path_or_pattern] - - -def _calculate_validation_iter_num( - val_data_dicts: List[Dict[str, Any]], - patch_size: Tuple[int, int, int], - min_iter: int = 50, - max_iter: int = 200, -) -> int: - """ - Calculate validation iter_num based on validation volume size and patch size. - - Args: - val_data_dicts: Validation data dictionaries - patch_size: Patch size (D, H, W) - min_iter: Minimum iterations per epoch - max_iter: Maximum iterations per epoch - - Returns: - Calculated validation iter_num - """ - try: - import nibabel as nib - import h5py - import tifffile - - # Get first validation volume size - img_path = Path(val_data_dicts[0]["image"]) - - # Load volume to get shape - if img_path.suffix in [".nii", ".gz"]: - # NIfTI file - vol = nib.load(str(img_path)) - vol_shape = vol.shape - elif img_path.suffix in [".h5", ".hdf5"]: - # HDF5 file - with h5py.File(img_path, "r") as f: - vol_shape = f[list(f.keys())[0]].shape - elif img_path.suffix in [".tif", ".tiff"]: - # TIFF file - vol = tifffile.imread(img_path) - vol_shape = vol.shape - else: - # Unknown format, use default - print(f" ⚠️ Unknown file format {img_path.suffix}, using default val_iter_num=100") - return 100 - - # Handle channel dimension if present - if len(vol_shape) == 4: - vol_shape = vol_shape[1:] # Remove channel dim: (C, D, H, W) -> (D, H, W) - - # Calculate number of possible patches (with 50% overlap) - stride = tuple(p // 2 for p in patch_size) # 50% overlap - num_patches_per_dim = [ - max(1, (vol_shape[i] - patch_size[i]) // stride[i] + 1) - for i in range(3) - ] - total_possible_patches = num_patches_per_dim[0] * num_patches_per_dim[1] * num_patches_per_dim[2] - - # Calculate validation iter_num as a fraction of possible patches - # Use 5-10% of possible patches, but clamp to [min_iter, max_iter] - val_iter_num = int(total_possible_patches * 0.075) # 7.5% of possible patches - val_iter_num = max(min_iter, min(max_iter, val_iter_num)) - - print(f" Validation volume shape: {vol_shape}") - print(f" Patch size: {patch_size}") - print(f" Stride (50% overlap): {stride}") - print(f" Possible patches per dim: {num_patches_per_dim}") - print(f" Total possible patches: {total_possible_patches}") - print(f" Using 7.5% of patches: {val_iter_num}") - - return val_iter_num - - except Exception as e: - print(f" ⚠️ Error calculating validation iter_num: {e}") - print(f" ℹ️ Using default val_iter_num=100") - return 100 - - -def create_datamodule( - cfg: Config, mode: str = "train", fast_dev_run: bool = False -) -> ConnectomicsDataModule: - """ - Create Lightning DataModule from config. - - Args: - cfg: Hydra Config object - mode: 'train', 'test', or 'tune' - fast_dev_run: If True, config overrides have already been applied in setup_config() - - Returns: - ConnectomicsDataModule instance - """ - print("Creating datasets...") - - # Auto-download tutorial data if missing - if mode == "train" and cfg.data.train_image: - from pathlib import Path as PathLib - - # Check if data exists (support glob patterns and lists) - data_exists = False - - # Handle list of files - if isinstance(cfg.data.train_image, list): - # Check if at least one file in the list exists - data_exists = any(PathLib(img).exists() for img in cfg.data.train_image) - # Handle glob pattern - elif "*" in cfg.data.train_image or "?" in cfg.data.train_image: - # Glob pattern - check if any files match - matched_files = glob(cfg.data.train_image) - data_exists = len(matched_files) > 0 - # Handle single file path - else: - data_exists = PathLib(cfg.data.train_image).exists() - - if not data_exists: - print(f"\n⚠️ Training data not found: {cfg.data.train_image}") - - # Try to infer dataset name from path - from ...utils.download import DATASETS, download_dataset - - path_str = str(cfg.data.train_image).lower() - dataset_name = None - for name in DATASETS.keys(): - if name in path_str and not name.endswith("++"): # Skip aliases - dataset_name = name - break - - if dataset_name: - print(f"💡 Attempting to auto-download '{dataset_name}' dataset...") - print(" (You can disable auto-download by manually downloading data)") - - # Prompt user - try: - size_mb = DATASETS[dataset_name]["size_mb"] - prompt = f" Download {dataset_name} dataset (~{size_mb} MB)? [Y/n]: " - response = input(prompt).strip().lower() - if response in ["", "y", "yes"]: - if download_dataset(dataset_name, base_dir=PathLib.cwd()): - print("✅ Data downloaded successfully!") - else: - print("❌ Download failed. Please download manually:") - print(f" wget {DATASETS[dataset_name]['url']}") - raise FileNotFoundError( - f"Training data not found: {cfg.data.train_image}" - ) - else: - print("❌ Download cancelled. Please download manually.") - raise FileNotFoundError(f"Training data not found: {cfg.data.train_image}") - except KeyboardInterrupt: - print("\n❌ Download cancelled by user") - raise FileNotFoundError(f"Training data not found: {cfg.data.train_image}") - else: - print("💡 Available datasets:") - from ...utils.download import list_datasets - - list_datasets() - raise FileNotFoundError(f"Training data not found: {cfg.data.train_image}") - - # Check dataset type early - dataset_type = getattr(cfg.data, "dataset_type", None) - - # Build transforms - train_transforms = build_train_transforms(cfg) - val_transforms = build_val_transforms(cfg) - test_transforms = build_test_transforms(cfg) if mode in ["test", "tune"] else val_transforms - - print(f" Train transforms: {len(train_transforms.transforms)} steps") - print(f" Val transforms: {len(val_transforms.transforms)} steps") - if mode in ["test", "tune"]: - print( - f" Test transforms: {len(test_transforms.transforms)} steps (no sliding-window crop)" - ) - - # For test/tune modes, skip training data setup entirely - if mode in ["test", "tune"]: - train_data_dicts = [] - val_data_dicts = None - # Check if automatic train/val split is enabled - elif cfg.data.split_enabled and not cfg.data.val_image: - print("🔀 Using automatic train/val split (DeepEM-style)") - from ...data.utils.split import split_volume_train_val - - # Load full volume - import h5py - import tifffile - - train_path = Path(cfg.data.train_image) - if train_path.suffix in [".h5", ".hdf5"]: - with h5py.File(train_path, "r") as f: - volume_shape = f[list(f.keys())[0]].shape - elif train_path.suffix in [".tif", ".tiff"]: - volume = tifffile.imread(train_path) - volume_shape = volume.shape - else: - raise ValueError(f"Unsupported file format: {train_path.suffix}") - - print(f" Volume shape: {volume_shape}") - - # Calculate split ranges - train_ratio = cfg.data.split_train_range[1] - cfg.data.split_train_range[0] - - train_slices, val_slices = split_volume_train_val( - volume_shape=volume_shape, - train_ratio=train_ratio, - axis=cfg.data.split_axis, - ) - - # Calculate train and val regions - axis = cfg.data.split_axis - train_start = int(volume_shape[axis] * cfg.data.split_train_range[0]) - train_end = int(volume_shape[axis] * cfg.data.split_train_range[1]) - val_start = int(volume_shape[axis] * cfg.data.split_val_range[0]) - val_end = int(volume_shape[axis] * cfg.data.split_val_range[1]) - - print(f" Split axis: {axis} ({'Z' if axis == 0 else 'Y' if axis == 1 else 'X'})") - print(f" Train region: [{train_start}:{train_end}] ({train_end - train_start} slices)") - print(f" Val region: [{val_start}:{val_end}] ({val_end - val_start} slices)") - - if cfg.data.split_pad_val: - target_size = tuple(cfg.data.patch_size) - print(f" Val padding enabled: target size = {target_size}") - - # Create data dictionaries with split info - train_data_dicts = create_data_dicts_from_paths( - image_paths=[cfg.data.train_image], - label_paths=[cfg.data.train_label] if cfg.data.train_label else None, - ) - - # Add split metadata to train dict - train_data_dicts[0]["split_slices"] = train_slices - train_data_dicts[0]["split_mode"] = "train" - - # Create validation data dicts using same volume - val_data_dicts = create_data_dicts_from_paths( - image_paths=[cfg.data.train_image], - label_paths=[cfg.data.train_label] if cfg.data.train_label else None, - ) - - # Add split metadata to val dict - val_data_dicts[0]["split_slices"] = val_slices - val_data_dicts[0]["split_mode"] = "val" - val_data_dicts[0]["split_pad"] = cfg.data.split_pad_val - val_data_dicts[0]["split_pad_mode"] = cfg.data.split_pad_mode - if cfg.data.split_pad_val: - val_data_dicts[0]["split_pad_size"] = tuple(cfg.data.patch_size) - - else: - # Check dataset type to determine how to load data - if dataset_type == "filename": - # Check if train_json is empty or doesn't exist - train_json_empty = False - if cfg.data.train_json is None or cfg.data.train_json == "": - train_json_empty = True - else: - try: - import json - - json_path = Path(cfg.data.train_json) - if not json_path.exists(): - train_json_empty = True - else: - # Check if JSON file is empty or has no images - with open(json_path, "r") as f: - json_data = json.load(f) - image_files = json_data.get(cfg.data.train_image_key, []) - if not image_files: - train_json_empty = True - except (FileNotFoundError, json.JSONDecodeError, KeyError): - train_json_empty = True - - if train_json_empty: - # Fallback to volume-based dataset when train_json is empty - print(" ⚠️ Train JSON is empty or invalid, falling back to volume-based dataset") - print(f" Train JSON: {cfg.data.train_json}") - dataset_type = None # Switch to volume-based - else: - # Filename-based dataset: uses JSON file lists - print(" Using filename-based dataset") - print(f" Train JSON: {cfg.data.train_json}") - print(f" Image key: {cfg.data.train_image_key}") - print(f" Label key: {cfg.data.train_label_key}") - - # For filename dataset, we'll create data dicts later in the DataModule - # Here we just need placeholder dicts - train_data_dicts = [{"dataset_type": "filename"}] - val_data_dicts = None # Handled by train_val_split in DataModule - - if dataset_type != "filename": - # Standard mode: separate train and val files (supports glob patterns) - if cfg.data.train_image is None: - raise ValueError( - "For volume-based datasets, data.train_image must be specified.\n" - "Either set data.train_image or use data.dataset_type='filename' with " - "data.train_json" - ) - - train_image_paths = expand_file_paths(cfg.data.train_image) - train_label_paths = ( - expand_file_paths(cfg.data.train_label) if cfg.data.train_label else None - ) - train_mask_paths = ( - expand_file_paths(cfg.data.train_mask) if cfg.data.train_mask else None - ) - - print(f" Training volumes: {len(train_image_paths)} files") - if len(train_image_paths) <= 5: - for path in train_image_paths: - print(f" - {path}") - else: - print(f" - {train_image_paths[0]}") - print(f" - ... ({len(train_image_paths) - 2} more files)") - print(f" - {train_image_paths[-1]}") - - if train_mask_paths: - print(f" Training masks: {len(train_mask_paths)} files") - - train_data_dicts = create_data_dicts_from_paths( - image_paths=train_image_paths, - label_paths=train_label_paths, - mask_paths=train_mask_paths, - ) - - val_data_dicts = None - if cfg.data.val_image: - val_image_paths = expand_file_paths(cfg.data.val_image) - val_label_paths = ( - expand_file_paths(cfg.data.val_label) if cfg.data.val_label else None - ) - val_mask_paths = expand_file_paths(cfg.data.val_mask) if cfg.data.val_mask else None - - print(f" Validation volumes: {len(val_image_paths)} files") - if val_mask_paths: - print(f" Validation masks: {len(val_mask_paths)} files") - - val_data_dicts = create_data_dicts_from_paths( - image_paths=val_image_paths, - label_paths=val_label_paths, - mask_paths=val_mask_paths, - ) - - # Create test data dicts if in test or tune mode - test_data_dicts = None - if mode == "test": - if ( - not hasattr(cfg, "test") - or cfg.test is None - or not hasattr(cfg.test, "data") - or not cfg.test.data.test_image - ): - test_image_val = ( - cfg.test.data.test_image - if hasattr(cfg, "test") and cfg.test and hasattr(cfg.test, "data") - else "N/A" - ) - raise ValueError( - f"Test mode requires test.data.test_image to be set in config.\n" - f"Current config has: test.data.test_image = {test_image_val}" - ) - print(f" 🧪 Creating test dataset from: {cfg.test.data.test_image}") - - # Expand glob patterns for test data (same as train data) - test_image_paths = expand_file_paths(cfg.test.data.test_image) - test_label_paths = ( - expand_file_paths(cfg.test.data.test_label) if cfg.test.data.test_label else None - ) - test_mask_paths = ( - expand_file_paths(cfg.test.data.test_mask) - if hasattr(cfg.test.data, "test_mask") and cfg.test.data.test_mask - else None - ) - elif mode == "tune": - # For tune mode, read from cfg.tune.data - if ( - not hasattr(cfg, "tune") - or cfg.tune is None - or not hasattr(cfg.tune, "data") - or not cfg.tune.data.tune_image - ): - tune_image_val = ( - cfg.tune.data.tune_image - if hasattr(cfg, "tune") and cfg.tune and hasattr(cfg.tune, "data") - else "N/A" - ) - raise ValueError( - f"Tune mode requires tune.data.tune_image to be set in config.\n" - f"Current config has tune.data.tune_image: {tune_image_val}" - ) - - print(f" 🎯 Creating tune dataset from: {cfg.tune.data.tune_image}") - - # Expand glob patterns for tune data - test_image_paths = expand_file_paths(cfg.tune.data.tune_image) - test_label_paths = ( - expand_file_paths(cfg.tune.data.tune_label) if cfg.tune.data.tune_label else None - ) - test_mask_paths = ( - expand_file_paths(cfg.tune.data.tune_mask) if cfg.tune.data.tune_mask else None - ) - - # Common printing and data dict creation for test and tune modes - if mode in ["test", "tune"]: - mode_label = "Test" if mode == "test" else "Tune" - print(f" {mode_label} volumes: {len(test_image_paths)} files") - if len(test_image_paths) <= 5: - for path in test_image_paths: - print(f" - {path}") - else: - print(f" - {test_image_paths[0]}") - print(f" - ... ({len(test_image_paths) - 2} more files)") - print(f" - {test_image_paths[-1]}") - - if test_mask_paths: - print(f" {mode_label} masks: {len(test_mask_paths)} files") - - test_data_dicts = create_data_dicts_from_paths( - image_paths=test_image_paths, - label_paths=test_label_paths, - mask_paths=test_mask_paths, - ) - print(f" {mode_label} dataset size: {len(test_data_dicts)}") - - if mode == "train": - print(f" Train dataset size: {len(train_data_dicts)}") - if val_data_dicts: - print(f" Val dataset size: {len(val_data_dicts)}") - - # Auto-compute iter_num from volume size if not specified (only for training) - iter_num = None - if mode == "train": - iter_num = cfg.data.iter_num_per_epoch - if iter_num == -1 and dataset_type != "filename": - # For filename datasets, iter_num is determined by the number of files - print("📊 Auto-computing iter_num from volume size...") - from ...data.utils import compute_total_samples - import h5py - import tifffile - - # Get volume sizes - volume_sizes = [] - for data_dict in train_data_dicts: - img_path = Path(data_dict["image"]) - if img_path.suffix in [".h5", ".hdf5"]: - with h5py.File(img_path, "r") as f: - vol_shape = f[list(f.keys())[0]].shape - elif img_path.suffix in [".tif", ".tiff"]: - vol = tifffile.imread(img_path) - vol_shape = vol.shape - else: - raise ValueError(f"Unsupported file format: {img_path.suffix}") - - # Handle both (z, y, x) and (c, z, y, x) - if len(vol_shape) == 4: - vol_shape = vol_shape[1:] # Skip channel dim - volume_sizes.append(vol_shape) - - # Compute total possible samples - total_samples, samples_per_vol = compute_total_samples( - volume_sizes=volume_sizes, - patch_size=tuple(cfg.data.patch_size), - stride=tuple(cfg.data.stride), - ) - - iter_num = total_samples - print(f" Volume sizes: {volume_sizes}") - print(f" Patch size: {cfg.data.patch_size}") - print(f" Stride: {cfg.data.stride}") - print(f" Samples per volume: {samples_per_vol}") - print(f" ✅ Total possible samples (iter_num): {iter_num:,}") - print(f" ✅ Batches per epoch: {iter_num // cfg.system.training.batch_size:,}") - elif iter_num == -1 and dataset_type == "filename": - # For filename datasets, iter_num will be determined by dataset length - print(" Filename dataset: iter_num will be determined by number of files in JSON") - - # Create DataModule - print("Creating data loaders...") - - # For test/tune modes, disable iter_num (process full volumes once) - if mode in ["test", "tune"]: - iter_num_for_dataset = -1 # Process full volumes without random sampling - else: - iter_num_for_dataset = iter_num - - # Select appropriate batch_size and num_workers based on mode - # For test/tune modes, use system.inference settings instead of system.training - if mode in ["test", "tune", "tune-test"]: - batch_size = cfg.system.inference.batch_size - num_workers = cfg.system.inference.num_workers - print(f" Using inference settings: batch_size={batch_size}, num_workers={num_workers}") - else: - batch_size = cfg.system.training.batch_size - num_workers = cfg.system.training.num_workers - print(f" Using training settings: batch_size={batch_size}, num_workers={num_workers}") - - # Use optimized pre-loaded cache when iter_num > 0 (only for training mode and volume datasets) - use_preloaded = ( - cfg.data.use_preloaded_cache - and iter_num is not None - and iter_num > 0 - and mode == "train" - and dataset_type != "filename" - ) - - if use_preloaded: - print(" ⚡ Using pre-loaded volume cache (loads once, crops in memory)") - from ...data.dataset.dataset_volume_cached import CachedVolumeDataset - from torch.utils.data import DataLoader - import pytorch_lightning as pl - - # Build transforms without loading/cropping (handled by dataset) - augment_only_transforms = build_train_transforms(cfg, skip_loading=True) - - # Get padding parameters from config - pad_size = getattr(cfg.data.image_transform, "pad_size", None) - pad_mode = getattr(cfg.data.image_transform, "pad_mode", "reflect") - - # Create optimized cached datasets - train_dataset = CachedVolumeDataset( - image_paths=[d["image"] for d in train_data_dicts], - label_paths=[d.get("label") for d in train_data_dicts], - patch_size=tuple(cfg.data.patch_size), - iter_num=iter_num, - transforms=augment_only_transforms, - mode="train", - pad_size=tuple(pad_size) if pad_size else None, - pad_mode=pad_mode, - ) - - # Use fewer workers since we're loading from memory - preloaded_num_workers = min(num_workers, 2) - print(f" Using {preloaded_num_workers} workers (in-memory operations are fast)") - - # Create simple dataloader - train_loader = DataLoader( - train_dataset, - batch_size=batch_size, - shuffle=False, # Already random - num_workers=preloaded_num_workers, - pin_memory=cfg.data.pin_memory, - persistent_workers=preloaded_num_workers > 0, - ) - - # Create validation dataset and loader if validation data exists - val_loader = None - if val_data_dicts and len(val_data_dicts) > 0: - print(" Creating validation dataset with pre-loaded cache...") - - # Build validation transforms (no augmentation, only normalization) - val_only_transforms = build_val_transforms(cfg, skip_loading=True) - - # Get validation iter_num (auto-calculate if not specified) - val_iter_num = cfg.data.val_iter_num if hasattr(cfg.data, 'val_iter_num') and cfg.data.val_iter_num else None - - if val_iter_num is None: - # Auto-calculate validation iter_num from volume size - print(" 📊 Auto-calculating validation iter_num from volume size...") - import h5py - import tifffile - from pathlib import Path - - # Get first validation volume shape - val_img_path = Path(val_data_dicts[0]["image"]) - if val_img_path.suffix in [".h5", ".hdf5"]: - with h5py.File(val_img_path, "r") as f: - val_vol_shape = f[list(f.keys())[0]].shape - elif val_img_path.suffix in [".tif", ".tiff"]: - val_vol = tifffile.imread(val_img_path) - val_vol_shape = val_vol.shape - else: - val_vol_shape = (100, 4096, 4096) # Default fallback - - # Handle both (z, y, x) and (c, z, y, x) - if len(val_vol_shape) == 4: - val_vol_shape = val_vol_shape[1:] # Skip channel dim - - patch_size = tuple(cfg.data.patch_size) - stride = tuple([p // 2 for p in patch_size]) # 50% overlap - - # Calculate possible patches - possible_patches = [ - max(1, (vol_dim - patch_dim) // stride_dim + 1) - for vol_dim, patch_dim, stride_dim in zip(val_vol_shape, patch_size, stride) - ] - total_possible = int(np.prod(possible_patches)) - - # Use 7.5% of patches (same as before) - val_iter_num = max(1, int(total_possible * 0.075)) - - print(f" Validation volume shape: {val_vol_shape}") - print(f" Patch size: {patch_size}") - print(f" Stride (50% overlap): {stride}") - print(f" Possible patches per dim: {possible_patches}") - print(f" Total possible patches: {total_possible}") - print(f" Using 7.5% of patches: {val_iter_num}") - print(f" ✅ Validation iter_num: {val_iter_num} (auto-calculated)") - - # Create validation dataset - val_dataset = CachedVolumeDataset( - image_paths=[d["image"] for d in val_data_dicts], - label_paths=[d.get("label") for d in val_data_dicts], - patch_size=tuple(cfg.data.patch_size), - iter_num=val_iter_num, - transforms=val_only_transforms, - mode="val", - pad_size=tuple(pad_size) if pad_size else None, - pad_mode=pad_mode, - ) - - # Create validation dataloader - val_loader = DataLoader( - val_dataset, - batch_size=batch_size, - shuffle=False, - num_workers=preloaded_num_workers, - pin_memory=cfg.data.pin_memory, - persistent_workers=preloaded_num_workers > 0, - ) - print(f" ✅ Validation dataloader created with {val_iter_num} iterations") - - # Create data module wrapper that inherits from LightningDataModule - class SimpleDataModule(pl.LightningDataModule): - def __init__(self, train_loader, val_loader=None): - super().__init__() - self.train_loader = train_loader - self._val_loader = val_loader - - def train_dataloader(self): - return self.train_loader - - def val_dataloader(self): - if self._val_loader is not None: - return self._val_loader - return [] - - def test_dataloader(self): - # For test mode, return empty list (user should use standard datamodule) - return [] - - def setup(self, stage=None): - pass - - datamodule = SimpleDataModule(train_loader, val_loader) - elif dataset_type == "filename": - # Filename-based dataset using JSON file lists - print(" Creating filename-based datamodule...") - from ...data.dataset.dataset_filename import create_filename_datasets - from torch.utils.data import DataLoader - import pytorch_lightning as pl - - # Create train and val datasets from JSON - train_dataset, val_dataset = create_filename_datasets( - json_path=cfg.data.train_json, - train_transforms=train_transforms, - val_transforms=val_transforms, - train_val_split=( - cfg.data.train_val_split if hasattr(cfg.data, "train_val_split") else 0.9 - ), - random_seed=cfg.system.seed if hasattr(cfg.system, "seed") else 42, - images_key=cfg.data.train_image_key, - labels_key=cfg.data.train_label_key, - use_labels=True, - ) - - print(f" Train dataset size: {len(train_dataset)}") - print(f" Val dataset size: {len(val_dataset)}") - - # Create simple datamodule wrapper - class FilenameDataModule(pl.LightningDataModule): - def __init__( - self, train_ds, val_ds, batch_size, num_workers, pin_memory, persistent_workers - ): - super().__init__() - self.train_ds = train_ds - self.val_ds = val_ds - self.batch_size = batch_size - self.num_workers = num_workers - self.pin_memory = pin_memory - self.persistent_workers = persistent_workers - - def train_dataloader(self): - return DataLoader( - self.train_ds, - batch_size=self.batch_size, - shuffle=True, - num_workers=self.num_workers, - pin_memory=self.pin_memory, - persistent_workers=self.persistent_workers and self.num_workers > 0, - ) - - def val_dataloader(self): - if self.val_ds is None or len(self.val_ds) == 0: - return [] - return DataLoader( - self.val_ds, - batch_size=self.batch_size, - shuffle=False, - num_workers=self.num_workers, - pin_memory=self.pin_memory, - persistent_workers=self.persistent_workers and self.num_workers > 0, - ) - - def test_dataloader(self): - return [] - - def setup(self, stage=None): - pass - - datamodule = FilenameDataModule( - train_ds=train_dataset, - val_ds=val_dataset, - batch_size=batch_size, - num_workers=num_workers, - pin_memory=cfg.data.pin_memory, - persistent_workers=cfg.data.persistent_workers, - ) - else: - # Standard data module - # Disable caching for test/tune modes to avoid issues with partial cache returning 0 length - use_cache = cfg.data.use_cache and mode == "train" - - if mode in ["test", "tune"] and cfg.data.use_cache: - print(" ⚠️ Caching disabled for test/tune mode (incompatible with partial cache)") - - # Note: transpose_axes handled in transform builders (build_train/val/test_transforms) - # They embed the transpose in LoadVolumed, so no need to pass it here - - # Get validation iter_num (separate from training iter_num) - val_iter_num = getattr(cfg.data, 'val_iter_num', None) - if val_iter_num is None and val_data_dicts: - # Auto-calculate validation iter_num based on volume size and patch size - print(" 📊 Auto-calculating validation iter_num from volume size...") - val_iter_num = _calculate_validation_iter_num( - val_data_dicts=val_data_dicts, - patch_size=tuple(cfg.data.patch_size), - min_iter=50, - max_iter=200, - ) - print(f" ✅ Validation iter_num: {val_iter_num} (auto-calculated)") - - datamodule = ConnectomicsDataModule( - train_data_dicts=train_data_dicts, - val_data_dicts=val_data_dicts, - test_data_dicts=test_data_dicts, - transforms={ - "train": train_transforms, - "val": val_transforms, - "test": test_transforms, - }, - dataset_type="cached" if use_cache else "standard", - batch_size=batch_size, - num_workers=num_workers, - pin_memory=cfg.data.pin_memory, - persistent_workers=cfg.data.persistent_workers, - cache_rate=cfg.data.cache_rate if use_cache else 0.0, - iter_num=iter_num_for_dataset, - val_iter_num=val_iter_num, - seed=cfg.system.seed, # [FIX 1] Pass seed for validation reseeding - sample_size=tuple(cfg.data.patch_size), - do_2d=cfg.data.do_2d, - ) - # Setup datasets based on mode - if mode == "train": - datamodule.setup(stage="fit") - elif mode in ["test", "tune"]: - datamodule.setup(stage="test") - - # Print dataset info based on mode - if mode == "train": - print(f" Train batches: {len(datamodule.train_dataloader())}") - if val_data_dicts: - print(f" Val batches: {len(datamodule.val_dataloader())}") - elif mode in ["test", "tune"]: - print(f" Test batches: {len(datamodule.test_dataloader())}") - - return datamodule + return _expand_file_paths(path_or_pattern) def setup_run_directory(mode: str, cfg, checkpoint_dirpath: str): - """ - Setup run directory with timestamp for training mode. - Handles DDP subprocess coordination via timestamp files. - - Args: - mode: 'train', 'test', 'tune', or 'tune-test' - cfg: Config object (will be modified in-place for training mode) - checkpoint_dirpath: Path to checkpoint/output directory from config - - Returns: - Path: Run directory (timestamped for train, created for tune modes, dummy for test) - """ - import os - from datetime import datetime - from ...config import save_config - - checkpoint_dir = Path(checkpoint_dirpath) - checkpoint_subdir = checkpoint_dir.name or "checkpoints" - output_base = checkpoint_dir.parent - - if mode == "train": - # Check if this is a DDP re-launch (LOCAL_RANK is set by PyTorch Lightning) - is_ddp_subprocess = "LOCAL_RANK" in os.environ - local_rank = int(os.environ.get("LOCAL_RANK", 0)) - timestamp_file = output_base / ".latest_timestamp" - - if not is_ddp_subprocess: - # First invocation (main process) - create new timestamp - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - run_dir = output_base / timestamp - checkpoint_path = run_dir / checkpoint_subdir - cfg.monitor.checkpoint.dirpath = str(checkpoint_path) - - checkpoint_path.mkdir(parents=True, exist_ok=True) - print(f"📁 Run directory: {run_dir}") - - # Save config to run directory - config_save_path = run_dir / "config.yaml" - save_config(cfg, config_save_path) - print(f"💾 Config saved to: {config_save_path}") - - # Save timestamp for DDP subprocesses to read - output_base.mkdir(parents=True, exist_ok=True) - timestamp_file.write_text(timestamp) - else: - # DDP subprocess - read existing timestamp - import time - - max_wait = 30 # Maximum 30 seconds - waited = 0 - while not timestamp_file.exists() and waited < max_wait: - time.sleep(0.1) - waited += 0.1 - - if timestamp_file.exists(): - timestamp = timestamp_file.read_text().strip() - run_dir = output_base / timestamp - checkpoint_path = run_dir / checkpoint_subdir - cfg.monitor.checkpoint.dirpath = str(checkpoint_path) - print(f"📁 [DDP Rank {local_rank}] Using run directory: {run_dir}") - else: - raise RuntimeError( - f"DDP subprocess (LOCAL_RANK={local_rank}) timed out waiting for timestamp file" - ) - elif mode in ["tune", "tune-test"]: - # For tune modes, create the directory for Optuna outputs - run_dir = Path(checkpoint_dirpath) - run_dir.mkdir(parents=True, exist_ok=True) - print(f"📁 Tuning output directory: {run_dir}") - elif mode == "test": - # For test mode, create the results directory - run_dir = Path(checkpoint_dirpath) - run_dir.mkdir(parents=True, exist_ok=True) - print(f"📁 Test output directory: {run_dir}") - else: - # Fallback for unknown modes - run_dir = output_base / f"{mode}_run" - print(f"📝 Running in {mode} mode") - - return run_dir + """Backward-compatible wrapper for runtime directory setup.""" + return _setup_run_directory(mode, cfg, checkpoint_dirpath) def cleanup_run_directory(output_base: Path): - """ - Clean up timestamp file after training completes. - Only runs in main process (not DDP subprocesses). - - Args: - output_base: Base output directory containing timestamp file - """ - import os - - is_ddp_subprocess = "LOCAL_RANK" in os.environ - if not is_ddp_subprocess: - timestamp_file = output_base / ".latest_timestamp" - if timestamp_file.exists(): - try: - timestamp_file.unlink() - except Exception: - pass # Ignore cleanup errors + """Backward-compatible wrapper for runtime directory cleanup.""" + return _cleanup_run_directory(output_base) def modify_checkpoint_state( @@ -966,72 +82,15 @@ def modify_checkpoint_state( reset_epoch: bool = False, reset_early_stopping: bool = False, ) -> Optional[str]: - """ - Modify checkpoint state for training resumption with selective resets. - - Args: - checkpoint_path: Path to checkpoint file to modify (None if no checkpoint) - run_dir: Run directory for saving modified checkpoint - reset_optimizer: If True, reset optimizer state - reset_scheduler: If True, reset scheduler state - reset_epoch: If True, reset epoch counter - reset_early_stopping: If True, reset early stopping patience - - Returns: - Path to modified checkpoint file, or original path if no modifications needed - """ - import torch - - # Early return if no checkpoint or no resets requested - if not checkpoint_path: - return None - - if not (reset_optimizer or reset_scheduler or reset_epoch or reset_early_stopping): - return checkpoint_path - - print("\n🔄 Modifying checkpoint state:") - if reset_optimizer: - print(" - Resetting optimizer state") - if reset_scheduler: - print(" - Resetting scheduler state") - if reset_epoch: - print(" - Resetting epoch counter") - if reset_early_stopping: - print(" - Resetting early stopping patience counter") - - # Load checkpoint (weights_only=False needed for PyTorch 2.6+ to load Lightning - # checkpoints with custom configs) - checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) - - # Reset optimizer state - if reset_optimizer and "optimizer_states" in checkpoint: - del checkpoint["optimizer_states"] - - # Reset scheduler state - if reset_scheduler and "lr_schedulers" in checkpoint: - del checkpoint["lr_schedulers"] - - # Reset epoch counter - if reset_epoch: - if "epoch" in checkpoint: - checkpoint["epoch"] = 0 - if "global_step" in checkpoint: - checkpoint["global_step"] = 0 - - # Reset early stopping state - if reset_early_stopping and "callbacks" in checkpoint: - for callback_state in checkpoint["callbacks"].values(): - if "wait_count" in callback_state: - callback_state["wait_count"] = 0 - if "best_score" in callback_state: - callback_state["best_score"] = None - - # Save modified checkpoint to temporary file - temp_ckpt_path = run_dir / "temp_modified_checkpoint.ckpt" - torch.save(checkpoint, temp_ckpt_path) - print(f" ✅ Modified checkpoint saved to: {temp_ckpt_path}") - - return str(temp_ckpt_path) + """Backward-compatible wrapper for runtime checkpoint mutation.""" + return _modify_checkpoint_state( + checkpoint_path, + run_dir, + reset_optimizer=reset_optimizer, + reset_scheduler=reset_scheduler, + reset_epoch=reset_epoch, + reset_early_stopping=reset_early_stopping, + ) __all__ = [ diff --git a/connectomics/training/lit/data_factory.py b/connectomics/training/lit/data_factory.py new file mode 100644 index 00000000..829cf679 --- /dev/null +++ b/connectomics/training/lit/data_factory.py @@ -0,0 +1,797 @@ +"""DataModule factory functions for Lightning training.""" + +from __future__ import annotations + +from glob import glob +from pathlib import Path +from typing import List, Optional + +from ...config import Config +from ...data.augment.build import ( + build_test_transforms, + build_train_transforms, + build_val_transforms, +) +from ...data.dataset import create_data_dicts_from_paths +from .data import ConnectomicsDataModule +from .path_utils import expand_file_paths as _expand_file_paths + + +def expand_file_paths(path_or_pattern) -> List[str]: + """Expand file path inputs via shared path helper.""" + return _expand_file_paths(path_or_pattern) + + +def _calculate_validation_iter_num( + val_data_dicts: List[dict], + patch_size: tuple[int, int, int], + min_iter: int = 50, + max_iter: Optional[int] = 200, + default_iter_num: int = 100, + fallback_volume_shape: Optional[tuple[int, int, int]] = None, + return_default_on_error: bool = True, +) -> int: + """ + Calculate validation iter_num based on validation volume size and patch size. + + Args: + val_data_dicts: Validation data dictionaries + patch_size: Patch size (D, H, W) + min_iter: Minimum iterations per epoch + max_iter: Maximum iterations per epoch + default_iter_num: Default iter_num when calculation fails + fallback_volume_shape: Volume shape fallback for unknown file formats + return_default_on_error: Return default_iter_num on errors instead of raising + + Returns: + Calculated validation iter_num + """ + try: + # Get first validation volume size + img_path = Path(val_data_dicts[0]["image"]) + + # Load volume to get shape + if img_path.suffix in [".nii", ".gz"]: + # NIfTI file + import nibabel as nib + + vol = nib.load(str(img_path)) + vol_shape = vol.shape + elif img_path.suffix in [".h5", ".hdf5"]: + # HDF5 file + import h5py + + with h5py.File(img_path, "r") as f: + vol_shape = f[list(f.keys())[0]].shape + elif img_path.suffix in [".tif", ".tiff"]: + # TIFF file + import tifffile + + vol = tifffile.imread(img_path) + vol_shape = vol.shape + elif fallback_volume_shape is not None: + vol_shape = fallback_volume_shape + else: + # Unknown format, use default + print( + f" ⚠️ Unknown file format {img_path.suffix}, " + f"using default val_iter_num={default_iter_num}" + ) + return default_iter_num + + # Handle channel dimension if present + if len(vol_shape) == 4: + vol_shape = vol_shape[1:] # Remove channel dim: (C, D, H, W) -> (D, H, W) + + # Calculate number of possible patches (with 50% overlap) + stride = tuple(p // 2 for p in patch_size) # 50% overlap + num_patches_per_dim = [ + max(1, (vol_shape[i] - patch_size[i]) // stride[i] + 1) for i in range(3) + ] + total_possible_patches = ( + num_patches_per_dim[0] * num_patches_per_dim[1] * num_patches_per_dim[2] + ) + + # Calculate validation iter_num as a fraction of possible patches + val_iter_num = int(total_possible_patches * 0.075) # 7.5% of possible patches + val_iter_num = max(min_iter, val_iter_num) + if max_iter is not None: + val_iter_num = min(max_iter, val_iter_num) + + print(f" Validation volume shape: {vol_shape}") + print(f" Patch size: {patch_size}") + print(f" Stride (50% overlap): {stride}") + print(f" Possible patches per dim: {num_patches_per_dim}") + print(f" Total possible patches: {total_possible_patches}") + print(f" Using 7.5% of patches: {val_iter_num}") + + return val_iter_num + + except Exception as e: + if not return_default_on_error: + raise + print(f" ⚠️ Error calculating validation iter_num: {e}") + print(f" ℹ️ Using default val_iter_num={default_iter_num}") + return default_iter_num + + +def create_datamodule( + cfg: Config, mode: str = "train", fast_dev_run: bool = False +) -> ConnectomicsDataModule: + """ + Create Lightning DataModule from config. + + Args: + cfg: Hydra Config object + mode: 'train', 'test', or 'tune' + fast_dev_run: If True, config overrides have already been applied in setup_config() + + Returns: + ConnectomicsDataModule instance + """ + print("Creating datasets...") + + # Auto-download tutorial data if missing + if mode == "train" and cfg.data.train_image: + from pathlib import Path as PathLib + + # Check if data exists (support glob patterns and lists) + data_exists = False + + # Handle list of files + if isinstance(cfg.data.train_image, list): + # Check if at least one file in the list exists + data_exists = any(PathLib(img).exists() for img in cfg.data.train_image) + # Handle glob pattern + elif "*" in cfg.data.train_image or "?" in cfg.data.train_image: + # Glob pattern - check if any files match + matched_files = glob(cfg.data.train_image) + data_exists = len(matched_files) > 0 + # Handle single file path + else: + data_exists = PathLib(cfg.data.train_image).exists() + + if not data_exists: + print(f"\n⚠️ Training data not found: {cfg.data.train_image}") + + # Try to infer dataset name from path + from ...utils.download import DATASETS, download_dataset + + path_str = str(cfg.data.train_image).lower() + dataset_name = None + for name in DATASETS.keys(): + if name in path_str and not name.endswith("++"): # Skip aliases + dataset_name = name + break + + if dataset_name: + print(f"💡 Attempting to auto-download '{dataset_name}' dataset...") + print(" (You can disable auto-download by manually downloading data)") + + # Prompt user + try: + size_mb = DATASETS[dataset_name]["size_mb"] + prompt = f" Download {dataset_name} dataset (~{size_mb} MB)? [Y/n]: " + response = input(prompt).strip().lower() + if response in ["", "y", "yes"]: + if download_dataset(dataset_name, base_dir=PathLib.cwd()): + print("✅ Data downloaded successfully!") + else: + print("❌ Download failed. Please download manually:") + print(f" wget {DATASETS[dataset_name]['url']}") + raise FileNotFoundError( + f"Training data not found: {cfg.data.train_image}" + ) + else: + print("❌ Download cancelled. Please download manually.") + raise FileNotFoundError(f"Training data not found: {cfg.data.train_image}") + except KeyboardInterrupt: + print("\n❌ Download cancelled by user") + raise FileNotFoundError(f"Training data not found: {cfg.data.train_image}") + else: + print("💡 Available datasets:") + from ...utils.download import list_datasets + + list_datasets() + raise FileNotFoundError(f"Training data not found: {cfg.data.train_image}") + + # Check dataset type early + dataset_type = getattr(cfg.data, "dataset_type", None) + + # Build transforms + train_transforms = build_train_transforms(cfg) + val_transforms = build_val_transforms(cfg) + test_transforms = build_test_transforms(cfg) if mode in ["test", "tune"] else val_transforms + + print(f" Train transforms: {len(train_transforms.transforms)} steps") + print(f" Val transforms: {len(val_transforms.transforms)} steps") + if mode in ["test", "tune"]: + print( + f" Test transforms: {len(test_transforms.transforms)} steps (no sliding-window crop)" + ) + + # For test/tune modes, skip training data setup entirely + if mode in ["test", "tune"]: + train_data_dicts = [] + val_data_dicts = None + # Check if automatic train/val split is enabled + elif cfg.data.split_enabled and not cfg.data.val_image: + print("🔀 Using automatic train/val split (DeepEM-style)") + # Load full volume + import h5py + import tifffile + + from ...data.utils.split import split_volume_train_val + + train_path = Path(cfg.data.train_image) + if train_path.suffix in [".h5", ".hdf5"]: + with h5py.File(train_path, "r") as f: + volume_shape = f[list(f.keys())[0]].shape + elif train_path.suffix in [".tif", ".tiff"]: + volume = tifffile.imread(train_path) + volume_shape = volume.shape + else: + raise ValueError(f"Unsupported file format: {train_path.suffix}") + + print(f" Volume shape: {volume_shape}") + + # Calculate split ranges + train_ratio = cfg.data.split_train_range[1] - cfg.data.split_train_range[0] + + train_slices, val_slices = split_volume_train_val( + volume_shape=volume_shape, + train_ratio=train_ratio, + axis=cfg.data.split_axis, + ) + + # Calculate train and val regions + axis = cfg.data.split_axis + train_start = int(volume_shape[axis] * cfg.data.split_train_range[0]) + train_end = int(volume_shape[axis] * cfg.data.split_train_range[1]) + val_start = int(volume_shape[axis] * cfg.data.split_val_range[0]) + val_end = int(volume_shape[axis] * cfg.data.split_val_range[1]) + + print(f" Split axis: {axis} ({'Z' if axis == 0 else 'Y' if axis == 1 else 'X'})") + print(f" Train region: [{train_start}:{train_end}] ({train_end - train_start} slices)") + print(f" Val region: [{val_start}:{val_end}] ({val_end - val_start} slices)") + + if cfg.data.split_pad_val: + target_size = tuple(cfg.data.patch_size) + print(f" Val padding enabled: target size = {target_size}") + + # Create data dictionaries with split info + train_data_dicts = create_data_dicts_from_paths( + image_paths=[cfg.data.train_image], + label_paths=[cfg.data.train_label] if cfg.data.train_label else None, + ) + + # Add split metadata to train dict + train_data_dicts[0]["split_slices"] = train_slices + train_data_dicts[0]["split_mode"] = "train" + + # Create validation data dicts using same volume + val_data_dicts = create_data_dicts_from_paths( + image_paths=[cfg.data.train_image], + label_paths=[cfg.data.train_label] if cfg.data.train_label else None, + ) + + # Add split metadata to val dict + val_data_dicts[0]["split_slices"] = val_slices + val_data_dicts[0]["split_mode"] = "val" + val_data_dicts[0]["split_pad"] = cfg.data.split_pad_val + val_data_dicts[0]["split_pad_mode"] = cfg.data.split_pad_mode + if cfg.data.split_pad_val: + val_data_dicts[0]["split_pad_size"] = tuple(cfg.data.patch_size) + + else: + # Check dataset type to determine how to load data + if dataset_type == "filename": + # Check if train_json is empty or doesn't exist + train_json_empty = False + if cfg.data.train_json is None or cfg.data.train_json == "": + train_json_empty = True + else: + try: + import json + + json_path = Path(cfg.data.train_json) + if not json_path.exists(): + train_json_empty = True + else: + # Check if JSON file is empty or has no images + with open(json_path, "r") as f: + json_data = json.load(f) + image_files = json_data.get(cfg.data.train_image_key, []) + if not image_files: + train_json_empty = True + except (FileNotFoundError, json.JSONDecodeError, KeyError): + train_json_empty = True + + if train_json_empty: + # Fallback to volume-based dataset when train_json is empty + print(" ⚠️ Train JSON is empty or invalid, falling back to volume-based dataset") + print(f" Train JSON: {cfg.data.train_json}") + dataset_type = None # Switch to volume-based + else: + # Filename-based dataset: uses JSON file lists + print(" Using filename-based dataset") + print(f" Train JSON: {cfg.data.train_json}") + print(f" Image key: {cfg.data.train_image_key}") + print(f" Label key: {cfg.data.train_label_key}") + + # For filename dataset, we'll create data dicts later in the DataModule + # Here we just need placeholder dicts + train_data_dicts = [{"dataset_type": "filename"}] + val_data_dicts = None # Handled by train_val_split in DataModule + + if dataset_type != "filename": + # Standard mode: separate train and val files (supports glob patterns) + if cfg.data.train_image is None: + raise ValueError( + "For volume-based datasets, data.train_image must be specified.\n" + "Either set data.train_image or use data.dataset_type='filename' with " + "data.train_json" + ) + + train_image_paths = expand_file_paths(cfg.data.train_image) + train_label_paths = ( + expand_file_paths(cfg.data.train_label) if cfg.data.train_label else None + ) + train_mask_paths = ( + expand_file_paths(cfg.data.train_mask) if cfg.data.train_mask else None + ) + + print(f" Training volumes: {len(train_image_paths)} files") + if len(train_image_paths) <= 5: + for path in train_image_paths: + print(f" - {path}") + else: + print(f" - {train_image_paths[0]}") + print(f" - ... ({len(train_image_paths) - 2} more files)") + print(f" - {train_image_paths[-1]}") + + if train_mask_paths: + print(f" Training masks: {len(train_mask_paths)} files") + + train_data_dicts = create_data_dicts_from_paths( + image_paths=train_image_paths, + label_paths=train_label_paths, + mask_paths=train_mask_paths, + ) + + val_data_dicts = None + if cfg.data.val_image: + val_image_paths = expand_file_paths(cfg.data.val_image) + val_label_paths = ( + expand_file_paths(cfg.data.val_label) if cfg.data.val_label else None + ) + val_mask_paths = expand_file_paths(cfg.data.val_mask) if cfg.data.val_mask else None + + print(f" Validation volumes: {len(val_image_paths)} files") + if val_mask_paths: + print(f" Validation masks: {len(val_mask_paths)} files") + + val_data_dicts = create_data_dicts_from_paths( + image_paths=val_image_paths, + label_paths=val_label_paths, + mask_paths=val_mask_paths, + ) + + # Create test data dicts if in test or tune mode + test_data_dicts = None + if mode == "test": + if ( + not hasattr(cfg, "test") + or cfg.test is None + or not hasattr(cfg.test, "data") + or not cfg.test.data.test_image + ): + test_image_val = ( + cfg.test.data.test_image + if hasattr(cfg, "test") and cfg.test and hasattr(cfg.test, "data") + else "N/A" + ) + raise ValueError( + f"Test mode requires test.data.test_image to be set in config.\n" + f"Current config has: test.data.test_image = {test_image_val}" + ) + print(f" 🧪 Creating test dataset from: {cfg.test.data.test_image}") + + # Expand glob patterns for test data (same as train data) + test_image_paths = expand_file_paths(cfg.test.data.test_image) + test_label_paths = ( + expand_file_paths(cfg.test.data.test_label) if cfg.test.data.test_label else None + ) + test_mask_paths = ( + expand_file_paths(cfg.test.data.test_mask) + if hasattr(cfg.test.data, "test_mask") and cfg.test.data.test_mask + else None + ) + elif mode == "tune": + # For tune mode, read from cfg.tune.data + if ( + not hasattr(cfg, "tune") + or cfg.tune is None + or not hasattr(cfg.tune, "data") + or not cfg.tune.data.tune_image + ): + tune_image_val = ( + cfg.tune.data.tune_image + if hasattr(cfg, "tune") and cfg.tune and hasattr(cfg.tune, "data") + else "N/A" + ) + raise ValueError( + f"Tune mode requires tune.data.tune_image to be set in config.\n" + f"Current config has tune.data.tune_image: {tune_image_val}" + ) + + print(f" 🎯 Creating tune dataset from: {cfg.tune.data.tune_image}") + + # Expand glob patterns for tune data + test_image_paths = expand_file_paths(cfg.tune.data.tune_image) + test_label_paths = ( + expand_file_paths(cfg.tune.data.tune_label) if cfg.tune.data.tune_label else None + ) + test_mask_paths = ( + expand_file_paths(cfg.tune.data.tune_mask) if cfg.tune.data.tune_mask else None + ) + + # Common printing and data dict creation for test and tune modes + if mode in ["test", "tune"]: + mode_label = "Test" if mode == "test" else "Tune" + print(f" {mode_label} volumes: {len(test_image_paths)} files") + if len(test_image_paths) <= 5: + for path in test_image_paths: + print(f" - {path}") + else: + print(f" - {test_image_paths[0]}") + print(f" - ... ({len(test_image_paths) - 2} more files)") + print(f" - {test_image_paths[-1]}") + + if test_mask_paths: + print(f" {mode_label} masks: {len(test_mask_paths)} files") + + test_data_dicts = create_data_dicts_from_paths( + image_paths=test_image_paths, + label_paths=test_label_paths, + mask_paths=test_mask_paths, + ) + print(f" {mode_label} dataset size: {len(test_data_dicts)}") + + if mode == "train": + print(f" Train dataset size: {len(train_data_dicts)}") + if val_data_dicts: + print(f" Val dataset size: {len(val_data_dicts)}") + + # Auto-compute iter_num from volume size if not specified (only for training) + iter_num = None + if mode == "train": + iter_num = cfg.data.iter_num_per_epoch + if iter_num == -1 and dataset_type != "filename": + # For filename datasets, iter_num is determined by the number of files + print("📊 Auto-computing iter_num from volume size...") + import h5py + import tifffile + + from ...data.utils import compute_total_samples + + # Get volume sizes + volume_sizes = [] + for data_dict in train_data_dicts: + img_path = Path(str(data_dict["image"])) + if img_path.suffix in [".h5", ".hdf5"]: + with h5py.File(img_path, "r") as f: + vol_shape = f[list(f.keys())[0]].shape + elif img_path.suffix in [".tif", ".tiff"]: + vol = tifffile.imread(img_path) + vol_shape = vol.shape + else: + raise ValueError(f"Unsupported file format: {img_path.suffix}") + + # Handle both (z, y, x) and (c, z, y, x) + if len(vol_shape) == 4: + vol_shape = vol_shape[1:] # Skip channel dim + volume_sizes.append(vol_shape) + + # Compute total possible samples + total_samples, samples_per_vol = compute_total_samples( + volume_sizes=volume_sizes, + patch_size=tuple(cfg.data.patch_size), + stride=tuple(cfg.data.stride), + ) + + iter_num = total_samples + print(f" Volume sizes: {volume_sizes}") + print(f" Patch size: {cfg.data.patch_size}") + print(f" Stride: {cfg.data.stride}") + print(f" Samples per volume: {samples_per_vol}") + print(f" ✅ Total possible samples (iter_num): {iter_num:,}") + print(f" ✅ Batches per epoch: {iter_num // cfg.system.training.batch_size:,}") + elif iter_num == -1 and dataset_type == "filename": + # For filename datasets, iter_num will be determined by dataset length + print(" Filename dataset: iter_num will be determined by number of files in JSON") + + # Create DataModule + print("Creating data loaders...") + + # For test/tune modes, disable iter_num (process full volumes once) + if mode in ["test", "tune"]: + iter_num_for_dataset: int | None = -1 # Process full volumes without random sampling + else: + iter_num_for_dataset = iter_num + + # Select appropriate batch_size and num_workers based on mode + # For test/tune modes, use system.inference settings instead of system.training + if mode in ["test", "tune", "tune-test"]: + batch_size = cfg.system.inference.batch_size + num_workers = cfg.system.inference.num_workers + print(f" Using inference settings: batch_size={batch_size}, num_workers={num_workers}") + else: + batch_size = cfg.system.training.batch_size + num_workers = cfg.system.training.num_workers + print(f" Using training settings: batch_size={batch_size}, num_workers={num_workers}") + + # Use optimized pre-loaded cache when iter_num > 0 (only for training mode and volume datasets) + use_preloaded = ( + cfg.data.use_preloaded_cache + and iter_num is not None + and iter_num > 0 + and mode == "train" + and dataset_type != "filename" + ) + + if use_preloaded: + print(" ⚡ Using pre-loaded volume cache (loads once, crops in memory)") + import pytorch_lightning as pl + from torch.utils.data import DataLoader + + from ...data.dataset.dataset_volume_cached import CachedVolumeDataset + + # Build transforms without loading/cropping (handled by dataset) + augment_only_transforms = build_train_transforms(cfg, skip_loading=True) + + # Get padding parameters from config + pad_size = getattr(cfg.data.image_transform, "pad_size", None) + pad_mode = getattr(cfg.data.image_transform, "pad_mode", "reflect") + + # Create optimized cached datasets + train_dataset = CachedVolumeDataset( + image_paths=[d["image"] for d in train_data_dicts], + label_paths=[d.get("label") for d in train_data_dicts], + patch_size=tuple(cfg.data.patch_size), + iter_num=iter_num, + transforms=augment_only_transforms, + mode="train", + pad_size=tuple(pad_size) if pad_size else None, + pad_mode=pad_mode, + ) + + # Use fewer workers since we're loading from memory + preloaded_num_workers = min(num_workers, 2) + print(f" Using {preloaded_num_workers} workers (in-memory operations are fast)") + + # Create simple dataloader + train_loader = DataLoader( + train_dataset, + batch_size=batch_size, + shuffle=False, # Already random + num_workers=preloaded_num_workers, + pin_memory=cfg.data.pin_memory, + persistent_workers=preloaded_num_workers > 0, + ) + + # Create validation dataset and loader if validation data exists + val_loader = None + if val_data_dicts and len(val_data_dicts) > 0: + print(" Creating validation dataset with pre-loaded cache...") + + # Build validation transforms (no augmentation, only normalization) + val_only_transforms = build_val_transforms(cfg, skip_loading=True) + + # Get validation iter_num (auto-calculate if not specified) + val_iter_num = ( + cfg.data.val_iter_num + if hasattr(cfg.data, "val_iter_num") and cfg.data.val_iter_num + else None + ) + + if val_iter_num is None: + # Auto-calculate validation iter_num from volume size + print(" 📊 Auto-calculating validation iter_num from volume size...") + val_iter_num = _calculate_validation_iter_num( + val_data_dicts=val_data_dicts, + patch_size=tuple(cfg.data.patch_size), + min_iter=1, + max_iter=None, + default_iter_num=100, + fallback_volume_shape=(100, 4096, 4096), + return_default_on_error=False, + ) + print(f" ✅ Validation iter_num: {val_iter_num} (auto-calculated)") + + # Create validation dataset + val_dataset = CachedVolumeDataset( + image_paths=[d["image"] for d in val_data_dicts], + label_paths=[d.get("label") for d in val_data_dicts], + patch_size=tuple(cfg.data.patch_size), + iter_num=val_iter_num, + transforms=val_only_transforms, + mode="val", + pad_size=tuple(pad_size) if pad_size else None, + pad_mode=pad_mode, + ) + + # Create validation dataloader + val_loader = DataLoader( + val_dataset, + batch_size=batch_size, + shuffle=False, + num_workers=preloaded_num_workers, + pin_memory=cfg.data.pin_memory, + persistent_workers=preloaded_num_workers > 0, + ) + print(f" ✅ Validation dataloader created with {val_iter_num} iterations") + + # Create data module wrapper that inherits from LightningDataModule + class SimpleDataModule(pl.LightningDataModule): + def __init__(self, train_loader, val_loader=None): + super().__init__() + self.train_loader = train_loader + self._val_loader = val_loader + + def train_dataloader(self): + return self.train_loader + + def val_dataloader(self): + if self._val_loader is not None: + return self._val_loader + return [] + + def test_dataloader(self): + # For test mode, return empty list (user should use standard datamodule) + return [] + + def setup(self, stage=None): + pass + + datamodule = SimpleDataModule(train_loader, val_loader) + elif dataset_type == "filename": + # Filename-based dataset using JSON file lists + print(" Creating filename-based datamodule...") + import pytorch_lightning as pl + from torch.utils.data import DataLoader + + from ...data.dataset.dataset_filename import create_filename_datasets + + # Create train and val datasets from JSON + train_dataset, val_dataset = create_filename_datasets( + json_path=cfg.data.train_json, + train_transforms=train_transforms, + val_transforms=val_transforms, + train_val_split=( + cfg.data.train_val_split if hasattr(cfg.data, "train_val_split") else 0.9 + ), + random_seed=cfg.system.seed if hasattr(cfg.system, "seed") else 42, + images_key=cfg.data.train_image_key, + labels_key=cfg.data.train_label_key, + use_labels=True, + ) + + print(f" Train dataset size: {len(train_dataset)}") + print(f" Val dataset size: {len(val_dataset)}") + + # Create simple datamodule wrapper + class FilenameDataModule(pl.LightningDataModule): + def __init__( + self, train_ds, val_ds, batch_size, num_workers, pin_memory, persistent_workers + ): + super().__init__() + self.train_ds = train_ds + self.val_ds = val_ds + self.batch_size = batch_size + self.num_workers = num_workers + self.pin_memory = pin_memory + self.persistent_workers = persistent_workers + + def train_dataloader(self): + return DataLoader( + self.train_ds, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers and self.num_workers > 0, + ) + + def val_dataloader(self): + if self.val_ds is None or len(self.val_ds) == 0: + return [] + return DataLoader( + self.val_ds, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers and self.num_workers > 0, + ) + + def test_dataloader(self): + return [] + + def setup(self, stage=None): + pass + + datamodule = FilenameDataModule( + train_ds=train_dataset, + val_ds=val_dataset, + batch_size=batch_size, + num_workers=num_workers, + pin_memory=cfg.data.pin_memory, + persistent_workers=cfg.data.persistent_workers, + ) + else: + # Standard data module + # Disable caching for test/tune modes to avoid issues with partial cache returning 0 length + use_cache = cfg.data.use_cache and mode == "train" + + if mode in ["test", "tune"] and cfg.data.use_cache: + print(" ⚠️ Caching disabled for test/tune mode (incompatible with partial cache)") + + # Note: transpose_axes handled in transform builders (build_train/val/test_transforms) + # They embed the transpose in LoadVolumed, so no need to pass it here + + # Get validation iter_num (separate from training iter_num) + val_iter_num = getattr(cfg.data, "val_iter_num", None) + if val_iter_num is None and val_data_dicts: + # Auto-calculate validation iter_num based on volume size and patch size + print(" 📊 Auto-calculating validation iter_num from volume size...") + val_iter_num = _calculate_validation_iter_num( + val_data_dicts=val_data_dicts, + patch_size=tuple(cfg.data.patch_size), + min_iter=50, + max_iter=200, + ) + print(f" ✅ Validation iter_num: {val_iter_num} (auto-calculated)") + + datamodule = ConnectomicsDataModule( + train_data_dicts=train_data_dicts, + val_data_dicts=val_data_dicts, + test_data_dicts=test_data_dicts, + transforms={ + "train": train_transforms, + "val": val_transforms, + "test": test_transforms, + }, + dataset_type="cached" if use_cache else "standard", + batch_size=batch_size, + num_workers=num_workers, + pin_memory=cfg.data.pin_memory, + persistent_workers=cfg.data.persistent_workers, + cache_rate=cfg.data.cache_rate if use_cache else 0.0, + iter_num=iter_num_for_dataset, + val_iter_num=val_iter_num, + seed=cfg.system.seed, # [FIX 1] Pass seed for validation reseeding + sample_size=tuple(cfg.data.patch_size), + do_2d=cfg.data.do_2d, + ) + # Setup datasets based on mode + if mode == "train": + datamodule.setup(stage="fit") + elif mode in ["test", "tune"]: + datamodule.setup(stage="test") + + # Print dataset info based on mode + if mode == "train": + print(f" Train batches: {len(datamodule.train_dataloader())}") + if val_data_dicts: + print(f" Val batches: {len(datamodule.val_dataloader())}") + elif mode in ["test", "tune"]: + print(f" Test batches: {len(datamodule.test_dataloader())}") + + return datamodule + + +__all__ = [ + "create_datamodule", + "_calculate_validation_iter_num", +] diff --git a/connectomics/training/lit/path_utils.py b/connectomics/training/lit/path_utils.py new file mode 100644 index 00000000..fea16aa9 --- /dev/null +++ b/connectomics/training/lit/path_utils.py @@ -0,0 +1,28 @@ +"""Path-related helpers for Lightning training modules.""" + +from __future__ import annotations + +from glob import glob +from typing import List + + +def expand_file_paths(path_or_pattern) -> List[str]: + """ + Expand file path inputs into a list of paths. + + Args: + path_or_pattern: Single file path, glob pattern, or list of paths/patterns. + + Returns: + List of expanded file paths, sorted for glob inputs. + """ + if isinstance(path_or_pattern, list): + return path_or_pattern + + if "*" in path_or_pattern or "?" in path_or_pattern: + paths = sorted(glob(path_or_pattern)) + if not paths: + raise FileNotFoundError(f"No files found matching pattern: {path_or_pattern}") + return paths + + return [path_or_pattern] diff --git a/connectomics/training/lit/runtime.py b/connectomics/training/lit/runtime.py new file mode 100644 index 00000000..513c96b8 --- /dev/null +++ b/connectomics/training/lit/runtime.py @@ -0,0 +1,193 @@ +"""Runtime and checkpoint orchestration helpers for Lightning training.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional + + +def setup_run_directory(mode: str, cfg, checkpoint_dirpath: str): + """ + Setup run directory with timestamp for training mode. + Handles DDP subprocess coordination via timestamp files. + + Args: + mode: 'train', 'test', 'tune', or 'tune-test' + cfg: Config object (will be modified in-place for training mode) + checkpoint_dirpath: Path to checkpoint/output directory from config + + Returns: + Path: Run directory (timestamped for train, created for tune modes, dummy for test) + """ + import os + from datetime import datetime + + from ...config import save_config + + checkpoint_dir = Path(checkpoint_dirpath) + checkpoint_subdir = checkpoint_dir.name or "checkpoints" + output_base = checkpoint_dir.parent + + if mode == "train": + # Check if this is a DDP re-launch (LOCAL_RANK is set by PyTorch Lightning) + is_ddp_subprocess = "LOCAL_RANK" in os.environ + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + timestamp_file = output_base / ".latest_timestamp" + + if not is_ddp_subprocess: + # First invocation (main process) - create new timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + run_dir = output_base / timestamp + checkpoint_path = run_dir / checkpoint_subdir + cfg.monitor.checkpoint.dirpath = str(checkpoint_path) + + checkpoint_path.mkdir(parents=True, exist_ok=True) + print(f"📁 Run directory: {run_dir}") + + # Save config to run directory + config_save_path = run_dir / "config.yaml" + save_config(cfg, config_save_path) + print(f"💾 Config saved to: {config_save_path}") + + # Save timestamp for DDP subprocesses to read + output_base.mkdir(parents=True, exist_ok=True) + timestamp_file.write_text(timestamp) + else: + # DDP subprocess - read existing timestamp + import time + + max_wait = 30 # Maximum 30 seconds + waited = 0.0 + while not timestamp_file.exists() and waited < max_wait: + time.sleep(0.1) + waited += 0.1 + + if timestamp_file.exists(): + timestamp = timestamp_file.read_text().strip() + run_dir = output_base / timestamp + checkpoint_path = run_dir / checkpoint_subdir + cfg.monitor.checkpoint.dirpath = str(checkpoint_path) + print(f"📁 [DDP Rank {local_rank}] Using run directory: {run_dir}") + else: + raise RuntimeError( + f"DDP subprocess (LOCAL_RANK={local_rank}) timed out waiting for timestamp file" + ) + elif mode in ["tune", "tune-test"]: + # For tune modes, create the directory for Optuna outputs + run_dir = Path(checkpoint_dirpath) + run_dir.mkdir(parents=True, exist_ok=True) + print(f"📁 Tuning output directory: {run_dir}") + elif mode == "test": + # For test mode, create the results directory + run_dir = Path(checkpoint_dirpath) + run_dir.mkdir(parents=True, exist_ok=True) + print(f"📁 Test output directory: {run_dir}") + else: + # Fallback for unknown modes + run_dir = output_base / f"{mode}_run" + print(f"📝 Running in {mode} mode") + + return run_dir + + +def cleanup_run_directory(output_base: Path): + """ + Clean up timestamp file after training completes. + Only runs in main process (not DDP subprocesses). + + Args: + output_base: Base output directory containing timestamp file + """ + import os + + is_ddp_subprocess = "LOCAL_RANK" in os.environ + if not is_ddp_subprocess: + timestamp_file = output_base / ".latest_timestamp" + if timestamp_file.exists(): + try: + timestamp_file.unlink() + except Exception: + pass # Ignore cleanup errors + + +def modify_checkpoint_state( + checkpoint_path: Optional[str], + run_dir: Path, + reset_optimizer: bool = False, + reset_scheduler: bool = False, + reset_epoch: bool = False, + reset_early_stopping: bool = False, +) -> Optional[str]: + """ + Modify checkpoint state for training resumption with selective resets. + + Args: + checkpoint_path: Path to checkpoint file to modify (None if no checkpoint) + run_dir: Run directory for saving modified checkpoint + reset_optimizer: If True, reset optimizer state + reset_scheduler: If True, reset scheduler state + reset_epoch: If True, reset epoch counter + reset_early_stopping: If True, reset early stopping patience + + Returns: + Path to modified checkpoint file, or original path if no modifications needed + """ + import torch + + # Early return if no checkpoint or no resets requested + if not checkpoint_path: + return None + + if not (reset_optimizer or reset_scheduler or reset_epoch or reset_early_stopping): + return checkpoint_path + + print("\n🔄 Modifying checkpoint state:") + if reset_optimizer: + print(" - Resetting optimizer state") + if reset_scheduler: + print(" - Resetting scheduler state") + if reset_epoch: + print(" - Resetting epoch counter") + if reset_early_stopping: + print(" - Resetting early stopping patience counter") + + # Load checkpoint (weights_only=False needed for PyTorch 2.6+ to load Lightning + # checkpoints with custom configs) + checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) + + # Reset optimizer state + if reset_optimizer and "optimizer_states" in checkpoint: + del checkpoint["optimizer_states"] + + # Reset scheduler state + if reset_scheduler and "lr_schedulers" in checkpoint: + del checkpoint["lr_schedulers"] + + # Reset epoch counter + if reset_epoch: + if "epoch" in checkpoint: + checkpoint["epoch"] = 0 + if "global_step" in checkpoint: + checkpoint["global_step"] = 0 + + # Reset early stopping state + if reset_early_stopping and "callbacks" in checkpoint: + for callback_state in checkpoint["callbacks"].values(): + if "wait_count" in callback_state: + callback_state["wait_count"] = 0 + if "best_score" in callback_state: + callback_state["best_score"] = None + + # Save modified checkpoint to temporary file + temp_ckpt_path = run_dir / "temp_modified_checkpoint.ckpt" + torch.save(checkpoint, temp_ckpt_path) + print(f" ✅ Modified checkpoint saved to: {temp_ckpt_path}") + + return str(temp_ckpt_path) + + +__all__ = [ + "setup_run_directory", + "cleanup_run_directory", + "modify_checkpoint_state", +] diff --git a/connectomics/training/lit/utils.py b/connectomics/training/lit/utils.py index 20c9c8a2..ff924284 100644 --- a/connectomics/training/lit/utils.py +++ b/connectomics/training/lit/utils.py @@ -9,9 +9,9 @@ """ from __future__ import annotations + import argparse import re -from glob import glob from pathlib import Path from typing import List, Optional @@ -20,10 +20,11 @@ from ...config import ( Config, load_config, + resolve_data_paths, update_from_cli, validate_config, - resolve_data_paths, ) +from .path_utils import expand_file_paths as _expand_file_paths def parse_args(): @@ -195,8 +196,10 @@ def setup_config(args) -> Config: print("🔧 Fast-dev-run mode: Overriding config for debugging") print(f" - num_gpus: {cfg.system.training.num_gpus} → 1") print(f" - num_cpus: {cfg.system.training.num_cpus} → 1") - print(f" - num_workers: {cfg.system.training.num_workers} → 0 " - "(avoid multiprocessing in debug mode)") + print( + f" - num_workers: {cfg.system.training.num_workers} → 0 " + "(avoid multiprocessing in debug mode)" + ) print( f" - batch_size: Controlled by PyTorch Lightning (--fast-dev-run={args.fast_dev_run})" ) @@ -269,7 +272,7 @@ def setup_config(args) -> Config: def expand_file_paths(path_or_pattern) -> List[str]: """ - Expand glob patterns to list of file paths. + Backward-compatible wrapper for shared path expansion helper. Args: path_or_pattern: Single file path, glob pattern, or list of paths/patterns @@ -277,20 +280,7 @@ def expand_file_paths(path_or_pattern) -> List[str]: Returns: List of expanded file paths, sorted alphabetically """ - # If already a list, return it (may have been expanded by resolve_data_paths) - if isinstance(path_or_pattern, list): - return path_or_pattern - - # Check if pattern contains wildcards - if "*" in path_or_pattern or "?" in path_or_pattern: - # Expand glob pattern - paths = sorted(glob(path_or_pattern)) - if not paths: - raise FileNotFoundError(f"No files found matching pattern: {path_or_pattern}") - return paths - else: - # Single file path - return [path_or_pattern] + return _expand_file_paths(path_or_pattern) def extract_best_score_from_checkpoint(ckpt_path: str, monitor_metric: str) -> Optional[float]: diff --git a/tests/integration/test_config_integration.py b/tests/integration/test_config_integration.py index 56ed031b..1867960f 100644 --- a/tests/integration/test_config_integration.py +++ b/tests/integration/test_config_integration.py @@ -2,33 +2,33 @@ """Simple integration test for config system.""" import pytest -from connectomics.config import load_config, Config, from_dict -from connectomics.training.lit import ConnectomicsModule, create_trainer + +from connectomics.config import Config, from_dict, load_config +from connectomics.training.lit import ConnectomicsModule, create_datamodule, create_trainer +from connectomics.training.lit.data_factory import create_datamodule as create_datamodule_impl def test_config_creation(): """Test basic config creation.""" cfg = Config() assert cfg is not None - assert hasattr(cfg, 'system') - assert hasattr(cfg, 'model') + assert hasattr(cfg, "system") + assert hasattr(cfg, "model") def test_config_from_dict(): """Test creating config from dict.""" - cfg = from_dict({ - 'system': {'training': {'num_gpus': 0}}, - 'model': {'architecture': 'monai_basic_unet3d'} - }) + cfg = from_dict( + {"system": {"training": {"num_gpus": 0}}, "model": {"architecture": "monai_basic_unet3d"}} + ) assert cfg.system.training.num_gpus == 0 - assert cfg.model.architecture == 'monai_basic_unet3d' + assert cfg.model.architecture == "monai_basic_unet3d" def test_config_from_yaml(tmp_path): """Test loading config from a YAML file.""" config_path = tmp_path / "sample.yaml" - config_path.write_text( - """ + config_path.write_text(""" experiment_name: sample model: architecture: monai_basic_unet3d @@ -37,8 +37,7 @@ def test_config_from_yaml(tmp_path): system: training: num_gpus: 0 -""" - ) +""") cfg = load_config(config_path) assert cfg is not None @@ -48,37 +47,38 @@ def test_config_from_yaml(tmp_path): def test_lightning_module_creation(): """Test creating Lightning module.""" - cfg = from_dict({ - 'system': {'training': {'num_gpus': 0}}, - 'model': { - 'architecture': 'monai_basic_unet3d', - 'in_channels': 1, - 'out_channels': 2, - 'filters': [8, 16], - 'loss_functions': ['DiceLoss'], - 'loss_weights': [1.0] - }, - 'optimization': { - 'optimizer': {'name': 'AdamW', 'lr': 1e-4}, - 'max_epochs': 1 + cfg = from_dict( + { + "system": {"training": {"num_gpus": 0}}, + "model": { + "architecture": "monai_basic_unet3d", + "in_channels": 1, + "out_channels": 2, + "filters": [8, 16], + "loss_functions": ["DiceLoss"], + "loss_weights": [1.0], + }, + "optimization": {"optimizer": {"name": "AdamW", "lr": 1e-4}, "max_epochs": 1}, } - }) - + ) + module = ConnectomicsModule(cfg) assert module is not None def test_trainer_creation(tmp_path): """Test creating trainer.""" - cfg = from_dict({ - 'system': {'training': {'num_gpus': 0}}, - 'optimization': {'max_epochs': 1} - }) - + cfg = from_dict({"system": {"training": {"num_gpus": 0}}, "optimization": {"max_epochs": 1}}) + trainer = create_trainer(cfg, run_dir=tmp_path) assert trainer is not None assert trainer.max_epochs == 1 -if __name__ == '__main__': - pytest.main([__file__, '-v']) +def test_datamodule_factory_export_consistency(): + """Test that lit package exposes the extracted data factory entrypoint.""" + assert create_datamodule is create_datamodule_impl + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unit/test_em_augmentations.py b/tests/unit/test_em_augmentations.py index 63c4894e..a117d9e0 100644 --- a/tests/unit/test_em_augmentations.py +++ b/tests/unit/test_em_augmentations.py @@ -131,6 +131,8 @@ def test_respects_boundaries(self, sample_data_dict): num_sections=1, ) + original_first = sample_data_dict["image"][0, 0, :, :].clone() + original_last = sample_data_dict["image"][0, -1, :, :].clone() original_depth = sample_data_dict["image"].shape[1] # Run multiple times @@ -138,12 +140,9 @@ def test_respects_boundaries(self, sample_data_dict): result = transform(sample_data_dict.copy()) # Shape should be preserved assert result["image"].shape[1] == original_depth - # First and last sections should not be zero (preserved) - first_sum = result["image"][0, 0, :, :].sum() - last_sum = result["image"][0, -1, :, :].sum() - # First and last should have non-zero values (preserved) - assert first_sum > 0 - assert last_sum > 0 + # First and last sections should be unchanged (preserved) + assert torch.equal(result["image"][0, 0, :, :], original_first) + assert torch.equal(result["image"][0, -1, :, :], original_last) def test_probability_control(self, sample_data_dict): """Test probability control.""" diff --git a/tests/unit/test_lit_utils.py b/tests/unit/test_lit_utils.py index 19dc88fc..73a1815a 100644 --- a/tests/unit/test_lit_utils.py +++ b/tests/unit/test_lit_utils.py @@ -1,7 +1,12 @@ import argparse from pathlib import Path +import pytest + from connectomics.config import Config, save_config +from connectomics.training.lit.config import expand_file_paths as config_expand_file_paths +from connectomics.training.lit.data_factory import _calculate_validation_iter_num +from connectomics.training.lit.path_utils import expand_file_paths as canonical_expand_file_paths from connectomics.training.lit.utils import ( expand_file_paths, extract_best_score_from_checkpoint, @@ -64,6 +69,40 @@ def test_expand_file_paths_handles_globs_and_lists(tmp_path): # Passing a list should be returned unchanged assert expand_file_paths([str(data_dir / "a.txt")]) == [str(data_dir / "a.txt")] + # Compatibility wrappers in config and utils should match canonical helper behavior + assert config_expand_file_paths(str(data_dir / "*.txt")) == expanded + assert canonical_expand_file_paths(str(data_dir / "*.txt")) == expanded + + with pytest.raises(FileNotFoundError): + canonical_expand_file_paths(str(data_dir / "*.missing")) + + +def test_calculate_validation_iter_num_unknown_suffix_defaults(): + val_data_dicts = [{"image": "dummy.unknown"}] + + assert ( + _calculate_validation_iter_num( + val_data_dicts=val_data_dicts, + patch_size=(16, 16, 16), + ) + == 100 + ) + + +def test_calculate_validation_iter_num_uses_fallback_shape_without_clamp(): + val_data_dicts = [{"image": "dummy.unknown"}] + + val_iter_num = _calculate_validation_iter_num( + val_data_dicts=val_data_dicts, + patch_size=(32, 32, 32), + min_iter=1, + max_iter=None, + fallback_volume_shape=(100, 4096, 4096), + return_default_on_error=False, + ) + + assert val_iter_num == 24384 + def test_extract_best_score_from_checkpoint(): score = extract_best_score_from_checkpoint( diff --git a/tests/unit/test_main_cli_contract.py b/tests/unit/test_main_cli_contract.py new file mode 100644 index 00000000..720c9124 --- /dev/null +++ b/tests/unit/test_main_cli_contract.py @@ -0,0 +1,53 @@ +import sys + +import pytest + +from connectomics.training.lit.utils import parse_args + + +def _parse_with_argv(monkeypatch, argv): + monkeypatch.setattr(sys, "argv", ["scripts/main.py", *argv]) + return parse_args() + + +@pytest.mark.parametrize("mode", ["train", "test", "tune", "tune-test"]) +def test_parse_args_accepts_documented_modes(monkeypatch, mode): + args = _parse_with_argv(monkeypatch, ["--mode", mode]) + assert args.mode == mode + + +def test_parse_args_rejects_infer_mode(monkeypatch): + with pytest.raises(SystemExit): + _parse_with_argv(monkeypatch, ["--mode", "infer"]) + + +def test_parse_args_fast_dev_run_default_and_explicit(monkeypatch): + args = _parse_with_argv(monkeypatch, []) + assert args.fast_dev_run == 0 + + args = _parse_with_argv(monkeypatch, ["--fast-dev-run"]) + assert args.fast_dev_run == 1 + + args = _parse_with_argv(monkeypatch, ["--fast-dev-run", "3"]) + assert args.fast_dev_run == 3 + + +def test_parse_args_preserves_overrides_passthrough(monkeypatch): + args = _parse_with_argv( + monkeypatch, + [ + "--config", + "tutorials/lucchi++.yaml", + "data.batch_size=8", + "optimization.max_epochs=3", + ], + ) + + assert args.config == "tutorials/lucchi++.yaml" + assert args.overrides == ["data.batch_size=8", "optimization.max_epochs=3"] + + +def test_parse_args_demo_mode_requires_no_config(monkeypatch): + args = _parse_with_argv(monkeypatch, ["--demo"]) + assert args.demo is True + assert args.config is None diff --git a/tests/unit/test_run_directory_contract.py b/tests/unit/test_run_directory_contract.py new file mode 100644 index 00000000..4fde02aa --- /dev/null +++ b/tests/unit/test_run_directory_contract.py @@ -0,0 +1,120 @@ +import re +from pathlib import Path + +import torch + +from connectomics.config import Config +from connectomics.training.lit.config import ( + cleanup_run_directory, + modify_checkpoint_state, + setup_run_directory, +) +from connectomics.training.lit.runtime import ( + modify_checkpoint_state as runtime_modify_checkpoint_state, +) + + +def test_setup_run_directory_train_creates_timestamped_layout(tmp_path): + cfg = Config() + checkpoint_dir = tmp_path / "outputs" / "exp" / "checkpoints" + + run_dir = setup_run_directory("train", cfg, str(checkpoint_dir)) + + assert run_dir.parent == checkpoint_dir.parent + assert re.fullmatch(r"\d{8}_\d{6}", run_dir.name) + assert (run_dir / "checkpoints").exists() + assert (run_dir / "config.yaml").exists() + + timestamp_file = checkpoint_dir.parent / ".latest_timestamp" + assert timestamp_file.exists() + assert timestamp_file.read_text().strip() == run_dir.name + assert Path(cfg.monitor.checkpoint.dirpath) == run_dir / "checkpoints" + + +def test_setup_run_directory_train_ddp_reuses_timestamp_file(tmp_path, monkeypatch): + cfg = Config() + checkpoint_dir = tmp_path / "outputs" / "exp" / "checkpoints" + output_base = checkpoint_dir.parent + output_base.mkdir(parents=True, exist_ok=True) + + timestamp = "20250208_112233" + (output_base / ".latest_timestamp").write_text(timestamp) + monkeypatch.setenv("LOCAL_RANK", "1") + + run_dir = setup_run_directory("train", cfg, str(checkpoint_dir)) + assert run_dir == output_base / timestamp + assert Path(cfg.monitor.checkpoint.dirpath) == run_dir / "checkpoints" + + +def test_setup_run_directory_non_train_modes_create_requested_directory(tmp_path): + for mode in ["test", "tune", "tune-test"]: + cfg = Config() + target = tmp_path / mode / "artifacts" + run_dir = setup_run_directory(mode, cfg, str(target)) + assert run_dir == target + assert target.exists() + + +def test_cleanup_run_directory_removes_timestamp_file(tmp_path): + output_base = tmp_path / "outputs" / "exp" + output_base.mkdir(parents=True, exist_ok=True) + timestamp_file = output_base / ".latest_timestamp" + timestamp_file.write_text("20250208_112233") + + cleanup_run_directory(output_base) + assert not timestamp_file.exists() + + +def test_modify_checkpoint_state_returns_original_when_no_resets_requested(tmp_path): + run_dir = tmp_path / "run" + run_dir.mkdir(parents=True, exist_ok=True) + checkpoint = tmp_path / "checkpoint.ckpt" + torch.save({"epoch": 5, "global_step": 42}, checkpoint) + + assert modify_checkpoint_state(None, run_dir) is None + assert modify_checkpoint_state(str(checkpoint), run_dir) == str(checkpoint) + assert runtime_modify_checkpoint_state(None, run_dir) is None + + +def test_modify_checkpoint_state_applies_selected_resets(tmp_path): + run_dir = tmp_path / "run" + run_dir.mkdir(parents=True, exist_ok=True) + checkpoint = tmp_path / "checkpoint.ckpt" + torch.save( + { + "epoch": 7, + "global_step": 128, + "optimizer_states": [{"state": 1}], + "lr_schedulers": [{"state": 2}], + "callbacks": { + "EarlyStopping": { + "wait_count": 3, + "best_score": torch.tensor(0.123), + } + }, + }, + checkpoint, + ) + + modified = modify_checkpoint_state( + str(checkpoint), + run_dir, + reset_optimizer=True, + reset_scheduler=True, + reset_epoch=True, + reset_early_stopping=True, + ) + + assert modified is not None + modified_path = Path(modified) + assert modified_path.exists() + assert modified_path.parent == run_dir + assert modified_path.name == "temp_modified_checkpoint.ckpt" + + loaded = torch.load(modified_path, map_location="cpu", weights_only=False) + assert "optimizer_states" not in loaded + assert "lr_schedulers" not in loaded + assert loaded["epoch"] == 0 + assert loaded["global_step"] == 0 + assert loaded["callbacks"]["EarlyStopping"]["wait_count"] == 0 + assert loaded["callbacks"]["EarlyStopping"]["best_score"] is None diff --git a/tutorials/cellmap_cos7.yaml b/tutorials/cellmap_cos7.yaml deleted file mode 100644 index 92cc9047..00000000 --- a/tutorials/cellmap_cos7.yaml +++ /dev/null @@ -1,182 +0,0 @@ -# CellMap Challenge - Phase 1: Simple Semantic Validation -# -# PHASE 1: Make sure simple one works -# - 5 semantic classes (nuc, mito, er, golgi, ves) -# - Multi-class binary mask output -# - No instance separation needed (semantic only) -# -# This config validates the semantic segmentation pipeline: -# - Standard resolution (8nm) for semantic context -# - Softmax activation for mutually exclusive classes -# - Direct output (no post-processing) -# -# Dataset: COS7 cells from CellMap challenge -# Classes: nuc, mito, er, golgi, ves (5 organelles) -# Resolution: 8nm isotropic -# Data loader: Uses cellmap-data package (official challenge library) -# -# Usage: -# python scripts/cellmap/train_cellmap.py --config tutorials/cellmap_cos7.yaml - -experiment_name: cellmap_cos7_mednext -description: CellMap COS7 multi-organelle segmentation with MedNeXt - -# System -system: - training: - num_gpus: 4 - num_cpus: 8 - num_workers: 2 - batch_size: 8 # Per GPU (effective = 32) - inference: - num_gpus: 1 - num_cpus: 1 - num_workers: 1 - batch_size: 1 - seed: 42 - -# Model Configuration -model: - architecture: mednext # MedNeXt (recommended for CellMap) - - # Input/output configuration - input_size: [128, 128, 128] # Patch size - output_size: [128, 128, 128] - in_channels: 1 # Grayscale EM - out_channels: 5 # 5 organelle classes - - # MedNeXt configuration - mednext_size: M # M (17.6M params) - good balance - mednext_kernel_size: 5 # 5x5x5 kernels - deep_supervision: true # Multi-scale loss (RECOMMENDED) - - # Loss configuration (multi-class segmentation) - loss_functions: [DiceCELoss] - loss_weights: [1.0] - loss_kwargs: - - {sigmoid: false, softmax: true, to_onehot_y: false, include_background: false} - -# Data - CellMap Challenge Dataset -data: - # Dataset type (custom for CellMap) - dataset_type: cellmap # Special marker for CellMap data - - # CellMap-specific configuration - cellmap: - # Data paths - data_root: /projects/weilab/dataset/cellmap - datasplit_path: tutorials/cellmap_cos7_datasplit.csv # Auto-generated next to YAML if missing - - # Classes to segment - classes: [nuc, mito, er, golgi, ves] - force_all_classes: both # Keep crops with all classes in both train/val - - # Patch configuration - input_array_info: - shape: [128, 128, 128] - scale: [8, 8, 8] # 8nm isotropic - target_array_info: - shape: [128, 128, 128] - scale: [8, 8, 8] - - # Spatial augmentation (CellMap format) - spatial_transforms: - mirror: - axes: {x: 0.5, y: 0.5, z: 0.5} # 50% flip probability per axis - transpose: - axes: [x, y, z] # Random permutation - rotate: - axes: - x: [-180, 180] - y: [-180, 180] - z: [-180, 180] - - # Training configuration - iter_num_per_epoch: 2000 # Steps per epoch - persistent_workers: false # Keep workers alive - -# Optimizer - AdamW (MedNeXt default) -optimization: - max_epochs: 500 - gradient_clip_val: 1.0 - accumulate_grad_batches: 4 # Effective batch = 2 * 4 = 8 - precision: "16-mixed" # Mixed precision - - optimizer: - name: AdamW - lr: 1e-3 # MedNeXt default (constant LR) - weight_decay: 1e-4 - - # Scheduler - constant LR (MedNeXt recommendation) - scheduler: - name: constant # No decay - -# Monitoring -monitor: - detect_anomaly: false - - logging: - scalar: - loss: [train_loss_total_epoch] - loss_every_n_steps: 50 - val_check_interval: 1.0 # Validate every epoch - benchmark: true - - images: - enabled: true - max_images: 2 - num_slices: 4 - log_every_n_epochs: 5 - channel_mode: all - - # Checkpointing - checkpoint: - mode: min - save_top_k: 3 - save_last: true - save_every_n_epochs: 25 - dirpath: outputs/cellmap_cos7/checkpoints/ - use_timestamp: true - - # Early stopping - early_stopping: - enabled: true - monitor: train_loss_total_epoch - patience: 50 - mode: min - min_delta: 1e-4 - -# Inference configuration -inference: - sliding_window: - window_size: [128, 128, 128] - sw_batch_size: 1 - overlap: 0.25 - blending: gaussian - sigma_scale: 0.25 - padding_mode: replicate - - test_time_augmentation: - flip_axes: null # All flip augmentations - rotation90_axes: null # No rotation (fast inference) - select_channel: all - channel_activations: - - [0, 5, softmax] # Softmax over 5 classes - ensemble_mode: mean - apply_mask: false - - save_prediction: - enabled: true - intensity_scale: -1 # Scale to [0, 255] - -# Test configuration (optional - requires test data with labels) -test: - data: - test_image: null # Set when running test mode - test_label: null - test_mask: null - test_resolution: [8, 8, 8] - - evaluation: - enabled: false # Enable when test labels available - metrics: [] diff --git a/tutorials/cellmap_cos7_datasplit.csv b/tutorials/cellmap_cos7_datasplit.csv deleted file mode 100644 index d83cb434..00000000 --- a/tutorials/cellmap_cos7_datasplit.csv +++ /dev/null @@ -1,244 +0,0 @@ -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop249/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop258/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop245/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop259/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop240/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop242/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop255/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop241/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop235/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop291/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop238/[nuc,mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop57/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop113/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop4/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop6/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop23/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop14/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop155/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop9/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop96/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop8/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop3/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop7/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop1/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop95/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop28/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop15/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop54/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop19/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop58/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop94/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop16/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop59/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop18/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop55/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop13/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop56/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop208/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop219/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop188/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop201/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop206/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop213/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop203/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop186/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop212/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop216/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop211/[nuc,mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop210/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop218/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop187/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop217/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop209/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop202/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop189/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop376/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop370/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop368/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop342/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop369/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop341/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop333/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop340/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop122/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop123/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop134/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop120/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop121/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop178/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop268/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop276/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop275/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop270/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop321/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop320/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop277/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop280/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop278/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop325/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop313/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop298/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop323/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop326/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop273/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop266/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop279/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop267/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop319/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop322/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop274/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop377/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop269/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop272/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop196/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop222/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop197/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop198/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop200/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop224/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop191/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop225/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop195/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop228/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop227/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop226/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop190/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop192/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop199/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop193/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop220/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop214/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop116/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop119/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop130/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop117/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop118/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop25/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop80/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop82/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop83/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop22/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop26/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop84/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop97/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop20/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop99/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop81/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop98/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop237/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop254/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop247/[nuc,golgi]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop239/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop248/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop292/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop236/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop234/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop243/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop257/[nuc]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop252/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop256/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop37/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop70/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop38/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop35/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop126/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop36/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop107/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop47/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop69/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop66/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop182/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop93/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop92/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop67/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop68/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop180/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop112/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop91/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop71/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop43/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop172/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop171/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop183/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop133/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop157/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop144/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop142/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop139/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop151/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop125/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop175/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop124/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop137/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop136/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop135/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop143/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop150/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop416/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop145/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop131/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop417/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop138/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop132/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-nacc-1/jrc_mus-nacc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-nacc-1/jrc_mus-nacc-1.zarr","recon-1/labels/groundtruth/crop115/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop101/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop65/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop62/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop50/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop181/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop87/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop34/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop27/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop61/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop100/[nuc,mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop51/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop60/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop64/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop111/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop33/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop63/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop86/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop85/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop174/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop78/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop79/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop173/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop185/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop176/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop156/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop159/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop147/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop161/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop160/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop165/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop163/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop129/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop148/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop162/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop146/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop141/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop149/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop164/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop158/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop140/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop166/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop74/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop32/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop31/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop76/[nuc,mito,er,golgi,ves]" -"validate","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop88/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop90/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop40/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop109/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop72/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop89/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop73/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop77/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop110/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop39/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop42/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop48/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop49/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop75/[nuc,mito,er,golgi,ves]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop229/[mito]" diff --git a/tutorials/cellmap_instance_full.yaml b/tutorials/cellmap_instance_full.yaml deleted file mode 100644 index 45a73453..00000000 --- a/tutorials/cellmap_instance_full.yaml +++ /dev/null @@ -1,205 +0,0 @@ -# CellMap Challenge - Phase 2: Combination Approach (11 Instance) -# -# PHASE 2: Full submission - Combination approach -# Part 1 of 2: See cellmap_semantic_full.yaml (36 semantic classes) -# Part 2 of 2: 11 instance classes (multi-class binary mask + SDT) -# -# This config trains 11 instance classes (evaluated using Hausdorff Distance + Accuracy). -# -# Instance classes (require SDT post-processing for instance IDs): -# - mito (mitochondria) - CRITICAL: appears in 14/16 test crops -# - endo (endosomes) -# - ld (lipid droplets) -# - lyso (lysosomes) -# - np (nuclear pore) -# - ves (vesicles) -# - peroxisome -# - mt (microtubules) -# - cell (whole cell) -# - nuc (nucleus instances) -# - vim (vimentin) -# -# Strategy: -# - Higher resolution (4nm) for better boundary detection -# - Multi-class binary mask output (11 channels) -# - SDT (Signed Distance Transform) for instance separation -# - Post-processing: Binary mask → SDT → Watershed → Instance IDs -# - Large model (MedNeXt-L) for 11-class capacity -# - Extended training (1500 epochs) for instance quality -# - Deep supervision for multi-scale learning -# -# Usage: -# python scripts/cellmap/train_cellmap.py --config tutorials/cellmap_instance_full.yaml - -experiment_name: cellmap_instance_full_mednext_l -description: CellMap full instance segmentation (11 classes) with MedNeXt-L - -# System -system: - training: - num_gpus: 4 # Multi-GPU recommended - num_cpus: 16 - num_workers: 8 - batch_size: 1 # Per GPU (effective = 4) - inference: - num_gpus: 1 - num_cpus: 4 - num_workers: 2 - batch_size: 1 - seed: 42 - -# Model Configuration -model: - architecture: mednext # MedNeXt (SOTA for instance segmentation) - - # Input/output configuration - input_size: [96, 96, 96] # Higher res patches for boundaries - output_size: [96, 96, 96] - in_channels: 1 # Grayscale EM - out_channels: 11 # 11 instance classes - - # MedNeXt configuration (large model for 11 classes) - mednext_size: L # L (61.8M params) - best for multi-class - mednext_kernel_size: 7 # 7x7x7 kernels for better context - deep_supervision: true # Multi-scale loss (CRITICAL) - - # Loss configuration (multi-class instance segmentation) - loss_functions: [DiceLoss, BCEWithLogitsLoss] - loss_weights: [1.0, 1.0] - loss_kwargs: - - {sigmoid: true, smooth_nr: 1e-5, smooth_dr: 1e-5} # Dice for each class - - {reduction: mean} # BCE for boundaries - -# Data - CellMap Challenge Dataset -data: - # Dataset type (custom for CellMap) - dataset_type: cellmap - - # CellMap-specific configuration - cellmap: - # Data paths - data_root: /projects/weilab/dataset/cellmap - datasplit_path: tutorials/cellmap_instance_full_datasplit.csv # Auto-generated - - # All 11 instance segmentation classes - classes: [mito, endo, ld, lyso, np, ves, peroxisome, mt, cell, nuc, vim] - force_all_classes: false # Don't require all classes (some are rare) - - # Patch configuration (higher resolution for boundaries) - input_array_info: - shape: [96, 96, 96] - scale: [4, 4, 4] # 4nm isotropic - best for instance boundaries - target_array_info: - shape: [96, 96, 96] - scale: [4, 4, 4] - - # Spatial augmentation (aggressive for instance segmentation) - spatial_transforms: - mirror: - axes: {x: 0.5, y: 0.5, z: 0.5} # 50% flip probability per axis - transpose: - axes: [x, y, z] # Random permutation - rotate: - axes: - x: [-180, 180] - y: [-180, 180] - z: [-180, 180] - - # Training configuration - iter_num_per_epoch: 2000 # Steps per epoch - persistent_workers: true - -# Optimizer - AdamW with lower LR for large model -optimization: - max_epochs: 1500 # Extended training for quality - gradient_clip_val: 1.0 - accumulate_grad_batches: 4 # Effective batch = 4 GPUs * 1 * 4 = 16 - precision: "16-mixed" # Mixed precision - - optimizer: - name: AdamW - lr: 5e-4 # Lower LR for large model - weight_decay: 1e-4 - - # Scheduler - cosine annealing with warmup - scheduler: - name: CosineAnnealingLR - warmup_epochs: 10 - min_lr: 1e-6 - -# Monitoring -monitor: - detect_anomaly: false - - logging: - scalar: - loss: [train_loss_total_epoch] - loss_every_n_steps: 50 - val_check_interval: 1.0 - benchmark: true - - images: - enabled: true - max_images: 2 - num_slices: 4 - log_every_n_epochs: 20 # Less frequent for long training - channel_mode: all - - # Checkpointing - checkpoint: - mode: min - save_top_k: 5 # Keep more checkpoints - save_last: true - save_every_n_epochs: 100 # Save less frequently - dirpath: outputs/cellmap_instance_full/checkpoints/ - use_timestamp: true - - # Early stopping (very patient for long training) - early_stopping: - enabled: true - monitor: train_loss_total_epoch - patience: 200 # Patient for 1500 epoch training - mode: min - min_delta: 1e-6 - -# Inference configuration (optimized for instance boundaries) -inference: - sliding_window: - window_size: [96, 96, 96] - sw_batch_size: 4 # Larger for faster inference - overlap: 0.75 # High overlap for instance boundaries - blending: gaussian - sigma_scale: 0.25 - padding_mode: replicate - - test_time_augmentation: - flip_axes: [[2], [3], [4]] # All 3 axes - rotation90_axes: null # Skip rotation for speed - select_channel: all - channel_activations: - - [0, 11, sigmoid] # Sigmoid for multi-label - ensemble_mode: mean - apply_mask: false - - save_prediction: - enabled: true - intensity_scale: -1 # Scale to [0, 255] - -# Test configuration -test: - data: - test_image: null # Set when running test mode - test_label: null - test_mask: null - test_resolution: [4, 4, 4] - - evaluation: - enabled: false # Enable when test labels available - metrics: [] - -# Post-processing (SDT-based instance segmentation) -# Note: Post-processing is done separately using connectomics.decoding -# 1. Binary mask prediction (11 channels, sigmoid output) -# 2. SDT (Signed Distance Transform) computation -# 3. Watershed segmentation on SDT -# 4. Instance ID assignment per class diff --git a/tutorials/cellmap_mito.yaml b/tutorials/cellmap_mito.yaml deleted file mode 100644 index 1f8efeed..00000000 --- a/tutorials/cellmap_mito.yaml +++ /dev/null @@ -1,183 +0,0 @@ -# CellMap Challenge - Phase 1: Simple Instance Validation -# -# PHASE 1: Make sure simple one works -# - 1 instance class (mitochondria) -# - Binary mask output -# - SDT (Signed Distance Transform) for instance separation -# -# This config validates the instance segmentation pipeline: -# - Higher resolution (4nm) for better boundary detection -# - Binary mask prediction (sigmoid activation) -# - Post-processing: Binary mask → SDT → Watershed → Instance IDs -# -# Mitochondria is the hardest instance segmentation task: -# - High density, touching objects, complex shapes -# - Appears in 14/16 test crops -# - Critical for overall challenge score -# -# Usage: -# python scripts/cellmap/train_cellmap.py --config tutorials/cellmap_mito.yaml - -experiment_name: cellmap_mito_mednext_l -description: CellMap mitochondria instance segmentation with MedNeXt-L - -# System -system: - training: - num_gpus: 1 - num_cpus: 4 - num_workers: 4 - batch_size: 1 # Smaller batch for larger model + higher res - inference: - num_gpus: 1 - num_cpus: 1 - num_workers: 1 - batch_size: 1 - seed: 42 - -# Model Configuration -model: - architecture: mednext # MedNeXt (SOTA for instance segmentation) - - # Input/output configuration - input_size: [96, 96, 96] # Smaller patches for 4nm resolution - output_size: [96, 96, 96] - in_channels: 1 # Grayscale EM - out_channels: 1 # Binary mitochondria segmentation - - # MedNeXt configuration (optimized for single class) - mednext_size: L # L (61.8M params) - best for single class - mednext_kernel_size: 7 # 7x7x7 kernels for better context - deep_supervision: true # Multi-scale loss (CRITICAL for quality) - - # Loss configuration (binary segmentation) - loss_functions: [DiceLoss, BCEWithLogitsLoss] - loss_weights: [1.0, 1.0] - loss_kwargs: - - {sigmoid: true, smooth_nr: 1e-5, smooth_dr: 1e-5} # Dice - - {reduction: mean} # BCE - -# Data - CellMap Challenge Dataset -data: - # Dataset type (custom for CellMap) - dataset_type: cellmap - - # CellMap-specific configuration - cellmap: - # Data paths - data_root: /projects/weilab/dataset/cellmap - datasplit_path: tutorials/cellmap_mito_datasplit.csv # Auto-generated next to YAML if missing - - # Single class: mitochondria - classes: [mito] - force_all_classes: both # Keep crops with mito in both train/val - - # Patch configuration (higher resolution) - input_array_info: - shape: [96, 96, 96] - scale: [4, 4, 4] # 4nm isotropic - higher res for boundaries - target_array_info: - shape: [96, 96, 96] - scale: [4, 4, 4] - - # Spatial augmentation (CellMap format) - spatial_transforms: - mirror: - axes: {x: 0.5, y: 0.5, z: 0.5} - transpose: - axes: [x, y, z] - rotate: - axes: - x: [-180, 180] - y: [-180, 180] - z: [-180, 180] - - # Training configuration - iter_num_per_epoch: 2000 # Steps per epoch - persistent_workers: true - -# Optimizer - AdamW with lower LR for larger model -optimization: - max_epochs: 1000 # Extended training for quality - gradient_clip_val: 1.0 - accumulate_grad_batches: 8 # Effective batch = 1 * 8 = 8 - precision: "16-mixed" - - optimizer: - name: AdamW - lr: 5e-4 # Lower LR for large model - weight_decay: 1e-4 - - # Scheduler - constant LR (MedNeXt recommendation) - scheduler: - name: constant - -# Monitoring -monitor: - detect_anomaly: false - - logging: - scalar: - loss: [train_loss_total_epoch] - loss_every_n_steps: 50 - val_check_interval: 1.0 - benchmark: true - - images: - enabled: true - max_images: 2 - num_slices: 4 - log_every_n_epochs: 10 - channel_mode: all - - # Checkpointing - checkpoint: - mode: min - save_top_k: 3 - save_last: true - save_every_n_epochs: 50 - dirpath: outputs/cellmap_mito/checkpoints/ - use_timestamp: true - - # Early stopping (patient for long training) - early_stopping: - enabled: true - monitor: train_loss_total_epoch - patience: 100 # Patient for 1000 epoch training - mode: min - min_delta: 1e-5 - -# Inference configuration -inference: - sliding_window: - window_size: [96, 96, 96] - sw_batch_size: 1 - overlap: 0.5 # Higher overlap for better boundaries - blending: gaussian - sigma_scale: 0.25 - padding_mode: replicate - - test_time_augmentation: - flip_axes: null # All flip augmentations - rotation90_axes: null - select_channel: all - channel_activations: - - [0, 1, sigmoid] # Binary segmentation - ensemble_mode: mean - apply_mask: false - - save_prediction: - enabled: true - intensity_scale: -1 - -# Test configuration -test: - data: - test_image: null - test_label: null - test_mask: null - test_resolution: [4, 4, 4] - - evaluation: - enabled: false - metrics: [] diff --git a/tutorials/cellmap_mito_datasplit.csv b/tutorials/cellmap_mito_datasplit.csv deleted file mode 100644 index 24e2fb07..00000000 --- a/tutorials/cellmap_mito_datasplit.csv +++ /dev/null @@ -1,234 +0,0 @@ -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop249/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop258/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop245/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop259/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop240/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop242/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop255/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop241/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop235/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1b/jrc_cos7-1b.zarr","recon-1/labels/groundtruth/crop291/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop57/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop113/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop4/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop6/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop23/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop14/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop155/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop9/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop96/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop8/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop3/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop7/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop1/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop95/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop28/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop15/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop54/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop19/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop58/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop94/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop16/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop59/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop18/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop55/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop13/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-2/jrc_hela-2.zarr","recon-1/labels/groundtruth/crop56/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop208/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop219/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop188/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop201/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop206/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop213/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop203/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop186/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop212/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop216/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop210/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop218/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop187/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop217/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop209/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop202/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-4/jrc_sum159-4.zarr","recon-1/labels/groundtruth/crop189/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop376/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop370/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop368/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop342/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop369/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop341/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop333/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-2/jrc_mus-liver-zon-2.zarr","recon-1/labels/groundtruth/crop340/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop122/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop123/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop134/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop120/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop121/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-mb-1a/jrc_fly-mb-1a.zarr","recon-1/labels/groundtruth/crop178/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop268/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop276/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop275/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop321/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop320/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop277/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop280/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop278/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop325/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop313/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop298/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop323/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop326/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop273/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop266/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop279/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop267/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop319/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop322/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop274/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop377/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop269/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver-zon-1/jrc_mus-liver-zon-1.zarr","recon-1/labels/groundtruth/crop272/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop196/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop222/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop197/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop198/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop200/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop224/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop191/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop225/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop195/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop228/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop227/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop226/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop190/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop192/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop199/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop193/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop220/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ut21-1413-003/jrc_ut21-1413-003.zarr","recon-1/labels/groundtruth/crop214/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop116/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop119/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop130/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop117/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_ctl-id8-1/jrc_ctl-id8-1.zarr","recon-1/labels/groundtruth/crop118/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop25/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop80/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop82/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop83/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop22/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop26/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop84/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop97/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop20/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop99/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop81/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_sum159-1/jrc_sum159-1.zarr","recon-1/labels/groundtruth/crop98/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop237/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop254/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop239/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop248/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop292/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop236/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop234/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop243/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop252/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_cos7-1a/jrc_cos7-1a.zarr","recon-1/labels/groundtruth/crop256/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop37/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop70/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop38/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop35/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop126/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop36/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop107/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop47/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop69/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop66/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop182/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop93/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop92/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop67/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop68/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop180/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop112/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop91/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop71/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_jurkat-1/jrc_jurkat-1.zarr","recon-1/labels/groundtruth/crop43/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop172/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop171/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop183/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop133/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop157/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop144/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop142/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop139/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop151/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop125/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop175/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop124/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop143/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop150/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop416/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop145/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop131/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop417/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-liver/jrc_mus-liver.zarr","recon-1/labels/groundtruth/crop138/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-nacc-1/jrc_mus-nacc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-nacc-1/jrc_mus-nacc-1.zarr","recon-1/labels/groundtruth/crop115/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop101/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop65/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop62/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop50/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop181/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop87/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop34/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop27/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop61/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop100/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop51/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop60/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop64/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop111/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop33/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop63/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop86/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_hela-3/jrc_hela-3.zarr","recon-1/labels/groundtruth/crop85/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop174/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop78/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop79/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop173/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop185/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_fly-vnc-1/jrc_fly-vnc-1.zarr","recon-1/labels/groundtruth/crop176/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop156/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop159/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop147/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop229/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop161/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop160/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop165/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop163/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop148/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop162/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop146/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop141/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop149/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop164/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop158/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop140/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_mus-kidney/jrc_mus-kidney.zarr","recon-1/labels/groundtruth/crop166/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop74/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop32/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop31/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop76/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop88/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop90/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop40/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop109/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop72/[mito]" -"validate","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop89/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop73/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop77/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop110/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop39/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop42/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop48/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop49/[mito]" -"train","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/em/fibsem-uint8","/projects/weilab/dataset/cellmap/jrc_macrophage-2/jrc_macrophage-2.zarr","recon-1/labels/groundtruth/crop75/[mito]" diff --git a/tutorials/cellmap_semantic47.yaml b/tutorials/cellmap_semantic47.yaml deleted file mode 100644 index 520720a6..00000000 --- a/tutorials/cellmap_semantic47.yaml +++ /dev/null @@ -1,249 +0,0 @@ -# CellMap Challenge - Phase 2: Full Semantic Submission -# -# PHASE 2: Full submission - 47 semantic classes -# - All 47 classes treated as SEMANTIC (multi-class binary mask) -# - No instance separation (SDT not used) -# - Simple unified approach -# -# All 47 semantic classes: -# -# Instance classes (11): -# - mito, endo, ld, lyso, np, ves, peroxisome, mt, cell, nuc, vim -# -# Semantic classes (36): -# - ne, er_mem, ecs, pm, mito_mem, golgi_mem, ves_mem, MVB_mem, lyso_mem, LD_mem -# - er_lumen, eres_mem, nucleus, nucleolus, nuclear_pore_out, chromatin -# - NHChrom, EChrom, NEChrom, HChrom, NHEChrom, NHHChrom, NHNEChrom -# - ribosomes, cytoplasm, glycogen, lipid, microtubule_out -# - er, golgi, np_semantic, eres, centrosome, distal_app, subdistal_app -# - ribosomes_free, ribosomes_bound, vesicle_lumen -# -# Trade-offs: -# ✅ Simpler: Single model, single training run -# ✅ Faster: Train once instead of twice -# ✅ Unified: Same backbone for all predictions -# ⚠️ Lower accuracy: Jack-of-all-trades vs specialized models -# ⚠️ Harder optimization: Multi-task learning challenges -# -# Strategy: -# - Standard resolution (8nm) - compromise for both instance/semantic -# - Large model (MedNeXt-L) for 47-class capacity -# - Sigmoid activation (multi-label) to handle overlapping classes -# - Mixed instance + semantic loss -# - Post-processing only for instance classes -# -# Usage: -# python scripts/cellmap/train_cellmap.py --config tutorials/cellmap_all47_baseline.yaml - -experiment_name: cellmap_semantic47_mednext_l -description: CellMap all 47 classes as semantic (multi-class binary mask) with MedNeXt-L - -# System -system: - training: - num_gpus: 4 # Multi-GPU recommended - num_cpus: 16 - num_workers: 8 - batch_size: 2 # Per GPU (effective = 8) - inference: - num_gpus: 1 - num_cpus: 4 - num_workers: 2 - batch_size: 1 - seed: 42 - -# Model Configuration -model: - architecture: mednext # MedNeXt (SOTA for medical imaging) - - # Input/output configuration - input_size: [128, 128, 128] # Standard patches (compromise) - output_size: [128, 128, 128] - in_channels: 1 # Grayscale EM - out_channels: 47 # All 47 classes - - # MedNeXt configuration (large model for 47 classes) - mednext_size: L # L (61.8M params) - needed for 47 classes - mednext_kernel_size: 5 # 5x5x5 kernels (balance context/memory) - deep_supervision: true # Multi-scale loss (CRITICAL) - - # Loss configuration (multi-label for overlapping instance/semantic) - loss_functions: [DiceLoss, BCEWithLogitsLoss] - loss_weights: [1.0, 1.0] - loss_kwargs: - - {sigmoid: true, smooth_nr: 1e-5, smooth_dr: 1e-5} # Dice for each class - - {reduction: mean} # BCE for all classes - -# Data - CellMap Challenge Dataset -data: - # Dataset type (custom for CellMap) - dataset_type: cellmap - - # CellMap-specific configuration - cellmap: - # Data paths - data_root: /projects/weilab/dataset/cellmap - datasplit_path: tutorials/cellmap_semantic47_datasplit.csv # Auto-generated - - # All 47 classes (11 instance + 36 semantic) - # Note: Adjust based on available labels in your dataset - classes: [ - # === 11 Instance Classes (require post-processing) === - mito, endo, ld, lyso, np, ves, peroxisome, mt, cell, nuc, vim, - - # === 36 Semantic Classes (direct output) === - # Membrane structures - ne, er_mem, ecs, pm, mito_mem, golgi_mem, ves_mem, MVB_mem, lyso_mem, LD_mem, - # Lumen and interior spaces - er_lumen, nucleus, nucleolus, cytoplasm, - # Nuclear components - nuclear_pore_out, chromatin, NHChrom, EChrom, NEChrom, HChrom, - NHEChrom, NHHChrom, NHNEChrom, - # Organelle components - eres_mem, ribosomes, glycogen, lipid, microtubule_out, - # Additional semantic structures - er, golgi, eres, centrosome, distal_app, subdistal_app, - ribosomes_free, ribosomes_bound - ] - force_all_classes: false # Don't require all classes (many are rare) - - # Patch configuration (standard resolution - compromise) - input_array_info: - shape: [128, 128, 128] - scale: [8, 8, 8] # 8nm isotropic - standard resolution - target_array_info: - shape: [128, 128, 128] - scale: [8, 8, 8] - - # Spatial augmentation (standard) - spatial_transforms: - mirror: - axes: {x: 0.5, y: 0.5, z: 0.5} # 50% flip probability per axis - transpose: - axes: [x, y, z] # Random permutation - rotate: - axes: - x: [-180, 180] - y: [-180, 180] - z: [-180, 180] - - # Training configuration - iter_num_per_epoch: 2000 # Steps per epoch - persistent_workers: true - -# Optimizer - AdamW -optimization: - max_epochs: 1000 # Extended training for 47 classes - gradient_clip_val: 1.0 - accumulate_grad_batches: 2 # Effective batch = 4 GPUs * 2 * 2 = 16 - precision: "16-mixed" # Mixed precision - - optimizer: - name: AdamW - lr: 1e-3 # MedNeXt default - weight_decay: 1e-4 - - # Scheduler - cosine annealing with warmup - scheduler: - name: CosineAnnealingLR - warmup_epochs: 20 # Longer warmup for 47 classes - min_lr: 1e-6 - -# Monitoring -monitor: - detect_anomaly: false - - logging: - scalar: - loss: [train_loss_total_epoch] - loss_every_n_steps: 50 - val_check_interval: 1.0 - benchmark: true - - images: - enabled: true - max_images: 2 - num_slices: 4 - log_every_n_epochs: 10 - channel_mode: all # Show all 47 channels - - # Checkpointing - checkpoint: - mode: min - save_top_k: 5 - save_last: true - save_every_n_epochs: 50 - dirpath: outputs/cellmap_semantic47/checkpoints/ - use_timestamp: true - - # Early stopping - early_stopping: - enabled: true - monitor: train_loss_total_epoch - patience: 100 # Patient for 1000 epoch training - mode: min - min_delta: 1e-5 - -# Inference configuration -inference: - sliding_window: - window_size: [128, 128, 128] - sw_batch_size: 4 - overlap: 0.5 # Medium overlap (compromise) - blending: gaussian - sigma_scale: 0.25 - padding_mode: replicate - - test_time_augmentation: - flip_axes: [[2], [3], [4]] # All 3 axes - rotation90_axes: null # Skip rotation for speed - select_channel: all - channel_activations: - - [0, 47, sigmoid] # Sigmoid for multi-label (all 47 classes) - ensemble_mode: mean - apply_mask: false - - save_prediction: - enabled: true - intensity_scale: -1 # Scale to [0, 255] - -# Test configuration -test: - data: - test_image: null # Set when running test mode - test_label: null - test_mask: null - test_resolution: [8, 8, 8] - - evaluation: - enabled: false # Enable when test labels available - metrics: [] - -# Post-processing Notes: -# -# For challenge submission, you need to separate predictions into: -# -# 1. Instance classes (11) - channels 0-10: -# - Apply threshold (e.g., 0.5) -# - Connected components (cc3d) -# - Watershed segmentation -# - Instance ID assignment -# - Classes: mito, endo, ld, lyso, np, ves, peroxisome, mt, cell, nuc, vim -# - Evaluated with: Hausdorff Distance + Accuracy -# -# 2. Semantic classes (36) - channels 11-46: -# - Apply threshold (e.g., 0.5) OR argmax across semantic channels -# - Direct submission (no instance IDs needed) -# - Classes: ne, er_mem, ecs, pm, mito_mem, golgi_mem, etc. -# - Evaluated with: IoU + Dice Score -# -# Trade-off vs Specialized Models: -# - Baseline: Single model, faster training (~30h vs ~60h total) -# - Specialized: Two models (instance + semantic), higher accuracy -# - Recommendation: Start with baseline, then train specialized if needed - -# Expected Performance: -# - Training time: ~30 hours on 4x A100 GPUs -# - Memory: ~20GB per GPU with batch_size=2 -# - Expected leaderboard: Mid-tier (good baseline, not SOTA) -# - Improvement path: Train separate instance/semantic models diff --git a/tutorials/cellmap_semantic_full.yaml b/tutorials/cellmap_semantic_full.yaml deleted file mode 100644 index 31c0eb67..00000000 --- a/tutorials/cellmap_semantic_full.yaml +++ /dev/null @@ -1,233 +0,0 @@ -# CellMap Challenge - Phase 2: Combination Approach (36 Semantic) -# -# PHASE 2: Full submission - Combination approach -# Part 1 of 2: 36 semantic classes (multi-class binary mask) -# Part 2 of 2: See cellmap_instance_full.yaml (11 instance classes with SDT) -# -# This config trains 36 semantic classes only (evaluated using IoU + Dice Score). -# -# Semantic classes (no instance post-processing needed): -# Background + 35 organelle classes including: -# - ne (nuclear envelope) -# - er_mem (ER membrane) -# - ecs (extracellular space) -# - pm (plasma membrane) -# - mito_mem (mitochondrial membrane) -# - golgi_mem (Golgi membrane) -# - ves_mem (vesicle membrane) -# - MVB_mem (MVB membrane) -# - lyso_mem (lysosome membrane) -# - LD_mem (lipid droplet membrane) -# - er_lumen (ER lumen) -# - eres_mem (ERES membrane) -# - nucleus (nucleoplasm - semantic) -# - nucleolus -# - nuclear_pore_out (NP outer) -# - chromatin -# - NHChrom (non-histone chromatin) -# - EChrom (euchromatin) -# - NEChrom (nuclear envelope chromatin) -# - HChrom (heterochromatin) -# - NHEChrom (non-histone euchromatin) -# - NHHChrom (non-histone heterochromatin) -# - NHNEChrom (non-histone NE chromatin) -# - ribosomes -# - cytoplasm -# - glycogen -# - lipid (lipid content) -# - microtubule_out (MT outer) -# And more... -# -# Strategy: -# - Standard resolution (8nm) for semantic context -# - Large model (MedNeXt-L) for 36-class capacity -# - Long training (1000 epochs) for multi-class learning -# - Softmax activation for mutually exclusive classes -# -# Usage: -# python scripts/cellmap/train_cellmap.py --config tutorials/cellmap_semantic_full.yaml - -experiment_name: cellmap_semantic_full_mednext_l -description: CellMap full semantic segmentation (36 classes) with MedNeXt-L - -# System -system: - training: - num_gpus: 4 # Multi-GPU recommended - num_cpus: 16 - num_workers: 8 - batch_size: 2 # Per GPU (effective = 8) - inference: - num_gpus: 1 - num_cpus: 4 - num_workers: 2 - batch_size: 1 - seed: 42 - -# Model Configuration -model: - architecture: mednext # MedNeXt (SOTA for semantic segmentation) - - # Input/output configuration - input_size: [128, 128, 128] # Larger patches for semantic context - output_size: [128, 128, 128] - in_channels: 1 # Grayscale EM - out_channels: 36 # 36 semantic classes (including background) - - # MedNeXt configuration (large model for 36 classes) - mednext_size: L # L (61.8M params) - needed for 36 classes - mednext_kernel_size: 5 # 5x5x5 kernels (balance context/memory) - deep_supervision: true # Multi-scale loss (CRITICAL) - - # Loss configuration (multi-class semantic segmentation) - loss_functions: [DiceCELoss] - loss_weights: [1.0] - loss_kwargs: - - {sigmoid: false, softmax: true, to_onehot_y: true, include_background: true} - -# Data - CellMap Challenge Dataset -data: - # Dataset type (custom for CellMap) - dataset_type: cellmap - - # CellMap-specific configuration - cellmap: - # Data paths - data_root: /projects/weilab/dataset/cellmap - datasplit_path: tutorials/cellmap_semantic_full_datasplit.csv # Auto-generated - - # All 36 semantic segmentation classes - # Note: This is a representative subset - adjust based on available labels - classes: [ - # Core semantic structures - ne, er_mem, ecs, pm, - # Membrane structures - mito_mem, golgi_mem, ves_mem, MVB_mem, lyso_mem, LD_mem, - # Lumen/interior spaces - er_lumen, nucleus, nucleolus, cytoplasm, - # Nuclear components - nuclear_pore_out, chromatin, NHChrom, EChrom, NEChrom, HChrom, - NHEChrom, NHHChrom, NHNEChrom, - # Organelle membranes and components - eres_mem, ribosomes, glycogen, lipid, microtubule_out, - # Additional semantic classes (adjust based on dataset) - er, golgi, np, eres, centrosome, distal_app, subdistal_app, - ribosomes_free, ribosomes_bound, vesicle_lumen - ] - force_all_classes: false # Don't require all classes (many are rare) - - # Patch configuration (standard resolution for semantic) - input_array_info: - shape: [128, 128, 128] - scale: [8, 8, 8] # 8nm isotropic - good for semantic context - target_array_info: - shape: [128, 128, 128] - scale: [8, 8, 8] - - # Spatial augmentation (moderate for semantic) - spatial_transforms: - mirror: - axes: {x: 0.5, y: 0.5, z: 0.5} # 50% flip probability per axis - transpose: - axes: [x, y, z] # Random permutation - rotate: - axes: - x: [-180, 180] - y: [-180, 180] - z: [-180, 180] - - # Training configuration - iter_num_per_epoch: 2000 # Steps per epoch - persistent_workers: true - -# Optimizer - AdamW (MedNeXt default) -optimization: - max_epochs: 1000 # Long training for 36 classes - gradient_clip_val: 1.0 - accumulate_grad_batches: 2 # Effective batch = 4 GPUs * 2 * 2 = 16 - precision: "16-mixed" # Mixed precision - - optimizer: - name: AdamW - lr: 1e-3 # MedNeXt default (constant LR) - weight_decay: 1e-4 - - # Scheduler - constant LR (MedNeXt recommendation) - scheduler: - name: constant # No decay for semantic - -# Monitoring -monitor: - detect_anomaly: false - - logging: - scalar: - loss: [train_loss_total_epoch] - loss_every_n_steps: 50 - val_check_interval: 1.0 - benchmark: true - - images: - enabled: true - max_images: 2 - num_slices: 4 - log_every_n_epochs: 10 - channel_mode: all - - # Checkpointing - checkpoint: - mode: min - save_top_k: 5 - save_last: true - save_every_n_epochs: 50 - dirpath: outputs/cellmap_semantic_full/checkpoints/ - use_timestamp: true - - # Early stopping - early_stopping: - enabled: true - monitor: train_loss_total_epoch - patience: 100 # Patient for 1000 epoch training - mode: min - min_delta: 1e-5 - -# Inference configuration -inference: - sliding_window: - window_size: [128, 128, 128] - sw_batch_size: 4 - overlap: 0.25 # Lower overlap (semantic is more robust) - blending: gaussian - sigma_scale: 0.25 - padding_mode: replicate - - test_time_augmentation: - flip_axes: [[2], [3], [4]] # All 3 axes - rotation90_axes: null # Skip rotation for speed - select_channel: all - channel_activations: - - [0, 36, softmax] # Softmax over 36 classes - ensemble_mode: mean - apply_mask: false - - save_prediction: - enabled: true - intensity_scale: -1 # Scale to [0, 255] - -# Test configuration -test: - data: - test_image: null # Set when running test mode - test_label: null - test_mask: null - test_resolution: [8, 8, 8] - - evaluation: - enabled: false # Enable when test labels available - metrics: [] - -# Notes for CellMap Challenge Submission: -# 1. Semantic predictions are evaluated using IoU + Dice Score -# 2. No post-processing needed (unlike instance segmentation) -# 3. Output is argmax of softmax probabilities for each voxel -# 4. Submit at original test crop resolution (varies by crop) diff --git a/tutorials/fiber.yaml b/tutorials/fiber_linghu26.yaml similarity index 97% rename from tutorials/fiber.yaml rename to tutorials/fiber_linghu26.yaml index 9f2a2947..9b9be8c9 100644 --- a/tutorials/fiber.yaml +++ b/tutorials/fiber_linghu26.yaml @@ -225,17 +225,15 @@ monitor: threshold: 0.01 divergence_threshold: 100.0 -# Inference - MONAI SlidingWindowInferer for fiber segmentation (based on barcode-R-Base.yaml) -inference: +test: data: # Test on all available volumes (matches barcode INFERENCE IMAGE_NAME) - # Note: test_path will be combined with these relative paths - test_path: "/projects/weilab/dataset/barcode/train_r2/" - test_image: ["DG_LZ58/0702-2-C4-DG-40X002_1-raw.tif"] - test_label: ["DG_LZ58/0702-2-C4-DG-40X002_1-mask.tif"] + test_image: ["/projects/weilab/dataset/barcode/train_r2/DG_LZ58/0702-2-C4-DG-40X002_1-raw.tif"] + test_label: ["/projects/weilab/dataset/barcode/train_r2/DG_LZ58/0702-2-C4-DG-40X002_1-mask.tif"] test_resolution: [40, 16, 16] # Isotropic resolution - # Output filename (auto-pathed based on the checkpoint folder) - output_name: DG_train.h5 + +# Inference - MONAI SlidingWindowInferer for fiber segmentation (based on barcode-R-Base.yaml) +inference: # MONAI SlidingWindowInferer parameters sliding_window: diff --git a/tutorials/HYDRA_LV_FINETUNE_IMPROVEMENT_PLAN.md b/tutorials/misc/HYDRA_LV_FINETUNE_IMPROVEMENT_PLAN.md similarity index 100% rename from tutorials/HYDRA_LV_FINETUNE_IMPROVEMENT_PLAN.md rename to tutorials/misc/HYDRA_LV_FINETUNE_IMPROVEMENT_PLAN.md diff --git a/tutorials/example_tile_metadata.json b/tutorials/misc/example_tile_metadata.json similarity index 100% rename from tutorials/example_tile_metadata.json rename to tutorials/misc/example_tile_metadata.json diff --git a/tutorials/hydra-lv-finetune.yaml b/tutorials/misc/hydra-lv-finetune.yaml similarity index 100% rename from tutorials/hydra-lv-finetune.yaml rename to tutorials/misc/hydra-lv-finetune.yaml diff --git a/tutorials/hydra-lv-finetune_v0.yaml b/tutorials/misc/hydra-lv-finetune_v0.yaml similarity index 100% rename from tutorials/hydra-lv-finetune_v0.yaml rename to tutorials/misc/hydra-lv-finetune_v0.yaml diff --git a/tutorials/hydra-lv.yaml b/tutorials/misc/hydra-lv.yaml similarity index 100% rename from tutorials/hydra-lv.yaml rename to tutorials/misc/hydra-lv.yaml diff --git a/tutorials/mito2dsem_nnunet.yaml b/tutorials/misc/mito_2dsem_seg.yaml similarity index 100% rename from tutorials/mito2dsem_nnunet.yaml rename to tutorials/misc/mito_2dsem_seg.yaml diff --git a/tutorials/tsai_axon.yaml b/tutorials/misc/tsai_axon.yaml similarity index 100% rename from tutorials/tsai_axon.yaml rename to tutorials/misc/tsai_axon.yaml diff --git a/tutorials/worm2d.yaml b/tutorials/misc/worm2d.yaml similarity index 100% rename from tutorials/worm2d.yaml rename to tutorials/misc/worm2d.yaml diff --git a/tutorials/zebrafish_neurons.yaml b/tutorials/misc/zebrafish_neurons.yaml similarity index 100% rename from tutorials/zebrafish_neurons.yaml rename to tutorials/misc/zebrafish_neurons.yaml diff --git a/tutorials/betaSeg.yaml b/tutorials/mito_betaseg.yaml similarity index 100% rename from tutorials/betaSeg.yaml rename to tutorials/mito_betaseg.yaml diff --git a/tutorials/lucchi++.yaml b/tutorials/mito_lucchi++.yaml similarity index 100% rename from tutorials/lucchi++.yaml rename to tutorials/mito_lucchi++.yaml diff --git a/tutorials/mitoEM.yaml b/tutorials/mito_mitoEM.yaml similarity index 98% rename from tutorials/mitoEM.yaml rename to tutorials/mito_mitoEM.yaml index 9ccc07d8..5af84fe0 100644 --- a/tutorials/mitoEM.yaml +++ b/tutorials/mito_mitoEM.yaml @@ -94,7 +94,7 @@ data: # Augmentation - crucial for generalization augmentation: - enabled: true # Enable spatial + intensity augmentation + preset: "all" # Legacy `enabled: true` -> modern preset mode # Optimizer - MedNeXt recommended settings optimization: @@ -154,14 +154,16 @@ monitor: threshold: 0.01 divergence_threshold: 100.0 -# Inference - MONAI SlidingWindowInferer for MitoEM -inference: +# Test data paths (schema-compliant location) +test: data: test_image: /projects/weilab/dataset/mito/mitoEM/EM30-H/im_test.h5 test_label: /projects/weilab/dataset/mito/mitoEM/EM30-H/mito_test.h5 test_resolution: [30, 8, 8] output_path: outputs/mitoem_mednext_sdt/results/ +# Inference - MONAI SlidingWindowInferer for MitoEM +inference: # MONAI SlidingWindowInferer parameters sliding_window: window_size: [16, 256, 256] # Match training patch size diff --git a/tutorials/cem-mitolab.yaml b/tutorials/mito_mitolab.yaml similarity index 95% rename from tutorials/cem-mitolab.yaml rename to tutorials/mito_mitolab.yaml index fbfffb86..65c773b2 100644 --- a/tutorials/cem-mitolab.yaml +++ b/tutorials/mito_mitolab.yaml @@ -93,7 +93,7 @@ data: # Augmentation augmentation: - enabled: true + preset: "all" # Legacy `enabled: true` -> modern preset mode # Optimizer - AdamW (MedNeXt paper recommendation) optimization: @@ -158,21 +158,25 @@ monitor: threshold: 0.05 divergence_threshold: 2.0 -# Inference - For testing on held-out images -inference: +# Test data paths (schema-compliant location) +test: data: test_image: datasets/cem-mitolab/test.json # JSON file for filename-based dataset - test_label: null # Labels loaded from JSON + test_label: null # Labels loaded from JSON test_resolution: [0, 5, 5] output_path: outputs/cem-mitolab_mednext2d/results/ - + +# Inference - For testing on held-out images +inference: # No sliding window needed - images are already tiles (direct inference) # Test-Time Augmentation (TTA) test_time_augmentation: enabled: false # No TTA for pre-tiled images flip_axes: null - act: sigmoid # Sigmoid activation + channel_activations: + - [0, 2, sigmoid] # Binary + boundary channels + - [2, 3, tanh] # Distance channel select_channel: null ensemble_mode: mean diff --git a/tutorials/snemi.yaml b/tutorials/neuron_snemi.yaml similarity index 98% rename from tutorials/snemi.yaml rename to tutorials/neuron_snemi.yaml index 3dec7c99..df277853 100644 --- a/tutorials/snemi.yaml +++ b/tutorials/neuron_snemi.yaml @@ -115,7 +115,7 @@ data: # Augmentation augmentation: - enabled: true + preset: "some" # Standard augmentations flip: @@ -234,14 +234,16 @@ monitor: early_stopping: enabled: false # Train for full duration -# Inference -inference: +# Test data paths (schema-compliant location) +test: data: test_image: datasets/SNEMI/train-input.tif test_label: datasets/SNEMI/train-labels.tif test_resolution: [30, 6, 6] output_path: outputs/rsunet_snemi_aug3_long/results/ +# Inference +inference: # MONAI SlidingWindowInferer parameters sliding_window: window_size: [16, 180, 180] # Patch size extracted from volume @@ -255,7 +257,7 @@ inference: test_time_augmentation: enabled: true flip_axes: null # Flip strategy: "all" (8 flips), null (no aug), or custom list - act: softmax # Activation: 'softmax', 'sigmoid', null + channel_activations: [[0, 12, softmax]] select_channel: [1] # Channel selection: [1] (foreground only), null (all) ensemble_mode: mean # Ensemble strategy: 'mean', 'min', 'max' diff --git a/tutorials/nucmm-z.yaml b/tutorials/nuc_nucmm-z.yaml similarity index 95% rename from tutorials/nucmm-z.yaml rename to tutorials/nuc_nucmm-z.yaml index e24892a1..387deebb 100644 --- a/tutorials/nucmm-z.yaml +++ b/tutorials/nuc_nucmm-z.yaml @@ -98,7 +98,7 @@ data: # Augmentation augmentation: - enabled: true + preset: "all" # Legacy `enabled: true` -> modern preset mode # Optimizer - AdamW with NucMM-specific hyperparameters optimization: @@ -161,13 +161,15 @@ monitor: threshold: 0.01 divergence_threshold: 100.0 -# Inference - MONAI SlidingWindowInferer for NucMM -inference: +# Test data paths (schema-compliant location) +test: data: test_image: datasets/NucMM-Z/Image/val/img_0000_0640_0896.h5 test_label: datasets/NucMM-Z/Label/val/seg_0000_0640_0896.h5 test_resolution: [1.0, 1.0, 1.0] +# Inference - MONAI SlidingWindowInferer for NucMM +inference: # MONAI SlidingWindowInferer parameters sliding_window: window_size: [64, 64, 64] # Match training patch size @@ -181,7 +183,9 @@ inference: test_time_augmentation: enabled: true flip_axes: null # No augmentation for NucMM - act: sigmoid # Primary activation (binary segmentation) + channel_activations: + - [0, 2, sigmoid] # Binary + boundary channels + - [2, 3, tanh] # Distance channel select_channel: null # Use all channels ensemble_mode: mean