Split decoding schema and migrate tutorials

Author: Donglai Wei

connectomics/config/pipeline/profile_engine.py

@@ -608,8 +608,8 @@ def _build_reference_profile_specs() -> List[Tuple[str, List[str]]]:
     (
         "decoding_templates",
         (_STAGE_ROOT, _STAGE_DEFAULT, _STAGE_TUNE, _STAGE_TEST),
-        "decoding",
-        "decoding",
+        "decoding.steps",
+        "steps",
     ),
 ]
 

connectomics/config/pipeline/stage_resolver.py

@@ -16,8 +16,6 @@
 from ..schema.root import MergeContext
 from .dict_utils import as_plain_dict
 
-_NO_PATCH = object()
-
 
 def _collect_explicit_paths(yaml_node: Any, path: str = "") -> set[str]:
     """Collect all explicit key paths present in YAML."""
@@ -149,18 +147,9 @@ def _walk(node: Any, path: str) -> Any:
     return extracted if isinstance(extracted, dict) else {}
 
 
-def _extract_explicit_value(
-    section_obj: Any,
-    section_path: str,
-    has_explicit_path: Callable[[str], bool],
-) -> Any:
-    """Extract an explicitly provided non-mapping section value."""
-    return section_obj if has_explicit_path(section_path) else _NO_PATCH
-
-
 def _has_section_patch(value: Any) -> bool:
     """Return whether a section override should participate in runtime merging."""
-    return value is not _NO_PATCH and value != {}
+    return value != {}
 
 
 def _resolve_stage_key(mode: str) -> str:
@@ -229,7 +218,7 @@ def _collect_default_overrides(
             "default.inference",
             has_explicit_path,
         ),
-        "decoding": _extract_explicit_value(
+        "decoding": _extract_explicit_patch(
             getattr(default_stage, "decoding", None),
             "default.decoding",
             has_explicit_path,
@@ -258,18 +247,11 @@ def _collect_stage_overrides(
     for section_name in _MODE_SECTIONS[stage_key]:
         section_obj = getattr(stage_cfg, section_name, None)
         section_path = f"{stage_key}.{section_name}"
-        if section_name == "decoding":
-            stage_overrides[section_name] = _extract_explicit_value(
-                section_obj,
-                section_path,
-                has_explicit_path,
-            )
-        else:
-            stage_overrides[section_name] = _extract_explicit_patch(
-                section_obj,
-                section_path,
-                has_explicit_path,
-            )
+        stage_overrides[section_name] = _extract_explicit_patch(
+            section_obj,
+            section_path,
+            has_explicit_path,
+        )
 
     return stage_overrides
 
@@ -289,13 +271,6 @@ def _merge_runtime_sections(
         if not _has_section_patch(default_section) and not _has_section_patch(stage_section):
             continue
 
-        if section_name == "decoding":
-            if _has_section_patch(stage_section):
-                cfg.decoding = stage_section
-            elif _has_section_patch(default_section):
-                cfg.decoding = default_section
-            continue
-
         target_section = getattr(cfg, section_name)
         if not _has_section_patch(default_section):
             default_section = {}

connectomics/config/profiles/pipeline_profiles.yaml

@@ -17,7 +17,8 @@ pipeline_profiles:
       label_transform:
         profile: label_bcd
     decoding:
-      - template: decoding_bcd
+      steps:
+        - template: decoding_bcd
     inference:
       test_time_augmentation:
         ensemble_mode: mean
@@ -31,7 +32,8 @@ pipeline_profiles:
       label_transform:
         profile: label_aff9
     decoding:
-      - template: decoding_waterz
+      steps:
+        - template: decoding_waterz
     inference:
       test_time_augmentation:
         ensemble_mode: min
@@ -45,7 +47,8 @@ pipeline_profiles:
       label_transform:
         profile: label_aff12
     decoding:
-      - template: decoding_waterz
+      steps:
+        - template: decoding_waterz
     inference:
       select_channel: [0, 1, 2]
       test_time_augmentation:
@@ -60,7 +63,8 @@ pipeline_profiles:
       label_transform:
         profile: label_aff9_sdt
     decoding:
-      - template: decoding_waterz
+      steps:
+        - template: decoding_waterz
     inference:
       test_time_augmentation:
         ensemble_mode: [["0:9", min], ["9:", mean]]

connectomics/config/schema/__init__.py

@@ -32,18 +32,21 @@
     SliceShiftZConfig,
     StripeConfig,
 )
-from .inference import (
+from .decoding import (
     BinaryPostprocessingConfig,
-    ChunkingConfig,
-    ChunkStitchingConfig,
     ConnectedComponentsConfig,
     DecodeBinaryContourDistanceWatershedConfig,
     DecodeModeConfig,
-    EvaluationConfig,
+    DecodingConfig,
+    PostprocessingConfig,
+)
+from .evaluation import EvaluationConfig
+from .inference import (
+    ChunkingConfig,
+    ChunkStitchingConfig,
     InferenceConfig,
     InferenceDataConfig,
     InferenceMemoryCleanupConfig,
-    PostprocessingConfig,
     PredictionTransformConfig,
     SavePredictionConfig,
     SlidingWindowConfig,
@@ -133,6 +136,7 @@
     "PredictionTransformConfig",
     "SavePredictionConfig",
     "InferenceMemoryCleanupConfig",
+    "DecodingConfig",
     "PostprocessingConfig",
     "BinaryPostprocessingConfig",
     "ConnectedComponentsConfig",

connectomics/config/schema/decoding.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+
+@dataclass
+class ConnectedComponentsConfig:
+    """Connected components filtering configuration."""
+
+    enabled: bool = False
+    top_k: Optional[int] = None
+    min_size: int = 0
+    connectivity: int = 1
+
+
+@dataclass
+class BinaryPostprocessingConfig:
+    """Binary postprocessing pipeline configuration."""
+
+    enabled: bool = False
+    median_filter_size: Optional[Tuple[int, ...]] = None
+    opening_iterations: int = 0
+    closing_iterations: int = 0
+    connected_components: Optional[ConnectedComponentsConfig] = None
+
+
+@dataclass
+class PostprocessingConfig:
+    """Postprocessing configuration for decoded outputs."""
+
+    enabled: bool = False
+    binary: Optional[Dict[str, Any]] = field(default_factory=dict)
+    instance_cc3d: Optional[Dict[str, Any]] = None
+    output_transpose: List[int] = field(default_factory=list)
+
+
+@dataclass
+class DecodeBinaryContourDistanceWatershedConfig:
+    """Parameters for binary+contour+distance watershed decoding."""
+
+    binary_threshold: Tuple[float, float] = (0.9, 0.8)
+    contour_threshold: Tuple[float, float] = (0.5, 0.5)
+    distance_threshold: Tuple[float, float] = (0.5, -0.5)
+    min_instance_size: int = 10
+    min_seed_size: int = 10
+    seed_distance_scale: float = 1.0
+
+
+@dataclass
+class DecodeModeConfig:
+    """Single decode mode configuration."""
+
+    enabled: bool = True
+    name: str = "decode_semantic"
+    kwargs: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class DecodingConfig:
+    """Decoded-output orchestration configuration."""
+
+    steps: List[DecodeModeConfig] = field(default_factory=list)
+    postprocessing: PostprocessingConfig = field(default_factory=PostprocessingConfig)
+    output_path: str = ""
+    input_prediction_path: str = ""
+    tuning: Optional[Dict[str, Any]] = None

connectomics/config/schema/evaluation.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+
+@dataclass
+class EvaluationConfig:
+    """Evaluation configuration."""
+
+    enabled: bool = False
+    metrics: Optional[List[str]] = None
+    prediction_threshold: float = 0.5
+    instance_iou_threshold: float = 0.5
+    nerl_graph: Any = None
+    nerl_mask: Any = None
+    nerl_resolution: Optional[List[float]] = None
+    nerl_merge_threshold: int = 1
+    nerl_chunk_num: int = 1
+    nerl_skeleton_id_attribute: str = "id"
+    nerl_skeleton_position_attribute: str = "index_position"
+    nerl_skeleton_edge_length_attribute: str = "edge_length"
+    nerl_skeleton_position_order: str = "xyz"
+    nerl_prediction_position_order: Optional[str] = None

connectomics/config/schema/inference.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
 
 from .system import SystemConfig
 
@@ -157,102 +157,6 @@ class InferenceMemoryCleanupConfig:
     release_model_after_inference: bool = False
 
 
-@dataclass
-class DecodeBinaryContourDistanceWatershedConfig:
-    """Parameters for binary+contour+distance watershed decoding."""
-
-    binary_threshold: Tuple[float, float] = (0.9, 0.8)
-    contour_threshold: Tuple[float, float] = (0.5, 0.5)
-    distance_threshold: Tuple[float, float] = (0.5, -0.5)
-    min_instance_size: int = 10
-    min_seed_size: int = 10
-    seed_distance_scale: float = 1.0
-
-
-@dataclass
-class DecodeModeConfig:
-    """Single decode mode configuration."""
-
-    enabled: bool = True
-    name: str = "decode_semantic"
-    kwargs: Dict[str, Any] = field(default_factory=dict)
-
-
-@dataclass
-class BinaryPostprocessingConfig:
-    """Binary postprocessing pipeline configuration."""
-
-    enabled: bool = False  # Enable binary postprocessing pipeline
-    median_filter_size: Optional[Tuple[int, ...]] = (
-        None  # Median filter kernel size (e.g., (3, 3) for 2D)
-    )
-    opening_iterations: int = 0  # Number of morphological opening iterations
-    closing_iterations: int = 0  # Number of morphological closing iterations
-    connected_components: Optional[ConnectedComponentsConfig] = None  # CC filtering config
-
-
-@dataclass
-class ConnectedComponentsConfig:
-    """Connected components filtering configuration."""
-
-    enabled: bool = False  # Enable connected components filtering
-    top_k: Optional[int] = None  # Keep only top-k largest components (None = keep all)
-    min_size: int = 0  # Minimum component size in voxels
-    connectivity: int = 1  # Connectivity for CC (1=face, 2=face+edge, 3=face+edge+corner)
-
-
-@dataclass
-class PostprocessingConfig:
-    """Postprocessing configuration for inference output.
-
-    Controls how decoded outputs are transformed after decoding:
-    - Binary refinement: Morphological operations and connected components filtering
-    - Transpose: Reorder axes (e.g., [2,1,0] for zyx->xyz)
-
-    Note: raw prediction scaling/dtype changes that should affect decoding are
-    handled by PredictionTransformConfig. Save-only encoding is handled by
-    SavePredictionConfig.
-    """
-
-    enabled: bool = False  # Enable postprocessing pipeline
-
-    # Binary segmentation refinement (morphological ops, connected components)
-    binary: Optional[Dict[str, Any]] = field(
-        default_factory=dict
-    )  # Binary postprocessing config (e.g., {'opening_iterations': 2})
-
-    # Instance cc3d relabeling: split disconnected components and remove small ones
-    instance_cc3d: Optional[Dict[str, Any]] = None
-    # Example: {connectivity: 6, min_size: 100, remove_boundary: false}
-
-    # Axis permutation
-    output_transpose: List[int] = field(
-        default_factory=list
-    )  # Axis permutation for output (e.g., [2,1,0] for zyx->xyz)
-
-
-@dataclass
-class EvaluationConfig:
-    """Evaluation configuration."""
-
-    enabled: bool = False  # Auto-enabled when evaluation keys are provided in YAML
-    metrics: Optional[List[str]] = None  # e.g., ['dice', 'jaccard', 'accuracy']
-    prediction_threshold: float = 0.5  # Probability/logit threshold for binary metrics
-    instance_iou_threshold: float = 0.5  # IoU threshold for instance matching metrics
-    # Neurite ERL evaluation via lib/em_erl. nerl_graph accepts an ERLGraph
-    # .npz or a BANIS/NISB-style NetworkX skeleton.pkl.
-    nerl_graph: Any = None
-    nerl_mask: Any = None
-    nerl_resolution: Optional[List[float]] = None
-    nerl_merge_threshold: int = 1
-    nerl_chunk_num: int = 1
-    nerl_skeleton_id_attribute: str = "id"
-    nerl_skeleton_position_attribute: str = "index_position"
-    nerl_skeleton_edge_length_attribute: str = "edge_length"
-    nerl_skeleton_position_order: str = "xyz"
-    nerl_prediction_position_order: Optional[str] = None
-
-
 @dataclass
 class InferenceConfig:
     """Inference configuration.
@@ -289,16 +193,10 @@ class InferenceConfig:
     # Optional explicit intermediate prediction file (.h5). If set in test
     # mode, pipeline loads this file directly and proceeds to top-level decoding.
     tta_result_path: str = ""
-    # Path to pre-computed affinity prediction HDF5 (dataset "main").
-    # When set, skips model inference — loads and decodes directly.
-    saved_prediction_path: str = ""
-    # Path to save decoded instance segmentation (separate from raw prediction).
-    decoding_path: str = ""
     prediction_transform: PredictionTransformConfig = field(
         default_factory=PredictionTransformConfig
     )
     save_prediction: SavePredictionConfig = field(default_factory=SavePredictionConfig)
-    postprocessing: PostprocessingConfig = field(default_factory=PostprocessingConfig)
     memory_cleanup: InferenceMemoryCleanupConfig = field(
         default_factory=InferenceMemoryCleanupConfig
     )