Skip to content

Commit ba534be

Browse files
donglaiwdonglaiw
authored
Merge pull request #191 from PytorchConnectomics/codex/m5-datafactory-extract
Integrate refactor milestones into master + stabilize CI type checks
2 parents e4ef728 + 8280cd6 commit ba534be

40 files changed

Lines changed: 1528 additions & 2693 deletions

‎.github/mypy_changed.ini‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[mypy]
2+
python_version = 3.11
3+
ignore_missing_imports = True
4+
follow_imports = skip

‎.github/workflows/tests.yml‎

Lines changed: 81 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ jobs:
4343
runs-on: ubuntu-latest
4444
steps:
4545
- uses: actions/checkout@v3
46+
with:
47+
fetch-depth: 0
4648

4749
- name: Set up Python
4850
uses: actions/setup-python@v4
@@ -54,14 +56,89 @@ jobs:
5456
python -m pip install --upgrade pip
5557
pip install black flake8 isort mypy
5658
59+
- name: Collect changed Python files
60+
id: changed-py
61+
shell: bash
62+
run: |
63+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
64+
git fetch --no-tags --depth=1 origin "${{ github.base_ref }}"
65+
files=$(git diff --name-only --diff-filter=ACMRT "origin/${{ github.base_ref }}...HEAD" -- 'connectomics/**/*.py')
66+
else
67+
if git rev-parse --verify HEAD~1 >/dev/null 2>&1; then
68+
files=$(git diff --name-only --diff-filter=ACMRT HEAD~1..HEAD -- 'connectomics/**/*.py')
69+
else
70+
files=$(git ls-files 'connectomics/**/*.py')
71+
fi
72+
fi
73+
74+
echo "files<<EOF" >> "$GITHUB_OUTPUT"
75+
echo "$files" >> "$GITHUB_OUTPUT"
76+
echo "EOF" >> "$GITHUB_OUTPUT"
77+
5778
- name: Run black
58-
run: black --check connectomics/
79+
env:
80+
CHANGED_FILES: ${{ steps.changed-py.outputs.files }}
81+
run: |
82+
python - <<'PY'
83+
import os
84+
import subprocess
85+
import sys
86+
87+
files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f]
88+
if not files:
89+
print("No changed Python files under connectomics/. Skipping black.")
90+
sys.exit(0)
91+
92+
subprocess.check_call(["black", "--check", *files])
93+
PY
5994
6095
- name: Run flake8
61-
run: flake8 connectomics/ --max-line-length=100
96+
env:
97+
CHANGED_FILES: ${{ steps.changed-py.outputs.files }}
98+
run: |
99+
python - <<'PY'
100+
import os
101+
import subprocess
102+
import sys
103+
104+
files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f]
105+
if not files:
106+
print("No changed Python files under connectomics/. Skipping flake8.")
107+
sys.exit(0)
108+
109+
subprocess.check_call(["flake8", "--max-line-length=100", *files])
110+
PY
62111
63112
- name: Run isort
64-
run: isort --check connectomics/
113+
env:
114+
CHANGED_FILES: ${{ steps.changed-py.outputs.files }}
115+
run: |
116+
python - <<'PY'
117+
import os
118+
import subprocess
119+
import sys
120+
121+
files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f]
122+
if not files:
123+
print("No changed Python files under connectomics/. Skipping isort.")
124+
sys.exit(0)
125+
126+
subprocess.check_call(["isort", "--check", *files])
127+
PY
65128
66129
- name: Run mypy
67-
run: mypy connectomics/ --ignore-missing-imports
130+
env:
131+
CHANGED_FILES: ${{ steps.changed-py.outputs.files }}
132+
run: |
133+
python - <<'PY'
134+
import os
135+
import subprocess
136+
import sys
137+
138+
files = [f for f in os.environ.get("CHANGED_FILES", "").splitlines() if f]
139+
if not files:
140+
print("No changed Python files under connectomics/. Skipping mypy.")
141+
sys.exit(0)
142+
143+
subprocess.check_call(["mypy", "--config-file", ".github/mypy_changed.ini", *files])
144+
PY

‎connectomics/data/dataset/__init__.py‎

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,40 +10,44 @@
1010
.claude/INFERENCE_DESIGN.md for details.
1111
"""
1212

13+
# Dataset factory functions (builder pattern)
14+
from .build import (
15+
create_connectomics_dataset,
16+
create_tile_data_dicts_from_json,
17+
create_tile_dataset,
18+
create_volume_dataset,
19+
)
20+
21+
# Shared data-dict helpers
22+
from .data_dicts import (
23+
create_data_dicts_from_paths,
24+
create_volume_data_dicts,
25+
)
26+
1327
# MONAI base datasets
1428
from .dataset_base import (
15-
MonaiConnectomicsDataset,
1629
MonaiCachedConnectomicsDataset,
30+
MonaiConnectomicsDataset,
1731
MonaiPersistentConnectomicsDataset,
1832
)
1933

20-
# Volume datasets
21-
from .dataset_volume import (
22-
MonaiVolumeDataset,
23-
MonaiCachedVolumeDataset,
34+
# Multi-dataset utilities
35+
from .dataset_multi import (
36+
StratifiedConcatDataset,
37+
UniformConcatDataset,
38+
WeightedConcatDataset,
2439
)
2540

2641
# Tile datasets
2742
from .dataset_tile import (
28-
MonaiTileDataset,
2943
MonaiCachedTileDataset,
44+
MonaiTileDataset,
3045
)
3146

32-
# Multi-dataset utilities
33-
from .dataset_multi import (
34-
WeightedConcatDataset,
35-
StratifiedConcatDataset,
36-
UniformConcatDataset,
37-
)
38-
39-
# Dataset factory functions (builder pattern)
40-
from .build import (
41-
create_data_dicts_from_paths,
42-
create_volume_data_dicts,
43-
create_tile_data_dicts_from_json,
44-
create_connectomics_dataset,
45-
create_volume_dataset,
46-
create_tile_dataset,
47+
# Volume datasets
48+
from .dataset_volume import (
49+
MonaiCachedVolumeDataset,
50+
MonaiVolumeDataset,
4751
)
4852

4953
__all__ = [

‎connectomics/data/dataset/build.py‎

Lines changed: 10 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,26 @@
1111

1212
from __future__ import annotations
1313

14-
from typing import Any, Dict, List, Optional, Sequence, Tuple, TYPE_CHECKING, Union
14+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union
15+
1516
from monai.transforms import Compose
1617

18+
from .data_dicts import (
19+
create_data_dicts_from_paths,
20+
create_volume_data_dicts,
21+
)
1722
from .dataset_base import (
18-
MonaiConnectomicsDataset,
1923
MonaiCachedConnectomicsDataset,
24+
MonaiConnectomicsDataset,
2025
MonaiPersistentConnectomicsDataset,
2126
)
2227
from .dataset_tile import (
23-
MonaiTileDataset,
2428
MonaiCachedTileDataset,
29+
MonaiTileDataset,
2530
)
2631

2732
if TYPE_CHECKING:
28-
from .dataset_volume import MonaiVolumeDataset, MonaiCachedVolumeDataset
33+
from .dataset_volume import MonaiCachedVolumeDataset, MonaiVolumeDataset
2934

3035

3136
__all__ = [
@@ -40,78 +45,6 @@
4045
]
4146

4247

43-
# ============================================================================
44-
# Data Dictionary Creation
45-
# ============================================================================
46-
47-
48-
def create_data_dicts_from_paths(
49-
image_paths: List[str],
50-
label_paths: Optional[List[str]] = None,
51-
mask_paths: Optional[List[str]] = None,
52-
) -> List[Dict[str, str]]:
53-
"""
54-
Create MONAI-style data dictionaries from file paths.
55-
56-
Args:
57-
image_paths: List of image file paths
58-
label_paths: Optional list of label file paths
59-
mask_paths: Optional list of mask file paths
60-
61-
Returns:
62-
List of dictionaries with 'image', 'label', and/or 'mask' keys
63-
64-
Examples:
65-
>>> image_paths = ['img1.h5', 'img2.h5']
66-
>>> label_paths = ['lbl1.h5', 'lbl2.h5']
67-
>>> data_dicts = create_data_dicts_from_paths(image_paths, label_paths)
68-
>>> # [{'image': 'img1.h5', 'label': 'lbl1.h5'}, ...]
69-
"""
70-
data_dicts = []
71-
72-
for i, image_path in enumerate(image_paths):
73-
data_dict = {"image": image_path}
74-
75-
if label_paths is not None:
76-
data_dict["label"] = label_paths[i]
77-
78-
if mask_paths is not None:
79-
data_dict["mask"] = mask_paths[i]
80-
81-
data_dicts.append(data_dict)
82-
83-
return data_dicts
84-
85-
86-
def create_volume_data_dicts(
87-
image_paths: List[str],
88-
label_paths: Optional[List[str]] = None,
89-
mask_paths: Optional[List[str]] = None,
90-
) -> List[Dict[str, str]]:
91-
"""
92-
Create MONAI data dictionaries for volume datasets.
93-
94-
This is a convenience wrapper around create_data_dicts_from_paths
95-
for volume-specific use cases.
96-
97-
Args:
98-
image_paths: List of image volume file paths
99-
label_paths: Optional list of label volume file paths
100-
mask_paths: Optional list of valid mask file paths
101-
102-
Returns:
103-
List of MONAI-style data dictionaries
104-
105-
Examples:
106-
>>> data_dicts = create_volume_data_dicts(['vol1.tif'], ['lbl1.tif'])
107-
"""
108-
return create_data_dicts_from_paths(
109-
image_paths=image_paths,
110-
label_paths=label_paths,
111-
mask_paths=mask_paths,
112-
)
113-
114-
11548
def create_tile_data_dicts_from_json(
11649
volume_json: str,
11750
label_json: Optional[str] = None,
@@ -403,7 +336,7 @@ def create_volume_dataset(
403336
... )
404337
"""
405338
# Lazy import to avoid circular dependency during module import
406-
from .dataset_volume import MonaiVolumeDataset, MonaiCachedVolumeDataset
339+
from .dataset_volume import MonaiCachedVolumeDataset, MonaiVolumeDataset
407340

408341
if dataset_type == "cached":
409342
return MonaiCachedVolumeDataset(

‎connectomics/data/dataset/data_dicts.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
"""Shared helpers for constructing MONAI-style dataset dictionaries."""
2+
3+
from __future__ import annotations
4+
5+
from typing import Dict, List, Optional
6+
7+
__all__ = [
8+
"create_data_dicts_from_paths",
9+
"create_volume_data_dicts",
10+
]
11+
12+
13+
def create_data_dicts_from_paths(
14+
image_paths: List[str],
15+
label_paths: Optional[List[str]] = None,
16+
mask_paths: Optional[List[str]] = None,
17+
) -> List[Dict[str, object]]:
18+
"""
19+
Create MONAI-style data dictionaries from file paths.
20+
21+
Args:
22+
image_paths: List of image file paths
23+
label_paths: Optional list of label file paths
24+
mask_paths: Optional list of mask file paths
25+
26+
Returns:
27+
List of dictionaries with 'image', 'label', and/or 'mask' keys
28+
"""
29+
data_dicts: List[Dict[str, object]] = []
30+
31+
for i, image_path in enumerate(image_paths):
32+
data_dict: Dict[str, object] = {"image": image_path}
33+
34+
if label_paths is not None:
35+
data_dict["label"] = label_paths[i]
36+
37+
if mask_paths is not None:
38+
data_dict["mask"] = mask_paths[i]
39+
40+
data_dicts.append(data_dict)
41+
42+
return data_dicts
43+
44+
45+
def create_volume_data_dicts(
46+
image_paths: List[str],
47+
label_paths: Optional[List[str]] = None,
48+
mask_paths: Optional[List[str]] = None,
49+
) -> List[Dict[str, object]]:
50+
"""
51+
Create MONAI data dictionaries for volume datasets.
52+
53+
This is a convenience wrapper around ``create_data_dicts_from_paths``
54+
for volume-specific use cases.
55+
"""
56+
return create_data_dicts_from_paths(
57+
image_paths=image_paths,
58+
label_paths=label_paths,
59+
mask_paths=mask_paths,
60+
)

‎connectomics/data/dataset/dataset_volume.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,16 @@
66
"""
77

88
from __future__ import annotations
9+
910
from typing import List, Optional, Tuple
1011

1112
from monai.data import CacheDataset
12-
from monai.transforms import Compose, RandSpatialCropd, CenterSpatialCropd
13+
from monai.transforms import CenterSpatialCropd, Compose, RandSpatialCropd
1314
from monai.utils import ensure_tuple_rep
1415

15-
from .dataset_base import MonaiConnectomicsDataset
16-
from .build import create_data_dicts_from_paths
1716
from ..io.monai_transforms import LoadVolumed
17+
from .data_dicts import create_data_dicts_from_paths
18+
from .dataset_base import MonaiConnectomicsDataset
1819

1920

2021
class MonaiVolumeDataset(MonaiConnectomicsDataset):

0 commit comments

Comments
 (0)