Skip to content

Commit c5376f5

Browse files
yiheng-wang-nvyiheng-wang-nv
authored
[part 2] Update all bundles to Huggingface and fix missing dependencies (#743)
Fixes #739 . ### Description A few sentences describing the changes proposed in this pull request. ### Status **Ready** ### Please ensure all the checkboxes: <!--- Put an `x` in all the boxes that apply, and remove the not applicable items --> - [x] Codeformat tests passed locally by running `./runtests.sh --codeformat`. - [ ] In-line docstrings updated. - [ ] Update `version` and `changelog` in `metadata.json` if changing an existing bundle. - [ ] Please ensure the naming rules in config files meet our requirements (please refer to: `CONTRIBUTING.md`). - [ ] Ensure versions of packages such as `monai`, `pytorch` and `numpy` are correct in `metadata.json`. - [ ] Descriptions should be consistent with the content, such as `eval_metrics` of the provided weights and TorchScript modules. - [ ] Files larger than 25MB are excluded and replaced by providing download links in `large_file.yml`. - [ ] Avoid using path that contains personal information within config files (such as use `/home/your_name/` for `"bundle_root"`). --------- Signed-off-by: Yiheng Wang <vennw@nvidia.com>
1 parent 7103ddc commit c5376f5

22 files changed

Lines changed: 143 additions & 72 deletions

File tree

‎ci/get_bundle_requirements.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919

2020
ALLOW_MONAI_RC = os.environ.get("ALLOW_MONAI_RC", "false").lower() in ("true", "1", "t", "y", "yes")
2121

22-
SPECIAL_LIB_LIST = ["xformers"]
23-
2422

2523
def increment_version(version):
2624
"""
@@ -81,12 +79,7 @@ def get_requirements(bundle, models_path, requirements_file):
8179
if package_key in metadata.keys():
8280
optional_dict = metadata[package_key]
8381
for name, version in optional_dict.items():
84-
if name not in SPECIAL_LIB_LIST:
85-
libs.append(f"{name}=={version}")
86-
else:
87-
if "pytorch_version" in metadata.keys():
88-
# remove torch from libs
89-
libs = [lib for lib in libs if "torch" not in lib or lib == "pytorch-ignite"]
82+
libs.append(f"{name}=={version}")
9083

9184
if len(libs) > 0:
9285
with open(requirements_file, "w") as f:

‎ci/run_premerge_cpu.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,35 @@ is_excluded() {
4141
return 1 # Return false (1) if not excluded
4242
}
4343

44+
init_venv() {
45+
if [ ! -d "model_zoo_venv" ]; then # Check if the venv directory does not exist
46+
echo "initializing pip environment"
47+
python -m venv model_zoo_venv
48+
source model_zoo_venv/bin/activate
49+
pip install --upgrade pip wheel
50+
pip install --upgrade setuptools
51+
pip install jsonschema gdown pyyaml parameterized fire
52+
export PYTHONPATH=$PWD
53+
else
54+
echo "Virtual environment model_zoo_venv already exists. Activating..."
55+
source model_zoo_venv/bin/activate
56+
pip install --upgrade pip wheel
57+
pip install --upgrade setuptools
58+
pip install jsonschema gdown pyyaml parameterized fire
59+
export PYTHONPATH=$PWD
60+
fi
61+
}
62+
63+
remove_venv() {
64+
if [ -d "model_zoo_venv" ]; then # Check if the venv directory exists
65+
echo "Removing virtual environment..."
66+
deactivate 2>/dev/null || true # Deactivate venv, ignore errors if not activated
67+
rm -rf model_zoo_venv # Remove the venv directory
68+
else
69+
echo "Virtual environment not found. Skipping removal."
70+
fi
71+
}
72+
4473
verify_bundle() {
4574
for dir in /opt/hostedtoolcache/*; do
4675
if [[ $dir != "/opt/hostedtoolcache/Python" ]]; then
@@ -77,13 +106,15 @@ verify_bundle() {
77106
else
78107
include_pre_release=""
79108
fi
109+
init_venv
80110
# Check if the requirements file exists and is not empty
81111
if [ -s "$requirements_file" ]; then
82112
echo "install required libraries for bundle: $bundle"
83113
pip install $include_pre_release -r "$requirements_file"
84114
fi
85115
# verify bundle
86116
python $(pwd)/ci/verify_bundle.py -b "$bundle" -m "min" # min tests on cpu
117+
remove_venv
87118
fi
88119
done
89120
else

‎ci/run_premerge_gpu.sh‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,14 @@ init_venv() {
3535
python -m venv model_zoo_venv
3636
source model_zoo_venv/bin/activate
3737
pip install --upgrade pip wheel
38+
pip install --upgrade setuptools
3839
pip install jsonschema gdown pyyaml parameterized fire
3940
export PYTHONPATH=$PWD
4041
else
4142
echo "Virtual environment model_zoo_venv already exists. Activating..."
4243
source model_zoo_venv/bin/activate
4344
pip install --upgrade pip wheel
45+
pip install --upgrade setuptools
4446
pip install jsonschema gdown pyyaml parameterized fire
4547
export PYTHONPATH=$PWD
4648
fi
@@ -56,12 +58,6 @@ remove_venv() {
5658
fi
5759
}
5860

59-
set_local_env() {
60-
echo "set local pip environment"
61-
pip install --upgrade pip wheel
62-
export PYTHONPATH=$PWD
63-
}
64-
6561
verify_bundle() {
6662
echo 'Run verify bundle...'
6763
head_ref=$(git rev-parse HEAD)
@@ -105,10 +101,7 @@ verify_bundle() {
105101
test_cmd="torchrun $(pwd)/ci/unit_tests/runner.py --b \"$bundle\" --dist True"
106102
fi
107103
eval $test_cmd
108-
# if not maisi_ct_generative, remove venv
109-
if [ "$bundle" != "maisi_ct_generative" ]; then
110-
remove_venv
111-
fi
104+
remove_venv
112105
done
113106
else
114107
echo "this pull request does not change any bundles, skip verify."

‎models/pancreas_ct_dints_segmentation/configs/metadata.json‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
2-
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
3-
"version": "0.5.0",
2+
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
3+
"version": "0.5.1",
44
"changelog": {
5+
"0.5.1": "update to huggingface hosting",
56
"0.5.0": "use monai 1.4 and update large files",
67
"0.4.9": "update to use monai 1.3.1",
78
"0.4.8": "add load_pretrain flag for infer",
@@ -31,12 +32,15 @@
3132
"monai_version": "1.4.0",
3233
"pytorch_version": "2.4.0",
3334
"numpy_version": "1.24.4",
34-
"optional_packages_version": {
35+
"required_packages_version": {
3536
"fire": "0.6.0",
3637
"nibabel": "5.2.1",
3738
"pytorch-ignite": "0.4.11",
38-
"PyYAML": "6.0.1"
39+
"PyYAML": "6.0.1",
40+
"scikit-learn": "1.5.1",
41+
"tensorboard": "2.17.0"
3942
},
43+
"supported_apps": {},
4044
"name": "Pancreas CT DiNTS segmentation",
4145
"task": "Neural architecture search on pancreas CT segmentation",
4246
"description": "Searched architectures for volumetric (3D) segmentation of the pancreas from CT image",

‎models/pathology_nuclei_classification/configs/metadata.json‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
2-
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
3-
"version": "0.2.0",
2+
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
3+
"version": "0.2.1",
44
"changelog": {
5+
"0.2.1": "update to huggingface hosting",
56
"0.2.0": "update issue for IgniteInfo",
67
"0.1.9": "update tensorrt benchmark results",
78
"0.1.8": "enable tensorrt",
@@ -26,11 +27,16 @@
2627
"monai_version": "1.4.0",
2728
"pytorch_version": "2.4.0",
2829
"numpy_version": "1.24.4",
29-
"optional_packages_version": {
30+
"required_packages_version": {
3031
"nibabel": "5.2.1",
3132
"pytorch-ignite": "0.4.11",
32-
"torchvision": "0.19.0"
33+
"torchvision": "0.19.0",
34+
"scipy": "1.13.1",
35+
"scikit-learn": "1.5.1",
36+
"tensorboard": "2.17.0",
37+
"scikit-image": "0.23.2"
3338
},
39+
"supported_apps": {},
3440
"name": "Pathology nuclei classification",
3541
"task": "Pathology Nuclei classification",
3642
"description": "A pre-trained model for Nuclei Classification within Haematoxylin & Eosin stained histology images",

‎models/pathology_nuclei_segmentation_classification/configs/metadata.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
22
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_hovernet_20221124.json",
3-
"version": "0.2.6",
3+
"version": "0.2.7",
44
"changelog": {
5+
"0.2.7": "update to huggingface hosting",
56
"0.2.6": "update tensorrt benchmark results",
67
"0.2.5": "enable tensorrt",
78
"0.2.4": "update to use monai 1.3.1",
@@ -28,7 +29,10 @@
2829
"torchvision": "0.19.0",
2930
"scipy": "1.13.1",
3031
"tqdm": "4.66.4",
31-
"pillow": "10.4.0"
32+
"pillow": "10.4.0",
33+
"pytorch-ignite": "0.4.11",
34+
"tensorboard": "2.17.0",
35+
"nibabel": "5.2.1"
3236
},
3337
"name": "Nuclear segmentation and classification",
3438
"task": "Nuclear segmentation and classification",

‎models/pathology_nuclick_annotation/configs/metadata.json‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
2-
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
3-
"version": "0.2.1",
2+
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
3+
"version": "0.2.2",
44
"changelog": {
5+
"0.2.2": "update to huggingface hosting",
56
"0.2.1": "update issue for IgniteInfo",
67
"0.2.0": "use monai 1.4 and update large files",
78
"0.1.9": "update to use monai 1.3.1",
@@ -27,11 +28,15 @@
2728
"monai_version": "1.4.0",
2829
"pytorch_version": "2.4.0",
2930
"numpy_version": "1.24.4",
30-
"optional_packages_version": {
31+
"required_packages_version": {
3132
"nibabel": "5.2.1",
3233
"pytorch-ignite": "0.4.11",
33-
"torchvision": "0.19.0"
34+
"torchvision": "0.19.0",
35+
"scipy": "1.13.1",
36+
"scikit-image": "0.23.2",
37+
"tensorboard": "2.17.0"
3438
},
39+
"supported_apps": {},
3540
"name": "Pathology nuclick annotation",
3641
"task": "Pathology Nuclick annotation",
3742
"description": "A pre-trained model for segmenting nuclei cells with user clicks/interactions",

‎models/pathology_tumor_detection/configs/metadata.json‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
2-
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
3-
"version": "0.6.2",
2+
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
3+
"version": "0.6.3",
44
"changelog": {
5+
"0.6.3": "update to huggingface hosting",
56
"0.6.2": "enhance readme for nccl timout issue",
67
"0.6.1": "fix multi-gpu issue",
78
"0.6.0": "use monai 1.4 and update large files",
@@ -36,11 +37,14 @@
3637
"monai_version": "1.4.0",
3738
"pytorch_version": "2.4.0",
3839
"numpy_version": "1.24.4",
39-
"optional_packages_version": {
40+
"required_packages_version": {
4041
"cucim-cu12": "24.6.0",
4142
"pandas": "2.2.1",
42-
"torchvision": "0.19.0"
43+
"torchvision": "0.19.0",
44+
"pytorch-ignite": "0.4.11",
45+
"tensorboard": "2.17.0"
4346
},
47+
"supported_apps": {},
4448
"name": "Pathology tumor detection",
4549
"task": "Pathology metastasis detection",
4650
"description": "A pre-trained model for metastasis detection on Camelyon 16 dataset.",

‎models/pediatric_abdominal_ct_segmentation/configs/metadata.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
22
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
3-
"version": "0.4.4",
3+
"version": "0.4.5",
44
"changelog": {
5+
"0.4.5": "update to huggingface hosting",
56
"0.4.4": "initial bundle assemblage."
67
},
78
"monai_version": "1.3.0",

‎models/prostate_mri_anatomy/configs/metadata.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
22
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
3-
"version": "0.3.4",
3+
"version": "0.3.5",
44
"changelog": {
5+
"0.3.5": "update to huggingface hosting",
56
"0.3.4": "support monai 1.4",
67
"0.3.3": "add invertd transformation",
78
"0.3.2": "add name tag",

0 commit comments

Comments
 (0)