[part 2] Update all bundles to Huggingface and fix missing dependencies

ci/get_bundle_requirements.py

@@ -19,8 +19,6 @@
 
 ALLOW_MONAI_RC = os.environ.get("ALLOW_MONAI_RC", "false").lower() in ("true", "1", "t", "y", "yes")
 
-SPECIAL_LIB_LIST = ["xformers"]
-
 
 def increment_version(version):
     """
@@ -81,12 +79,7 @@ def get_requirements(bundle, models_path, requirements_file):
             if package_key in metadata.keys():
                 optional_dict = metadata[package_key]
                 for name, version in optional_dict.items():
-                    if name not in SPECIAL_LIB_LIST:
-                        libs.append(f"{name}=={version}")
-                    else:
-                        if "pytorch_version" in metadata.keys():
-                            # remove torch from libs
-                            libs = [lib for lib in libs if "torch" not in lib or lib == "pytorch-ignite"]
+                    libs.append(f"{name}=={version}")
 
         if len(libs) > 0:
             with open(requirements_file, "w") as f:

ci/run_premerge_cpu.sh

@@ -41,6 +41,35 @@ is_excluded() {
     return 1 # Return false (1) if not excluded
 }
 
+init_venv() {
+    if [ ! -d "model_zoo_venv" ]; then  # Check if the venv directory does not exist
+        echo "initializing pip environment"
+        python -m venv model_zoo_venv
+        source model_zoo_venv/bin/activate
+        pip install --upgrade pip wheel
+        pip install --upgrade setuptools
+        pip install jsonschema gdown pyyaml parameterized fire
+        export PYTHONPATH=$PWD
+    else
+        echo "Virtual environment model_zoo_venv already exists. Activating..."
+        source model_zoo_venv/bin/activate
+        pip install --upgrade pip wheel
+        pip install --upgrade setuptools
+        pip install jsonschema gdown pyyaml parameterized fire
+        export PYTHONPATH=$PWD
+    fi
+}
+
+remove_venv() {
+    if [ -d "model_zoo_venv" ]; then  # Check if the venv directory exists
+        echo "Removing virtual environment..."
+        deactivate 2>/dev/null || true  # Deactivate venv, ignore errors if not activated
+        rm -rf model_zoo_venv  # Remove the venv directory
+    else
+        echo "Virtual environment not found. Skipping removal."
+    fi
+}
+
 verify_bundle() {
     for dir in /opt/hostedtoolcache/*; do
         if [[ $dir != "/opt/hostedtoolcache/Python" ]]; then
@@ -77,13 +106,15 @@ verify_bundle() {
                     else
                         include_pre_release=""
                     fi
+                    init_venv
                     # Check if the requirements file exists and is not empty
                     if [ -s "$requirements_file" ]; then
                         echo "install required libraries for bundle: $bundle"
                         pip install $include_pre_release -r "$requirements_file"
                     fi
                     # verify bundle
                     python $(pwd)/ci/verify_bundle.py -b "$bundle" -m "min"  # min tests on cpu
+                    remove_venv
                 fi
             done
         else

ci/run_premerge_gpu.sh

@@ -35,12 +35,14 @@ init_venv() {
         python -m venv model_zoo_venv
         source model_zoo_venv/bin/activate
         pip install --upgrade pip wheel
+        pip install --upgrade setuptools
         pip install jsonschema gdown pyyaml parameterized fire
         export PYTHONPATH=$PWD
     else
         echo "Virtual environment model_zoo_venv already exists. Activating..."
         source model_zoo_venv/bin/activate
         pip install --upgrade pip wheel
+        pip install --upgrade setuptools
         pip install jsonschema gdown pyyaml parameterized fire
         export PYTHONPATH=$PWD
     fi
@@ -56,12 +58,6 @@ remove_venv() {
     fi
 }
 
-set_local_env() {
-    echo "set local pip environment"
-    pip install --upgrade pip wheel
-    export PYTHONPATH=$PWD
-}
-
 verify_bundle() {
     echo 'Run verify bundle...'
     head_ref=$(git rev-parse HEAD)
@@ -105,10 +101,7 @@ verify_bundle() {
                 test_cmd="torchrun $(pwd)/ci/unit_tests/runner.py --b \"$bundle\" --dist True"
             fi
             eval $test_cmd
-            # if not maisi_ct_generative, remove venv
-            if [ "$bundle" != "maisi_ct_generative" ]; then
-                remove_venv
-            fi
+            remove_venv
         done
         else
             echo "this pull request does not change any bundles, skip verify."

models/pancreas_ct_dints_segmentation/configs/metadata.json

@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.5.0",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.5.1",
     "changelog": {
+        "0.5.1": "update to huggingface hosting",
         "0.5.0": "use monai 1.4 and update large files",
         "0.4.9": "update to use monai 1.3.1",
         "0.4.8": "add load_pretrain flag for infer",
@@ -31,12 +32,15 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "fire": "0.6.0",
         "nibabel": "5.2.1",
         "pytorch-ignite": "0.4.11",
-        "PyYAML": "6.0.1"
+        "PyYAML": "6.0.1",
+        "scikit-learn": "1.5.1",
+        "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "Pancreas CT DiNTS segmentation",
     "task": "Neural architecture search on pancreas CT segmentation",
     "description": "Searched architectures for volumetric (3D) segmentation of the pancreas from CT image",

models/pathology_nuclei_classification/configs/metadata.json

@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.2.0",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.2.1",
     "changelog": {
+        "0.2.1": "update to huggingface hosting",
         "0.2.0": "update issue for IgniteInfo",
         "0.1.9": "update tensorrt benchmark results",
         "0.1.8": "enable tensorrt",
@@ -26,11 +27,16 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "nibabel": "5.2.1",
         "pytorch-ignite": "0.4.11",
-        "torchvision": "0.19.0"
+        "torchvision": "0.19.0",
+        "scipy": "1.13.1",
+        "scikit-learn": "1.5.1",
+        "tensorboard": "2.17.0",
+        "scikit-image": "0.23.2"
     },
+    "supported_apps": {},
     "name": "Pathology nuclei classification",
     "task": "Pathology Nuclei classification",
     "description": "A pre-trained model for Nuclei Classification within Haematoxylin & Eosin stained histology images",

models/pathology_nuclei_segmentation_classification/configs/metadata.json

@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_hovernet_20221124.json",
-    "version": "0.2.6",
+    "version": "0.2.7",
     "changelog": {
+        "0.2.7": "update to huggingface hosting",
         "0.2.6": "update tensorrt benchmark results",
         "0.2.5": "enable tensorrt",
         "0.2.4": "update to use monai 1.3.1",
@@ -28,7 +29,10 @@
         "torchvision": "0.19.0",
         "scipy": "1.13.1",
         "tqdm": "4.66.4",
-        "pillow": "10.4.0"
+        "pillow": "10.4.0",
+        "pytorch-ignite": "0.4.11",
+        "tensorboard": "2.17.0",
+        "nibabel": "5.2.1"
     },
     "name": "Nuclear segmentation and classification",
     "task": "Nuclear segmentation and classification",

models/pathology_nuclick_annotation/configs/metadata.json

@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.2.1",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.2.2",
     "changelog": {
+        "0.2.2": "update to huggingface hosting",
         "0.2.1": "update issue for IgniteInfo",
         "0.2.0": "use monai 1.4 and update large files",
         "0.1.9": "update to use monai 1.3.1",
@@ -27,11 +28,15 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "nibabel": "5.2.1",
         "pytorch-ignite": "0.4.11",
-        "torchvision": "0.19.0"
+        "torchvision": "0.19.0",
+        "scipy": "1.13.1",
+        "scikit-image": "0.23.2",
+        "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "Pathology nuclick annotation",
     "task": "Pathology Nuclick annotation",
     "description": "A pre-trained model for segmenting nuclei cells with user clicks/interactions",

models/pathology_tumor_detection/configs/metadata.json

@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.6.2",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.6.3",
     "changelog": {
+        "0.6.3": "update to huggingface hosting",
         "0.6.2": "enhance readme for nccl timout issue",
         "0.6.1": "fix multi-gpu issue",
         "0.6.0": "use monai 1.4 and update large files",
@@ -36,11 +37,14 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "cucim-cu12": "24.6.0",
         "pandas": "2.2.1",
-        "torchvision": "0.19.0"
+        "torchvision": "0.19.0",
+        "pytorch-ignite": "0.4.11",
+        "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "Pathology tumor detection",
     "task": "Pathology metastasis detection",
     "description": "A pre-trained model for metastasis detection on Camelyon 16 dataset.",

models/pediatric_abdominal_ct_segmentation/configs/metadata.json

@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.4.4",
+    "version": "0.4.5",
     "changelog": {
+        "0.4.5": "update to huggingface hosting",
         "0.4.4": "initial bundle assemblage."
     },
     "monai_version": "1.3.0",

models/prostate_mri_anatomy/configs/metadata.json

@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.3.4",
+    "version": "0.3.5",
     "changelog": {
+        "0.3.5": "update to huggingface hosting",
         "0.3.4": "support monai 1.4",
         "0.3.3": "add invertd transformation",
         "0.3.2": "add name tag",

models/renalStructures_CECT_segmentation/configs/metadata.json

@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.2.1",
+    "version": "0.2.2",
     "changelog": {
+        "0.2.2": "update to huggingface hosting",
         "0.2.1": "fix pytype error",
         "0.2.0": "set image_only to False",
         "0.1.0": "complete the model package"
@@ -14,7 +15,9 @@
         "pytorch-ignite": "0.4.11",
         "einops": "0.6.1",
         "fire": "0.5.0",
-        "torchvision": "0.14.1"
+        "torchvision": "0.14.1",
+        "tensorboard": "2.17.0",
+        "scipy": "1.13.1"
     },
     "name": "Segmentation of renal structures based on contrast computed tomography scans",
     "task": "Renal structures segmentation",

models/renalStructures_UNEST_segmentation/configs/metadata.json

@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.2.5",
+    "version": "0.2.6",
     "changelog": {
+        "0.2.6": "update to huggingface hosting",
         "0.2.5": "update large files",
         "0.2.4": "fix black 24.1 format error",
         "0.2.3": "update AddChanneld with EnsureChannelFirstd and remove meta_dict",
@@ -22,7 +23,8 @@
         "einops": "0.7.0",
         "fire": "0.6.0",
         "timm": "0.6.7",
-        "torchvision": "0.19.0"
+        "torchvision": "0.19.0",
+        "tensorboard": "2.17.0"
     },
     "name": "Renal structures UNEST segmentation",
     "task": "Renal segmentation",

models/segmentation_template/configs/metadata.json

@@ -1,16 +1,17 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.0.2",
+    "version": "0.0.3",
     "changelog": {
+        "0.0.3": "update to huggingface hosting",
         "0.0.2": "Minor train.yaml clarifications",
         "0.0.1": "Initial version"
     },
-    "monai_version": "1.2.0",
-    "pytorch_version": "2.0.1",
+    "monai_version": "1.4.0",
+    "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
     "optional_packages_version": {
-        "nibabel": "5.1.0",
-        "pytorch-ignite": "0.4.12"
+        "nibabel": "5.2.1",
+        "pytorch-ignite": "0.4.11"
     },
     "name": "Segmentation Template",
     "task": "Segmentation of randomly generated spheres in 3D images",

models/spleen_ct_segmentation/configs/metadata.json

@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.5.9",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.6.0",
     "changelog": {
+        "0.6.0": "update to huggingface hosting",
         "0.5.9": "use monai 1.4 and update large files",
         "0.5.8": "update to use monai 1.3.2",
         "0.5.7": "update to use monai 1.3.1",
@@ -40,10 +41,12 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "nibabel": "5.2.1",
-        "pytorch-ignite": "0.4.11"
+        "pytorch-ignite": "0.4.11",
+        "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "Spleen CT segmentation",
     "task": "Decathlon spleen segmentation",
     "description": "A pre-trained model for volumetric (3D) segmentation of the spleen from CT image",

models/spleen_deepedit_annotation/configs/metadata.json

@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.5.6",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.5.7",
     "changelog": {
+        "0.5.7": "update to huggingface hosting",
         "0.5.6": "use monai 1.4 and update large files",
         "0.5.5": "update to use monai 1.3.1",
         "0.5.4": "add load_pretrain flag for infer",
@@ -36,12 +37,15 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "itk": "5.4.0",
         "pytorch-ignite": "0.4.11",
         "scikit-image": "0.23.2",
-        "einops": "0.7.0"
+        "einops": "0.7.0",
+        "tensorboard": "2.17.0",
+        "nibabel": "5.2.1"
     },
+    "supported_apps": {},
     "name": "Spleen DeepEdit annotation",
     "task": "Decathlon spleen segmentation",
     "description": "This is a pre-trained model for 3D segmentation of the spleen organ from CT images using DeepEdit.",