[part 1] Update all bundles to Huggingface and fix missing dependencies (#740)

Part of #739 

### Description
This PR is used to:
1. update a part of bundle versions to enable huggingface hosting.
2. update premerge CI test scripts to resolve dependency conflicts (for
example, `requirements-dev.txt` has conflicts with some bundles'
requirements)
3. fix missing dependencies issues on some bundles

### Status
**Ready**

### Please ensure all the checkboxes:
<!--- Put an `x` in all the boxes that apply, and remove the not
applicable items -->
- [x] Codeformat tests passed locally by running `./runtests.sh
--codeformat`.
- [ ] In-line docstrings updated.
- [ ] Update `version` and `changelog` in `metadata.json` if changing an
existing bundle.
- [ ] Please ensure the naming rules in config files meet our
requirements (please refer to: `CONTRIBUTING.md`).
- [ ] Ensure versions of packages such as `monai`, `pytorch` and `numpy`
are correct in `metadata.json`.
- [ ] Descriptions should be consistent with the content, such as
`eval_metrics` of the provided weights and TorchScript modules.
- [ ] Files larger than 25MB are excluded and replaced by providing
download links in `large_file.yml`.
- [ ] Avoid using path that contains personal information within config
files (such as use `/home/your_name/` for `"bundle_root"`).

---------

Signed-off-by: Yiheng Wang <vennw@nvidia.com>

Author: yiheng-wang-nv

.gitignore

@@ -131,4 +131,5 @@ temp/
 *.zip
 models/*/models/*
 models/*/output/*
+models/*/eval/*
 hf_models/*/eval/*

ci/bundle_custom_data.py

@@ -58,7 +58,7 @@
 # This dict is used for our CI tests to install required dependencies that cannot be installed by `pip install` directly.
 # If a bundle has this kind of dependencies, please add the bundle name (key), and the path of the install script (value)
 # into the dict.
-install_dependency_dict = {"maisi_ct_generative": "ci/install_scripts/install_maisi_ct_generative_dependency.sh"}
+install_dependency_dict = {}
 
 # This list is used for our CI tests to determine whether a bundle supports TensorRT export. Related
 # test will be employed for bundles in the dict.

ci/download_latest_bundle.py

@@ -14,23 +14,14 @@
 import os
 
 from monai.bundle import download
-from utils import get_checksum, get_hash_func, get_latest_version, get_version_checksum
+from utils import get_latest_version
 
 
 def download_latest_bundle(bundle_name: str, models_path: str, download_path: str):
     model_info_path = os.path.join(models_path, "model_info.json")
     version = get_latest_version(bundle_name=bundle_name, model_info_path=model_info_path)
-    checksum = get_version_checksum(bundle_name=bundle_name, version=version, model_info_path=model_info_path)
     download(name=bundle_name, source="monaihosting", version=version, bundle_dir=download_path)
 
-    # verify checksum
-    hash_func = get_hash_func(hash_type="sha1")
-    bundle_zip_name = f"{bundle_name}_v{version}.zip"
-    downloaded_checksum = get_checksum(dst_path=os.path.join(download_path, bundle_zip_name), hash_func=hash_func)
-    if checksum != downloaded_checksum:
-        raise ValueError(f"Checksum of {bundle_zip_name} is not correct.")
-    print(f"Checksum of downloaded bundle {bundle_name} is correct.")
-
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="")

ci/get_bundle_requirements.py

@@ -53,7 +53,7 @@ def increment_version(version):
     return result_version
 
 
-def get_requirements(bundle, models_path):
+def get_requirements(bundle, models_path, requirements_file):
     """
     This function is used to produce a requirements txt file, and print a string
     which shows the filename. The printed string can be used in shell scripts.
@@ -86,14 +86,12 @@ def get_requirements(bundle, models_path):
                     else:
                         if "pytorch_version" in metadata.keys():
                             # remove torch from libs
-                            libs = [lib for lib in libs if "torch" not in lib]
+                            libs = [lib for lib in libs if "torch" not in lib or lib == "pytorch-ignite"]
 
         if len(libs) > 0:
-            requirements_file_name = f"requirements_{bundle}.txt"
-            with open(requirements_file_name, "w") as f:
+            with open(requirements_file, "w") as f:
                 for line in libs:
                     f.write(f"{line}\n")
-            print(requirements_file_name)
 
 
 def get_install_script(bundle):
@@ -109,11 +107,13 @@ def get_install_script(bundle):
     parser.add_argument("-b", "--b", type=str, help="bundle name.")
     parser.add_argument("-p", "--p", type=str, default="models", help="models path.")
     parser.add_argument("--get_script", type=bool, default=False, help="whether to get the install script.")
+    parser.add_argument("--requirements_file", type=str, help="output filename for requirements.")
     args = parser.parse_args()
     bundle = args.b
     models_path = args.p
     get_script = args.get_script
     if get_script is True:
         get_install_script(bundle)
     else:
-        get_requirements(bundle, models_path)
+        requirements_file = args.requirements_file
+        get_requirements(bundle, models_path, requirements_file)

ci/run_premerge_cpu.sh

@@ -49,6 +49,8 @@ verify_bundle() {
     done
     echo 'Run verify bundle...'
     pip install -r requirements.txt
+    # install extra dependencies for get changed bundle
+    pip install jsonschema gdown pyyaml
     head_ref=$(git rev-parse HEAD)
     git fetch origin dev $head_ref
     # achieve all changed files in 'models'
@@ -66,18 +68,19 @@ verify_bundle() {
                 if is_excluded "$bundle"; then
                     echo "skip '$bundle' cpu premerge tests."
                 else
-                    pip install -r requirements-dev.txt
                     # get required libraries according to the bundle's metadata file
-                    requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+                    requirements_file="requirements_$bundle.txt"
+                    python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
                     # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
                     if [ $ALLOW_MONAI_RC = true ]; then
                         include_pre_release="--pre"
                     else
                         include_pre_release=""
                     fi
-                    if [ ! -z "$requirements" ]; then
+                    # Check if the requirements file exists and is not empty
+                    if [ -s "$requirements_file" ]; then
                         echo "install required libraries for bundle: $bundle"
-                        pip install $include_pre_release -r "$requirements"
+                        pip install $include_pre_release -r "$requirements_file"
                     fi
                     # verify bundle
                     python $(pwd)/ci/verify_bundle.py -b "$bundle" -m "min"  # min tests on cpu

ci/run_premerge_gpu.sh

@@ -31,15 +31,18 @@ fi
 
 init_venv() {
     if [ ! -d "model_zoo_venv" ]; then  # Check if the venv directory does not exist
-        echo "initializing pip environment: $1"
+        echo "initializing pip environment"
         python -m venv model_zoo_venv
         source model_zoo_venv/bin/activate
         pip install --upgrade pip wheel
-        pip install -r $1
+        pip install jsonschema gdown pyyaml parameterized fire
         export PYTHONPATH=$PWD
     else
         echo "Virtual environment model_zoo_venv already exists. Activating..."
         source model_zoo_venv/bin/activate
+        pip install --upgrade pip wheel
+        pip install jsonschema gdown pyyaml parameterized fire
+        export PYTHONPATH=$PWD
     fi
 }
 
@@ -54,9 +57,8 @@ remove_venv() {
 }
 
 set_local_env() {
-    echo "set local pip environment: $1"
+    echo "set local pip environment"
     pip install --upgrade pip wheel
-    pip install -r $1
     export PYTHONPATH=$PWD
 }
 
@@ -75,24 +77,20 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            init_venv
+            # Check if the requirements file exists and is not empty
+            if [ -s "$requirements_file" ]; then
                 echo "install required libraries for bundle: $bundle"
-                pip install $include_pre_release -r "$requirements"
+                pip install $include_pre_release -r "$requirements_file"
             fi
             # get extra install script if exists
             extra_script=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --get_script True)

ci/run_regular_tests_cpu.sh

@@ -34,21 +34,23 @@ verify_release_bundle() {
     echo 'Run verify bundle...'
     # get all bundles
     download_path="download"
-    pip install -r requirements-dev.txt
     pip install git+https://github.com/Project-MONAI/MONAI.git@dev  # project-monai/model-zoo issue #505
+    pip install jsonschema gdown
     # download bundle from releases
     python $(pwd)/ci/download_latest_bundle.py --b "$bundle" --models_path $(pwd)/models --p "$download_path"
     # get required libraries according to the bundle's metadata file
-    requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --p "$download_path")
+    requirements_file="requirements_$bundle.txt"
+    python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
     # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
     if [ $ALLOW_MONAI_RC = true ]; then
         include_pre_release="--pre"
     else
         include_pre_release=""
     fi
-    if [ ! -z "$requirements" ]; then
+    # Check if the requirements file exists and is not empty
+    if [ -s "$requirements_file" ]; then
         echo "install required libraries for bundle: $bundle"
-        pip install $include_pre_release -r "$requirements"
+        pip install $include_pre_release -r "$requirements_file"
     fi
     # verify bundle
     python $(pwd)/ci/verify_bundle.py -b "$bundle" -p "$download_path" -m "regular"  # regular tests on cpu

ci/utils.py

@@ -17,11 +17,11 @@
 import shutil
 from typing import List
 
-from huggingface_hub import HfApi
 from monai.apps.utils import download_url
 from monai.bundle.config_parser import ConfigParser
 from monai.utils import look_up_option, optional_import
 
+huggingface_hub, _ = optional_import("huggingface_hub")
 Github, _ = optional_import("github", name="Github")
 
 SUPPORTED_HASH_TYPES = {"md5": hashlib.md5, "sha1": hashlib.sha1, "sha256": hashlib.sha256, "sha512": hashlib.sha512}
@@ -178,7 +178,7 @@ def get_existing_bundle_list(model_info):
 
 
 def create_bundle_to_huggingface(bundle_name: str, org_name: str):
-    api = HfApi()
+    api = huggingface_hub.HfApi()
     try:
         _ = api.create_repo(repo_id=f"{org_name}/{bundle_name}", repo_type="model", private=False)
     except Exception as e:
@@ -189,7 +189,7 @@ def create_bundle_to_huggingface(bundle_name: str, org_name: str):
 
 
 def upload_version_to_huggingface(bundle_name: str, version: str, root_path: str, org_name: str):
-    api = HfApi()
+    api = huggingface_hub.HfApi()
 
     try:
         # if no file is changed, will skip uploading automatically

models/brain_image_synthesis_latent_diffusion_model/configs/metadata.json

@@ -1,15 +1,17 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
-    "version": "1.0.1",
+    "version": "1.0.2",
     "changelog": {
+        "1.0.2": "fix missing dependencies",
         "1.0.1": "update to huggingface hosting",
         "1.0.0": "Initial release"
     },
     "monai_version": "1.4.0",
     "pytorch_version": "2.5.1",
     "numpy_version": "1.26.4",
     "required_packages_version": {
-        "nibabel": "5.3.2"
+        "nibabel": "5.3.2",
+        "einops": "0.7.0"
     },
     "task": "Brain image synthesis",
     "description": "A generative model for creating high-resolution 3D brain MRI based on UK Biobank",

models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json

@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
-    "version": "1.1.2",
+    "version": "1.1.3",
     "changelog": {
+        "1.1.3": "update to huggingface hosting and fix missing dependencies",
         "1.1.2": "update issue for IgniteInfo",
         "1.1.1": "enable tensorrt",
         "1.1.0": "update to use monai 1.4, model ckpt not changed, rm GenerativeAI repo",
@@ -21,7 +22,10 @@
     "numpy_version": "1.24.4",
     "required_packages_version": {
         "nibabel": "5.2.1",
-        "lpips": "0.1.4"
+        "lpips": "0.1.4",
+        "tensorboard": "2.17.0",
+        "einops": "0.7.0",
+        "pytorch-ignite": "0.4.11"
     },
     "supported_apps": {},
     "name": "BraTS MRI axial slices latent diffusion generation",