Validate downloaded file integrity and raise ValueError on hash mismatch

monai/apps/utils.py

@@ -220,17 +220,16 @@ def download_url(
         HTTPError: See urllib.request.urlretrieve.
         ContentTooShortError: See urllib.request.urlretrieve.
         IOError: See urllib.request.urlretrieve.
-        RuntimeError: When the hash validation of the ``url`` downloaded file fails.
-
+        ValueError: When the hash validation of the ``url`` downloaded file fails.
     """
     if not filepath:
         filepath = Path(".", _basename(url)).resolve()
         logger.info(f"Default downloading to '{filepath}'")
     filepath = Path(filepath)
     if filepath.exists():
         if not check_hash(filepath, hash_val, hash_type):
-            raise RuntimeError(
-                f"{hash_type} check of existing file failed: filepath={filepath}, expected {hash_type}={hash_val}."
+            raise ValueError(
+                f"{hash_type} hash check of existing file failed: filepath={filepath}, expected {hash_type}={hash_val}."
             )
         logger.info(f"File exists: {filepath}, skipped downloading.")
         return
@@ -260,18 +259,20 @@ def download_url(
                 raise RuntimeError(
                     f"Download of file from {url} to {filepath} failed due to network issue or denied permission."
                 )
+            if not check_hash(tmp_name, hash_val, hash_type):
+                raise ValueError(
+                    f"{hash_type} hash check of downloaded file failed: URL={url}, "
+                    f"filepath={filepath}, expected {hash_type}={hash_val}, "
+                    f"The file may be corrupted or tampered with. "
+                    "Please retry the download or verify the source."
+                )
             file_dir = filepath.parent
             if file_dir:
                 os.makedirs(file_dir, exist_ok=True)
             shutil.move(f"{tmp_name}", f"{filepath}")  # copy the downloaded to a user-specified cache.
     except (PermissionError, NotADirectoryError):  # project-monai/monai issue #3613 #3757 for windows
         pass
     logger.info(f"Downloaded: {filepath}")
-    if not check_hash(filepath, hash_val, hash_type):
-        raise RuntimeError(
-            f"{hash_type} check of downloaded file failed: URL={url}, "
-            f"filepath={filepath}, expected {hash_type}={hash_val}."
-        )
 
 
 def _extract_zip(filepath, output_dir):

tests/test_utils.py

@@ -79,7 +79,7 @@
     "unexpected EOF",  # incomplete download
     "network issue",
     "gdown dependency",  # gdown not installed
-    "md5 check",
+    "hash check",  # check hash value of downloaded file
     "limit",  # HTTP Error 503: Egress is over the account limit
     "authenticate",
     "timed out",  # urlopen error [Errno 110] Connection timed out
@@ -182,6 +182,37 @@ def skip_if_downloading_fails():
             raise unittest.SkipTest(f"Error while downloading: {rt_e}") from rt_e  # incomplete download
 
         raise rt_e
+    except ValueError as v_e:
+        if "hash check" in str(v_e):
+            raise unittest.SkipTest(f"Hash value error while downloading: {v_e}") from v_e
+
+        raise v_e
+
+
+SAMPLE_TIFF = "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/CMU-1.tiff"
+SAMPLE_TIFF_HASH = "73a7e89bc15576587c3d68e55d9bf92f09690280166240b48ff4b48230b13bcd"
+SAMPLE_TIFF_HASH_TYPE = "sha256"
+
+
+class TestDownloadUrl(unittest.TestCase):
+    """Exercise ``download_url`` success and hash-mismatch paths."""
+
+    def test_download_url(self):
+        """Download a sample TIFF and validate hash handling.
+
+        Raises:
+            ValueError: When the downloaded file's hash does not match.
+        """
+        with tempfile.TemporaryDirectory() as tempdir:
+            model_path = os.path.join(tempdir, "model.tiff")
+
+            with skip_if_downloading_fails():
+                download_url(
+                    url=SAMPLE_TIFF, filepath=model_path, hash_val=SAMPLE_TIFF_HASH, hash_type=SAMPLE_TIFF_HASH_TYPE
+                )
+            with self.assertRaises(ValueError):
+                # checking for wrong hash
+                download_url(filepath=model_path, hash_val="0" * 64, hash_type=SAMPLE_TIFF_HASH_TYPE)
 
 
 def test_pretrained_networks(network, input_param, device):

tests/testing_data/data_config.json

@@ -1,87 +1,87 @@
 {
     "images": {
         "wsi_generic_tiff": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/CMU-1.tiff",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/CMU-1.tiff",
             "hash_type": "sha256",
             "hash_val": "73a7e89bc15576587c3d68e55d9bf92f09690280166240b48ff4b48230b13bcd"
         },
         "wsi_aperio_svs": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/Aperio-CMU-1.svs",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/Aperio-CMU-1.svs",
             "hash_type": "sha256",
             "hash_val": "00a3d54482cd707abf254fe69dccc8d06b8ff757a1663f1290c23418c480eb30"
         },
         "favicon": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/favicon.ico.zip",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/favicon.ico.zip",
             "hash_type": "sha256",
             "hash_val": "3a3635c8d8adb81feebc5926b4106e8eb643a24a4be2a69a9d35f9a578acadb5"
         },
         "icon": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/icon.tar.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/icon.tar.gz",
             "hash_type": "sha256",
             "hash_val": "90f24cd8f20f3932624da95190ce384302261acf0ea15b358f7832e3b6becac0"
         },
         "mednist": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MedNIST.tar.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/MedNIST.tar.gz",
             "hash_type": "sha256",
             "hash_val": "f2f4881ff8799a170b10a403495f0ce0ad7486491901cde67a647e6627e7f916"
         },
         "Prostate_T2W_AX_1": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/Prostate_T2W_AX_1.nii",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/Prostate_T2W_AX_1.nii",
             "hash_type": "sha256",
             "hash_val": "a14231f539c0f365a5f83f2a046969a9b9870e56ffd126fd8e7242364d25938a"
         },
         "0000_t2_tse_tra_4": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/ProstateX-0000_t2_tse_tra_4.nii.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/ProstateX-0000_t2_tse_tra_4.nii.gz",
             "hash_type": "md5",
             "hash_val": "adb3f1c4db66a6481c3e4a2a3033c7d5"
         },
         "0000_ep2d_diff_tra_7": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/ProstateX-0000_ep2d_diff_tra_7.nii.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/ProstateX-0000_ep2d_diff_tra_7.nii.gz",
             "hash_type": "md5",
             "hash_val": "f12a11ad0ebb0b1876e9e010564745d2"
         },
         "ref_avg152T1_LR": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/avg152T1_LR_nifti.nii.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/avg152T1_LR_nifti.nii.gz",
             "hash_type": "sha256",
             "hash_val": "c01a50caa7a563158ecda43d93a1466bfc8aa939bc16b06452ac1089c54661c8"
         },
         "ref_avg152T1_RL": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/avg152T1_RL_nifti.nii.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/avg152T1_RL_nifti.nii.gz",
             "hash_type": "sha256",
             "hash_val": "8a731128dac4de46ccb2cc60d972b98f75a52f21fb63ddb040ca96f0aed8b51a"
         },
         "MNI152_T1_2mm": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MNI152_T1_2mm.nii.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/MNI152_T1_2mm.nii.gz",
             "hash_type": "sha256",
             "hash_val": "0585cd056bf5ccfb8bf97a5f6a66082d4e7caad525718fc11e40d80a827fcb92"
         },
         "MNI152_T1_2mm_strucseg": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MNI152_T1_2mm_strucseg.nii.gz",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/MNI152_T1_2mm_strucseg.nii.gz",
             "hash_type": "sha256",
             "hash_val": "eb4f1e596ca85aadaefc359d409fb9a3e27d733e6def04b996953b7c54bc26d4"
         },
         "copd1_highres_INSP_STD_COPD_img": {
-            "url": "https://data.kitware.com/api/v1/file/62a0f067bddec9d0c4175c5a/download",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/copd1_highres_INSP_STD_COPD_img.nii.gz",
             "hash_type": "sha512",
             "hash_val": "60193cd6ef0cf055c623046446b74f969a2be838444801bd32ad5bedc8a7eeecb343e8a1208769c9c7a711e101c806a3133eccdda7790c551a69a64b9b3701e9"
         },
         "copd1_highres_EXP_STD_COPD_img": {
-            "url": "https://data.kitware.com/api/v1/item/62a0f045bddec9d0c4175c44/download",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/copd1_highres_EXP_STD_COPD_img.nii.gz",
             "hash_type": "sha512",
             "hash_val": "841ef303958541474e66c2d1ccdc8b7ed17ba2f2681101307766b979a07979f2ec818ddf13791c3f1ac5a8ec3258d6ea45b692b4b4a838de9188602618972b6d"
         },
         "CT_2D_head_fixed": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/CT_2D_head_fixed.mha",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/CT_2D_head_fixed.mha",
             "hash_type": "sha256",
             "hash_val": "06f2ce6fbf6a59f0874c735555fcf71717f631156b1b0697c1752442f7fc1cc5"
         },
         "CT_2D_head_moving": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/CT_2D_head_moving.mha",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/CT_2D_head_moving.mha",
             "hash_type": "sha256",
             "hash_val": "a37c5fe388c38b3f4ac564f456277d09d3982eda58c4da05ead8ee2332360f47"
         },
         "DICOM_single": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/CT_DICOM_SINGLE.zip",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/CT_DICOM_SINGLE.zip",
             "hash_type": "sha256",
             "hash_val": "a41f6e93d2e3d68956144f9a847273041d36441da12377d6a1d5ae610e0a7023"
         },
@@ -93,69 +93,69 @@
     },
     "videos": {
         "endovis": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/d1_im.mp4",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/d1_im.mp4",
             "hash_type": "md5",
             "hash_val": "9b103c07326439b0ea376018d7189384"
         },
         "ultrasound": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/example_data_Ultrasound_Q000_04_tu_segmented_ultrasound_256.avi",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/example_data_Ultrasound_Q000_04_tu_segmented_ultrasound_256.avi",
             "hash_type": "md5",
             "hash_val": "f0755960cc4a08a958561cda9a79a157"
         }
     },
     "models": {
         "senet154-c7b49a05": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/senet154-c7b49a05.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/senet154-c7b49a05.pth",
             "hash_type": "sha256",
             "hash_val": "c7b49a056b98b0bed65b0237c27acdead655e599669215573d357ad337460413"
         },
         "se_resnet101-7e38fcc6": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet101-7e38fcc6.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/se_resnet101-7e38fcc6.pth",
             "hash_type": "sha256",
             "hash_val": "7e38fcc64eff3225a3ea4e6081efeb6087e8d5a61c204d94edc2ed1aab0b9d70"
         },
         "se_resnet152-d17c99b7": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet152-d17c99b7.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/se_resnet152-d17c99b7.pth",
             "hash_type": "sha256",
             "hash_val": "d17c99b703dcca2d2507ddfb68f72625a2f7e23ee64396eb992f1b2cf7e6bdc1"
         },
         "se_resnet50-ce0d4300": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet50-ce0d4300.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/se_resnet50-ce0d4300.pth",
             "hash_type": "sha256",
             "hash_val": "ce0d430017d3f4aa6b5658c72209f3bfffb060207fd26a2ef0b203ce592eba01"
         },
         "se_resnext101_32x4d-3b2fe3d8": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext101_32x4d-3b2fe3d8.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/se_resnext101_32x4d-3b2fe3d8.pth",
             "hash_type": "sha256",
             "hash_val": "3b2fe3d8acb8de7d5976c4baf518f24a0237509272a69366e816682d3e57b989"
         },
         "se_resnext50_32x4d-a260b3a4": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext50_32x4d-a260b3a4.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/se_resnext50_32x4d-a260b3a4.pth",
             "hash_type": "sha256",
             "hash_val": "a260b3a40f82dfe37c58d26a612bcf7bef0d27c6fed096226b0e4e9fb364168e"
         },
         "ssl_pretrained_weights": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/ssl_pretrained_weights.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/ssl_pretrained_weights.pth",
             "hash_type": "sha256",
             "hash_val": "c3564f40a6a051d3753a6d8fae5cc8eaf21ce8d82a9a3baf80748d15664055e8"
         },
         "decoder_only_transformer_monai_generative_weights": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/decoder_only_transformer.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/decoder_only_transformer.pth",
             "hash_type": "sha256",
             "hash_val": "f93de37d64d77cf91f3bde95cdf93d161aee800074c89a92aff9d5699120ec0d"
         },
         "diffusion_model_unet_monai_generative_weights": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/diffusion_model_unet.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/diffusion_model_unet.pth",
             "hash_type": "sha256",
             "hash_val": "0d2171b386902f5b4fd3e967b4024f63e353694ca45091b114970019d045beee"
         },
         "autoencoderkl_monai_generative_weights": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/autoencoderkl.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/autoencoderkl.pth",
             "hash_type": "sha256",
             "hash_val": "6e02c9540c51b16b9ba98b5c0c75d6b84b430afe9a3237df1d67a520f8d34184"
         },
         "controlnet_monai_generative_weights": {
-            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/controlnet.pth",
+            "url": "https://huggingface.co/datasets/MONAI/testing_data/resolve/main/controlnet.pth",
             "hash_type": "sha256",
             "hash_val": "cd100d0c69f47569ae5b4b7df653a1cb19f5e02eff1630db3210e2646fb1ab2e"
         }