diff --git a/.changes/next-release/enhancement-cloudformation-37336.json b/.changes/next-release/enhancement-cloudformation-37336.json new file mode 100644 index 000000000000..75965215bb21 --- /dev/null +++ b/.changes/next-release/enhancement-cloudformation-37336.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "``cloudformation``", + "description": "High-level Cloudformation commands such as ``deploy`` and ``package`` will now fallback to SHA256 for checksumming if MD5 isn't present on the host" +} diff --git a/awscli/customizations/s3uploader.py b/awscli/customizations/s3uploader.py index 3b567d1e6909..e1374a0e8a59 100644 --- a/awscli/customizations/s3uploader.py +++ b/awscli/customizations/s3uploader.py @@ -11,7 +11,6 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import hashlib import logging import os import sys @@ -19,6 +18,7 @@ import botocore import botocore.exceptions +from botocore.compat import get_md5 from s3transfer.manager import TransferManager from s3transfer.subscribers import BaseSubscriber @@ -27,6 +27,16 @@ LOG = logging.getLogger(__name__) +def _get_checksum(): + hashlib_params = {"usedforsecurity": False} + try: + checksum = get_md5(**hashlib_params) + except botocore.exceptions.MD5UnavailableError: + import hashlib + checksum = hashlib.sha256(**hashlib_params) + return checksum + + class NoSuchBucketError(Exception): def __init__(self, **kwargs): msg = self.fmt.format(**kwargs) @@ -134,7 +144,7 @@ def upload(self, file_name, remote_path): def upload_with_dedup(self, file_name, extension=None): """ - Makes and returns name of the S3 object based on the file's MD5 sum + Makes and returns name of the S3 object based on the file's checksum :param file_name: file to upload :param extension: String of file extension to append to the object @@ -146,8 +156,8 @@ def upload_with_dedup(self, file_name, extension=None): # and re-upload only if necessary. So the template points to same file # in multiple places, this will upload only once - filemd5 = self.file_checksum(file_name) - remote_path = filemd5 + file_checksum = self.file_checksum(file_name) + remote_path = file_checksum if extension: remote_path = remote_path + "." + extension @@ -175,7 +185,7 @@ def make_url(self, obj_path): def file_checksum(self, file_name): with open(file_name, "rb") as file_handle: - md5 = hashlib.md5() + checksum = _get_checksum() # Read file in chunks of 4096 bytes block_size = 4096 @@ -185,13 +195,13 @@ def file_checksum(self, file_name): buf = file_handle.read(block_size) while len(buf) > 0: - md5.update(buf) + checksum.update(buf) buf = file_handle.read(block_size) # Restore file cursor's position file_handle.seek(curpos) - return md5.hexdigest() + return checksum.hexdigest() def to_path_style_s3_url(self, key, version=None): """ diff --git a/awscli/examples/cloudformation/_package_description.rst b/awscli/examples/cloudformation/_package_description.rst index 21566ac96c5d..2eff27560b1c 100644 --- a/awscli/examples/cloudformation/_package_description.rst +++ b/awscli/examples/cloudformation/_package_description.rst @@ -49,8 +49,8 @@ current working directory. The exception is ``AWS::ApiGateway::RestApi``; if you don't specify a ``BodyS3Location``, this command will not upload an artifact to S3. Before the command uploads artifacts, it checks if the artifacts are already -present in the S3 bucket to prevent unnecessary uploads. The command uses MD5 -checksums to compare files. If the values match, the command doesn't upload the -artifacts. Use the ``--force-upload flag`` to skip this check and always upload the -artifacts. +present in the S3 bucket to prevent unnecessary uploads. If the values match, the +command doesn't upload the artifacts. Use the ``--force-upload flag`` to skip this +check and always upload the artifacts. The command uses MD5 checksums to compare +files by default. If MD5 is not available in the environment, a SHA256 checksum is used. diff --git a/tests/unit/customizations/test_s3uploader.py b/tests/unit/customizations/test_s3uploader.py index 36c0253a70f2..bc419cf77472 100644 --- a/tests/unit/customizations/test_s3uploader.py +++ b/tests/unit/customizations/test_s3uploader.py @@ -348,6 +348,27 @@ def test_file_checksum(self): finally: shutil.rmtree(tempdir) + @mock.patch("awscli.customizations.s3uploader.get_md5") + def test_file_checksum_fips_fallback(self, get_md5_mock): + num_chars = 4096*5 + data = ''.join(random.choice(string.ascii_uppercase) + for _ in range(num_chars)).encode('utf-8') + checksum = hashlib.sha256(usedforsecurity=False) + checksum.update(data) + expected_checksum = checksum.hexdigest() + + tempdir = tempfile.mkdtemp() + get_md5_mock.side_effect = botocore.exceptions.MD5UnavailableError() + try: + filename = os.path.join(tempdir, 'tempfile') + with open(filename, 'wb') as f: + f.write(data) + + actual_checksum = self.s3uploader.file_checksum(filename) + self.assertEqual(expected_checksum, actual_checksum) + finally: + shutil.rmtree(tempdir) + def test_make_url(self): path = "Hello/how/are/you" expected = f"s3://{self.bucket_name}/{path}"