Provide ETag to transfer manager to validate multipart copies (aws#9857)

Author: hssyoo

awscli/customizations/s3/s3handler.py

@@ -433,6 +433,7 @@ def can_submit(self, fileinfo):
 
     def _add_additional_subscribers(self, subscribers, fileinfo):
         subscribers.append(ProvideSizeSubscriber(fileinfo.size))
+        subscribers.append(ProvideETagSubscriber(fileinfo.etag))
         if self._should_inject_content_type():
             subscribers.append(ProvideCopyContentTypeSubscriber())
         if self._cli_params.get('is_move', False):

tests/functional/s3/test_cp_command.py

@@ -219,7 +219,11 @@ def test_website_redirect_ignore_paramfile(self):
 
     def test_metadata_copy(self):
         self.parsed_responses = [
-            {"ContentLength": "100", "LastModified": "00:00:00Z"},
+            {
+                "ContentLength": "100",
+                "LastModified": "00:00:00Z",
+                'ETag': '"foo-1"',
+            },
             {'ETag': '"foo-1"'},
         ]
         cmdline = ('%s s3://bucket/key.txt s3://bucket/key2.txt'
@@ -267,7 +271,11 @@ def test_metadata_copy_with_multipart_upload(self):
 
     def test_metadata_directive_copy(self):
         self.parsed_responses = [
-            {"ContentLength": "100", "LastModified": "00:00:00Z"},
+            {
+                "ContentLength": "100",
+                "LastModified": "00:00:00Z",
+                "ETag": "'foo-1'",
+            },
             {'ETag': '"foo-1"'},
         ]
         cmdline = ('%s s3://bucket/key.txt s3://bucket/key2.txt'
@@ -610,7 +618,11 @@ def test_cp_upload_large_file_with_sse_kms_and_key_id(self):
 
     def test_cp_copy_with_sse_kms_and_key_id(self):
         self.parsed_responses = [
-            {'ContentLength': 5, 'LastModified': '00:00:00Z'},  # HeadObject
+            {
+                'ContentLength': 5,
+                'LastModified': '00:00:00Z',
+                'ETag': '"foo"',
+            },  # HeadObject
             {}  # CopyObject
         ]
         cmdline = (
@@ -636,8 +648,11 @@ def test_cp_copy_with_sse_kms_and_key_id(self):
 
     def test_cp_copy_large_file_with_sse_kms_and_key_id(self):
         self.parsed_responses = [
-            {'ContentLength': 10 * (1024 ** 2),
-             'LastModified': '00:00:00Z'},  # HeadObject
+            {
+                'ContentLength': 10 * (1024 ** 2),
+                'LastModified': '00:00:00Z',
+                'ETag': '"foo"',
+            },  # HeadObject
             {'UploadId': 'foo'},  # CreateMultipartUpload
             {'CopyPartResult': {'ETag': '"foo"'}},  # UploadPartCopy
             {'CopyPartResult': {'ETag': '"foo"'}},  # UploadPartCopy
@@ -669,8 +684,11 @@ def test_cannot_use_recursive_with_stream(self):
 
     def test_upload_unicode_path(self):
         self.parsed_responses = [
-            {'ContentLength': 10,
-             'LastModified': '00:00:00Z'},  # HeadObject
+            {
+                'ContentLength': 10,
+                'LastModified': '00:00:00Z',
+                'ETag': '"foo"',
+            },  # HeadObject
             {'ETag': '"foo"'}  # PutObject
         ]
         command = u's3 cp s3://bucket/\u2603 s3://bucket/\u2713'
@@ -1128,11 +1146,11 @@ def test_multipart_copy(self):
                 self.upload_part_copy_request(
                     'sourcebucket', 'sourcekey', 'mybucket', 'mykey',
                     upload_id, PartNumber=mock.ANY, RequestPayer='requester',
-                    CopySourceRange=mock.ANY),
+                    CopySourceRange=mock.ANY, CopySourceIfMatch='"foo-1"'),
                 self.upload_part_copy_request(
                     'sourcebucket', 'sourcekey', 'mybucket', 'mykey',
                     upload_id, PartNumber=mock.ANY, RequestPayer='requester',
-                    CopySourceRange=mock.ANY),
+                    CopySourceRange=mock.ANY, CopySourceIfMatch='"foo-1"'),
                 self.complete_mpu_request(
                     'mybucket', 'mykey', upload_id, num_parts=2,
                     RequestPayer='requester')

tests/functional/s3/test_mv_command.py

@@ -48,7 +48,11 @@ def test_website_redirect_ignore_paramfile(self):
 
     def test_metadata_directive_copy(self):
         self.parsed_responses = [
-            {"ContentLength": "100", "LastModified": "00:00:00Z"},
+            {
+                "ContentLength": "100",
+                "LastModified": "00:00:00Z",
+                "ETag": '"foo-1"',
+            },
             {'ETag': '"foo-1"'},
             {'ETag': '"foo-2"'}
         ]

tests/functional/s3/test_sync_command.py

@@ -306,7 +306,8 @@ def test_copy_with_checksum_algorithm_update_sha1(self):
                         'Key': 'mykey',
                         'LastModified': '00:00:00Z',
                         'Size': 100,
-                        'ChecksumAlgorithm': 'SHA1'
+                        'ChecksumAlgorithm': 'SHA1',
+                        'ETag': 'foo'
                     }
                 ],
                 'CommonPrefixes': []

tests/unit/customizations/s3/test_s3handler.py

@@ -630,6 +630,7 @@ def test_submit(self):
         # Make sure the subscriber applied are of the correct type and order
         ref_subscribers = [
             ProvideSizeSubscriber,
+            ProvideETagSubscriber,
             ProvideCopyContentTypeSubscriber,
             CopyResultSubscriber
         ]
@@ -663,6 +664,7 @@ def test_submit_when_content_type_specified(self):
             copy_call_kwargs['extra_args'], {'ContentType': 'text/plain'})
         ref_subscribers = [
             ProvideSizeSubscriber,
+            ProvideETagSubscriber,
             CopyResultSubscriber
         ]
         actual_subscribers = copy_call_kwargs['subscribers']
@@ -680,6 +682,7 @@ def test_submit_when_no_guess_content_mime_type(self):
         copy_call_kwargs = self.transfer_manager.copy.call_args[1]
         ref_subscribers = [
             ProvideSizeSubscriber,
+            ProvideETagSubscriber,
             CopyResultSubscriber
         ]
         actual_subscribers = copy_call_kwargs['subscribers']
@@ -797,6 +800,7 @@ def test_submit_move_adds_delete_source_subscriber(self):
         self.transfer_request_submitter.submit(fileinfo)
         ref_subscribers = [
             ProvideSizeSubscriber,
+            ProvideETagSubscriber,
             ProvideCopyContentTypeSubscriber,
             DeleteSourceObjectSubscriber,
             CopyResultSubscriber,