v1.1.1 - Bug Fixes

Author: pscheri

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file.
 
+## 1.1.1 - 2024-10-25
+### Bug Fixes
+- Fixed a typo on the CDK Stack preventing new deployments.
+- Fixed an issue with the `sync_files` lambda function while comparing timestamps. For now the `safe_time_compare` feature is disabled until a better method can be implemented.
+
 ## 1.1.0 - 2024-10-25
 ### Added features
 - Added the ability to define an optional KMS Key for the target buckets in the configuration files. This allows you to set up a default encryption with KMS on the target buckets.

transfer_sync_service/lambda/sync_files/sync_files.py

@@ -164,28 +164,32 @@ def should_transfer_file(file: Dict[str, Any], first_copy: bool, safe_time_compa
     # If it's the first copy, transfer all files
     if first_copy:
         return True
+    
+    #############################################################################################################################
+    ## Need to implement a better safe_time_compare method to avoid cost duplication. Disabled for now.                        ##
+    ## Only data updated in source SFTP between previous execution and S3 Object timestamp should be copied.                   ##
+    ## Also need to add a feature flag to allow the user to decide if deleted files in target S3 Bucket needs to be re-copied. ##
+    #############################################################################################################################
 
     # For subsequent copies, check if the file is new or modified
-    if safe_time_compare < file_time:
-        try:
-            # Check if the file exists in the destination and compare modification times
-            obj = s3.head_object(
-                Bucket=event['SyncSetting']['LocalRepository']['BucketName'],
-                Key=f"{event['SyncSetting']['LocalRepository']['Prefix']}{file['filePath']}"
-            )
-            if obj['LastModified'] < file_time:
-                logger.info(f"File {file['filePath']} has been modified since last copy.")
-                return True
-            else:
-                logger.info(f"File {file['filePath']} has not been modified since last copy.")
-                return False
-        except botocore.exceptions.ClientError:
-            # If the file doesn't exist in the destination, it should be transferred
-            logger.info(f"File {file['filePath']} has not been copied before.")
+    try:
+        # Check if the file exists in the destination and compare modification times
+        obj = s3.head_object(
+            Bucket=event['SyncSetting']['LocalRepository']['BucketName'],
+            Key=f"{event['SyncSetting']['LocalRepository']['Prefix']}{file['filePath']}"
+        )
+        if obj['LastModified'] < file_time:
+            logger.info(f"File {file['filePath']} has been modified since last copy.")
             return True
-    else:
-        logger.info(f"File {file['filePath']} is not new.")
-        return False
+        # elif obj['LastModified'] >= safe_time_compare:
+        #     logger.info(f"File {file['filePath']} has been modified since safe time.")
+        else:
+            logger.info(f"File {file['filePath']} has not been modified since last copy.")
+            return False
+    except botocore.exceptions.ClientError:
+        # If the file doesn't exist in the destination, it should be transferred
+        logger.info(f"File {file['filePath']} has not been copied before.")
+        return True
 
 def transfer_files(file_list: List[str], event: Dict[str, Any]) -> None:
     """
@@ -252,10 +256,10 @@ def calculate_safe_time_compare(schedule: str, start_time: datetime) -> datetime
     interval = (next_times[1] - next_times[0]).total_seconds() / 60  # in minutes
 
     if 1 <= interval <= 5:
-        n = 5
-    elif 5 < interval <= 60:
+        n = 3
+    elif 5 < interval <= 10:
         n = 2
-    elif 60 < interval <= 1440:  # 24 hours
+    elif 10 < interval <= 60:
         n = 1
     else:
         n = 0

transfer_sync_service/transfer_sync_service_stack.py

@@ -59,7 +59,7 @@ def install_package(package, target):
     if not os.path.exists('transfer_sync_service/lambda/sync_files/pyawscron'):
         print('Installing some dependencies for Lambda')
         install_package(
-            f'pyawscron~={solution_parameters['pyawscron_version']}',
+            f'pyawscron~={solution_parameters["pyawscron_version"]}',
             'transfer_sync_service/lambda/sync_files/'
         )
 
@@ -293,7 +293,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
         monitoring.add_large_header('Transfer Family Connectors')
 
         # Process configuration files
-        for file in self.files('./configuration/sftp/'):            
+        for file in self.files('./configuration/sftp/'):
             with open(f'./configuration/sftp/{file}', encoding='utf8') as service_config:
                 config = json.load(service_config)
                 print(f'Creating resources for {config["Name"]}...')
@@ -370,6 +370,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
                         iam.PolicyStatement(
                             actions=[
                                 's3:GetObjectAttributes',
+                                's3:GetObject',
                                 's3:PutObject',
                                 's3:PutObjectACL'
                             ],