v1.2.0 - Additional Tags support

Author: pscheri

CHANGELOG.md

@@ -2,8 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
+## 1.2.0 - 2024-11-07
+### Added features
+- Added the ability to define tags for every resource created by the solution. This can be configured using the `configuration/solution_parameters/parameters.json` file.
+
+### Changed
+- Re-implemented the `safe_time_compare` feature to better accommodate scheduling execution delays and avoid data transfer duplications.
+
 ## 1.1.1 - 2024-10-25
-### Bug Fixes
+### Bug Fixes
 - Fixed a typo on the CDK Stack preventing new deployments.
 - Fixed an issue with the `sync_files` lambda function while comparing timestamps. For now the `safe_time_compare` feature is disabled until a better method can be implemented.
 

README.md

@@ -193,6 +193,9 @@ This project is built using Python3 and CDK, before you start, make sure to have
 
 ## Deployment
 
+### Tagging resources
+By default the solution applies some native CDK and Name tags. If your deployment requires additional tags (for eg. Cost Allocation, Environment, Team, etc), you can now update the [solution parameters file](configuration/solution_parameters/parameters.json) by modifying the JSON for `additional_tags` with your specific Keys and Values.
+
 ### Permission Boundaries
 If you are enforcing the usage of IAM Permission Boundaries for IAM Roles created in the account, you can update the [solution parameters file](configuration/solution_parameters/parameters.json) and add the managed policy ARN you are using to the `permission_boundary_policy_arn` parameter.
 

configuration/solution_parameters/parameters.json

@@ -3,5 +3,9 @@
     "powertools_service_name": "transfer-sync-service",
     "powertools_log_level": "INFO",
     "boto_version": "1.35.47",
-    "pyawscron_version": "1.0.7"   
+    "pyawscron_version": "1.0.7",
+    "additional_tags": {
+        "Project": "transfer-sync-service",
+        "Environment": "dev"
+    }
 }

transfer_sync_service/lambda/sync_files/sync_files.py

@@ -164,32 +164,28 @@ def should_transfer_file(file: Dict[str, Any], first_copy: bool, safe_time_compa
     # If it's the first copy, transfer all files
     if first_copy:
         return True
-    
-    #############################################################################################################################
-    ## Need to implement a better safe_time_compare method to avoid cost duplication. Disabled for now.                        ##
-    ## Only data updated in source SFTP between previous execution and S3 Object timestamp should be copied.                   ##
-    ## Also need to add a feature flag to allow the user to decide if deleted files in target S3 Bucket needs to be re-copied. ##
-    #############################################################################################################################
 
     # For subsequent copies, check if the file is new or modified
-    try:
-        # Check if the file exists in the destination and compare modification times
-        obj = s3.head_object(
-            Bucket=event['SyncSetting']['LocalRepository']['BucketName'],
-            Key=f"{event['SyncSetting']['LocalRepository']['Prefix']}{file['filePath']}"
-        )
-        if obj['LastModified'] < file_time:
-            logger.info(f"File {file['filePath']} has been modified since last copy.")
+    if safe_time_compare < file_time:
+        try:
+            # Check if the file exists in the destination and compare modification times
+            obj = s3.head_object(
+                Bucket=event['SyncSetting']['LocalRepository']['BucketName'],
+                Key=f"{event['SyncSetting']['LocalRepository']['Prefix']}{file['filePath']}"
+            )
+            if obj['LastModified'] < file_time:
+                logger.info(f"File {file['filePath']} has been modified since last copy.")
+                return True
+            else:
+                logger.info(f"File {file['filePath']} has not been modified since last copy.")
+                return False
+        except botocore.exceptions.ClientError:
+            # If the file doesn't exist in the destination, it should be transferred
+            logger.info(f"File {file['filePath']} has not been copied before.")
             return True
-        # elif obj['LastModified'] >= safe_time_compare:
-        #     logger.info(f"File {file['filePath']} has been modified since safe time.")
-        else:
-            logger.info(f"File {file['filePath']} has not been modified since last copy.")
-            return False
-    except botocore.exceptions.ClientError:
-        # If the file doesn't exist in the destination, it should be transferred
-        logger.info(f"File {file['filePath']} has not been copied before.")
-        return True
+    else:
+        logger.info(f"File {file['filePath']} is not new.")
+        return False
 
 def transfer_files(file_list: List[str], event: Dict[str, Any]) -> None:
     """
@@ -245,27 +241,15 @@ def calculate_safe_time_compare(schedule: str, start_time: datetime) -> datetime
     Returns:
         datetime: The safe time to compare against.
     """
-    # Get the next 2 scheduled times
-    next_times = AWSCron.get_next_n_schedule(2, start_time, schedule)
-    
-    if len(next_times) < 2:
-        # If we can't get 2 next times, use the start time
-        return start_time
-    
-    # Calculate the interval between executions
-    interval = (next_times[1] - next_times[0]).total_seconds() / 60  # in minutes
 
-    if 1 <= interval <= 5:
-        n = 3
-    elif 5 < interval <= 10:
-        n = 2
-    elif 10 < interval <= 60:
-        n = 1
-    else:
-        n = 0
+    # Getting the previous 2 executions time based on the cron schedule
+    prev_times = AWSCron.get_prev_n_schedule(2, start_time, schedule)
+    
+    # Calculating the time gap between execution to accommodate execution delays safely
+    start_diff = (start_time - prev_times[0]).total_seconds()
+    expected_diff = (prev_times[0] - prev_times[1]).total_seconds()
 
-    if n > 0:
-        prev_times = AWSCron.get_prev_n_schedule(n, start_time, schedule)
-        return prev_times[-1] if prev_times else start_time
+    if expected_diff > start_diff:
+        return prev_times[1]
     else:
-        return start_time
+        return prev_times[0]

transfer_sync_service/transfer_sync_service_stack.py

@@ -4,6 +4,7 @@
     Stack,
     RemovalPolicy,
     CfnTag,
+    Tags,
     aws_s3 as s3,
     aws_sns as sns,
     aws_iam as iam,
@@ -29,6 +30,7 @@
 
 boto_version = solution_parameters['boto_version']
 permission_boundary_policy_arn = solution_parameters['permission_boundary_policy_arn']
+additional_tags = solution_parameters['additional_tags']
 
 class TransferSyncServiceStack(Stack):
     def install_package(package, target):
@@ -101,6 +103,10 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             )
             iam.PermissionsBoundary.of(self).apply(boundary_policy)
 
+        if additional_tags != '':
+            for key, value in additional_tags.items():
+                Tags.of(self).add(key=key,value=value)
+
         # Monitoring services
         # Create SNS topic for notifications
         notification_topic = sns.Topic(self, 'NotificationTopic')