datasync_in-cloud_transfer_quick_start_scheduler.yaml

---
AWSTemplateFormatVersion: 2010-09-09
Metadata:
  License: Apache-2.0
  Description: 'AWS DataSync In-cloud Transfer Quick Start Scheduler: A Quick Start template that
    creates a one-time or recurring schedule to transfer files between source and destination
    Amazon EFS file systems. These file systems could be in the same or different AWS regions.
    **WARNING**
    This template creates an Amazon EC2 Fleet (AWS DataSync agent on EC2), SSM Document,
    SSM Maintenance Windows, and AWS DataSync resources. To completely delete all resources created
    by this template, view the CloudFormation stack outputs and run the ouput
    "ViewDeleteDataSyncResourcesScript" SSM command in a terminal window. This wil generate the
    delete commands that you must copy, paste, and run in a terminal window. These commands, when
    executed, will delete all the AWS DataSync resources that were dynamically created by a script 
    executed on the DataSync agent EC2 instance at boot, as well as the CloudFormation stack. 
    You will be billed for the AWS resources used if you create a stack from this template.'
  AWS::CloudFormation::Interface:
    ParameterGroups:
    - Label:
        default: AWS DataSync EC2 agent
      Parameters:
        - Ec2KeyName
        - InstanceType
        - SourceSubnetId
        - SourceSecurityGroupId
    - Label:
        default: Source NFS location
      Parameters:
        - SourceDnsName
        - SourceSubdirectory
    - Label:
        default: Destination EFS location
      Parameters:
        - DestinationLocationRegion
        - DestinationEfsFilesystemId
        - DestinationSubnetId
        - DestinationSecurityGroupId
        - DestinationSubdirectory
    - Label:
        default: Task start frequency (schedule)
      Parameters:
        - TaskSchedule
    ParameterLabels:
      DestinationEfsFilesystemId:
        default: Destination EFS filesystem id
      DestinationLocationRegion:
        default: Destination location region
      DestinationSecurityGroupId:
        default: Destination security group id
      DestinationSubdirectory:
        default: Destination EFS subdirectory
      DestinationSubnetId:
        default: Destination subnet id
      Ec2KeyName:
        default: EC2 key pair
      InstanceType:
        default: Instance type
      SourceDnsName:
        default: Source NFS DNS name
      SourceSecurityGroupId:
        default: Source security group id
      SourceSubdirectory:
        default: Source NFS subdirectory
      SourceSubnetId:
        default: Source subnet id
      TaskSchedule:
        default: Schedule of task run
Parameters:
  DestinationEfsFilesystemId:
    Description: Destination EFS filesystem id.
    Type: String
  DestinationLocationRegion:
    AllowedValues:
    - us-east-1
    - us-east-2
    - us-west-1
    - us-west-2
    - eu-west-1
    - eu-central-1
    - ap-northeast-1
    - ap-northeast-2
    - ap-southeast-1
    - ap-southeast-2
    Description: Destination AWS region.
    Type: String 
  DestinationSecurityGroupId:
    AllowedPattern: ^(sg-)(?=[a-z0-9]*?)(?:.{8}|.{17})$
    Description: Destination EFS security group id.
    Type: String
  DestinationSubdirectory:
    Description: Destination EFS subdirectory.
    Type: String
    Default: "/"
  DestinationSubnetId:
    AllowedPattern: ^(subnet-)(?=[a-z0-9]*?)(?:.{8}|.{17})$
    Description: Destination EFS subnet id.
    Type: String    
  Ec2KeyName:
    Description: Name of an EC2 key pair.
    Type: AWS::EC2::KeyPair::KeyName
  InstanceType:
    AllowedValues:
    - c5.2xlarge
    - c5.4xlarge
    - c5.9xlarge
    - m5.2xlarge
    - m5.4xlarge
    - m5a.2xlarge
    - m5a.4xlarge
    Default: c5.2xlarge
    Description: Instance type
    Type: String
  SourceDnsName:
    Description: The DNS name of the source NFS file system.
    Type: String
  SourceSecurityGroupId:
    Description: Select the security group that has NFS (TCP 2049) network access to the source file system.
    Type: AWS::EC2::SecurityGroup::Id          
  SourceSubdirectory:
    Default: "/"
    Description: The directory of the source NFS file system.
    Type: String
  SourceSubnetId:
    Description: Select existing subnets.
    Type: AWS::EC2::Subnet::Id
  TaskSchedule:
    Default: "0 0/1 * * ? *"
    Description: Task cron schedule (cron format in UTC - [minute hour day/of/month month day/of/week year], e.g. every day @ 1:15pm UTC would be 15 13 ? * * *)
    Type: String
Resources:
  AmiInfoFunction:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        S3Bucket: !Sub solution-references-${AWS::Region}
        S3Key: datasync/amilookup-datasync-agent.zip
      Handler: amilookup-datasync-agent.handler
      Runtime: nodejs12.x
      Timeout: 30
      Role: !GetAtt LambdaExecutionRole.Arn
  AmiInfo:
    Type: Custom::AmiInfo
    Properties:
      ServiceToken: !GetAtt AmiInfoFunction.Arn
      Region: !Ref 'AWS::Region'
      OSName: "AWS DataSync EC2 Agent"
  LambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      ManagedPolicyArns:
        - 'arn:aws:iam::aws:policy/AWSLambdaExecute'
      Path: "/"
      Policies:
      - PolicyName: root
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - ec2:DescribeImages
            Resource: "*"
  AgentFleet:
    Type: AWS::EC2::EC2Fleet
    Properties:
      ExcessCapacityTerminationPolicy: termination
      LaunchTemplateConfigs: 
        - LaunchTemplateSpecification:
            LaunchTemplateId: !Ref LaunchTemplate
            Version: 1
      ReplaceUnhealthyInstances: true
      TargetCapacitySpecification:
        DefaultTargetCapacityType: on-demand
        TotalTargetCapacity: 1
      Type: maintain
  InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
      - !Ref InstanceRole
  InstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM
      - arn:aws:iam::aws:policy/AWSDataSyncFullAccess
      Path: /
  LaunchTemplate:
    Type: AWS::EC2::LaunchTemplate
    Properties:
      LaunchTemplateName: !Join [ '', [ 'DataSync-Agent-', !Ref 'AWS::StackName' ] ]
      LaunchTemplateData:
        IamInstanceProfile:
          Name: !Ref InstanceProfile
        ImageId: !GetAtt AmiInfo.Id
        InstanceType: !Ref InstanceType
        KeyName: !Ref Ec2KeyName
        Monitoring:
          Enabled: true
        NetworkInterfaces:
          - DeviceIndex: 0
            Groups:
              - !Ref SourceSecurityGroupId
            SubnetId: !Ref SourceSubnetId
        TagSpecifications:
          - ResourceType: instance
            Tags:
              -
                Key: Name
                Value: DataSync Agent
              -
                Key: StackName
                Value: !Ref 'AWS::StackName'
        UserData:
          Fn::Base64:
              !Sub |
                #cloud-config
                repo_update: true
                repo_upgrade: all

                write_files: 
                - content: |
                    #!/bin/bash

                    function getActivationKey
                    {
                        local destinationLocationRegion=$1

                        curl --silent "localhost:8080/?gatewayType=SYNC&activationRegion=${!destinationLocationRegion}&no_redirect"
                    }

                    function createAgent
                    {
                        local agentIp=$1
                        local destinationLocationRegion=$2
                        local stackName=$3
                        local activationKey=$4

                        aws datasync create-agent --activation-key ${!activationKey} --agent-name ${!agentIp} --region ${!destinationLocationRegion} --tags Key=StackName,Value=${!stackName} --output text
                    }

                    function createDataSyncLocationNfs
                    {
                        local serverHostname=$1
                        local sourceSubdirectory=$2
                        local agentArns=$3
                        local destinationLocationRegion=$4
                        local stackName=$5

                        aws datasync create-location-nfs --server-hostname ${!serverHostname} --subdirectory ${!sourceSubdirectory} --on-prem-config "AgentArns=${!agentArns}" --region ${!destinationLocationRegion} --tags Key=StackName,Value=${!stackName} --output text
                    }

                    function createDataSyncLocationEfs
                    {
                        local destinationEfsFilesystemId=$1
                        local destinationSubdirectory=$2
                        local destinationSubnetId=$3
                        local destinationSecurityGroupId=$4
                        local destinationLocationRegion=$5
                        local accountId=$6
                        local stackName=$7

                        local destinationEfsFilesystemArn="arn:aws:elasticfilesystem:${!destinationLocationRegion}:${!accountId}:file-system/${!destinationEfsFilesystemId}"
                        local destinationSubnetArn="arn:aws:ec2:${!destinationLocationRegion}:${!accountId}:subnet/${!destinationSubnetId}"
                        local destinationSecurityGroupArns="arn:aws:ec2:${!destinationLocationRegion}:${!accountId}:security-group/${!destinationSecurityGroupId}"

                        aws datasync create-location-efs --efs-filesystem-arn ${!destinationEfsFilesystemArn} --subdirectory ${!destinationSubdirectory} --ec2-config "SubnetArn=${!destinationSubnetArn},SecurityGroupArns=${!destinationSecurityGroupArns}" --region ${!destinationLocationRegion} --tags Key=StackName,Value=${!stackName} --output text
                    }

                    function createDataSyncTask
                    {
                        local sourceLocationArn=$1
                        local destinationLocationArn=$2
                        local destinationLocationRegion=$3
                        local agentIp=$4
                        local stackName=$5

                        aws datasync create-task --source-location-arn ${!sourceLocationArn} --destination-location-arn ${!destinationLocationArn} --schedule ScheduleExpression='"cron(${TaskSchedule})"' --name "Stack:${!stackName} Agent:${!agentIp}" --region ${!destinationLocationRegion} --tags Key=StackName,Value=${!stackName} --output text
                    }

                  path: /tmp/functions_datasync.sh
                  permissions: 0777

                - content: |
                    #!/bin/bash

                    agentIp=${!1}
                    sourceDnsName=${!2}
                    sourceSubdirectory=${!3}
                    destinationEfsFilesystemId=${!4}
                    destinationSubdirectory=${!5}
                    destinationSubnetId=${!6}
                    destinationSecurityGroupId=${!7}
                    destinationLocationRegion=${!8}
                    accountId=${!9}
                    stackName=${!10}

                    if [ $# -lt 10 ]; then
                      echo "Invalid # of arguments. Require: Source DNS name; Source Subdirectory; Destination EFS Filesystem Id; Destination Subdirectory; Destination Subnet Id; Destination Security Group Id; Destination Location Region; Account Id; Stack Name; Schedule"
                      exit 1
                    fi

                    # source datasync functions
                    source /tmp/functions_datasync.sh

                    # start create delete datasync resources script
                    echo -e "aws cloudformation delete-stack --stack-name ${!stackName} --region ${AWS::Region}" >> /tmp/delete_datasync_resources.sh

                    # get activation key
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Execute\tSelf-activating and generating activation key on agent on ${!agentIp}" >> /tmp/datasync_setup.log 2>&1
                    activationKey=$(getActivationKey ${!destinationLocationRegion})                    
                    if [ "$?" != "0" ]
                    then
                       echo -e "$(date -u +%FT%T.%3N)\tDataSync: Error\t\tFailed to self-activate or generate activation key on ${!agentIp}" >> /tmp/datasync_setup.log 2>&1
                       exit 1
                    fi
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Success\tKey generated: ${!activationKey}" >> /tmp/datasync_setup.log 2>&1

                    # activate agent
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Execute\tCreating agent in ${!destinationLocationRegion} for ${!agentIp} using activation key ${!activationKey}" >> /tmp/datasync_setup.log 2>&1
                    agentArn=$(createAgent ${!agentIp} ${!destinationLocationRegion} ${!stackName} ${!activationKey})
                    if [ "$?" != "0" ]
                    then
                        echo -e "$(date -u +%FT%T.%3N)\tDataSync: Error\t\tFailed to create agent in ${!destinationLocationRegion} for ${!agentIp} using activation key ${!activationKey}" >> /tmp/datasync_setup.log 2>&1
                        exit 1
                    fi
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Success\tAgent created: ${!agentArn}" >> /tmp/datasync_setup.log 2>&1
                    sed -i "1iaws datasync delete-agent --agent-arn ${!agentArn} --region ${!destinationLocationRegion}" /tmp/delete_datasync_resources.sh

                    # create source location (nfs)
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Execute\tCreating source location(nfs) for ${!sourceDnsName}" >> /tmp/datasync_setup.log 2>&1
                    sourceLocationArn=$(createDataSyncLocationNfs ${!sourceDnsName} ${!sourceSubdirectory} ${!agentArn} ${!destinationLocationRegion} ${!stackName})
                    if [ "$?" != "0" ]
                    then
                        echo -e "$(date -u +%FT%T.%3N)\tDataSync: Error\t\tFailed to create source location for ${!sourceDnsName}" >> /tmp/datasync_setup.log 2>&1
                        exit 1
                    fi
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Success\tSource location(nfs) created: ${!sourceLocationArn}" >> /tmp/datasync_setup.log 2>&1
                    sed -i "1iaws datasync delete-location --location-arn ${!sourceLocationArn} --region ${!destinationLocationRegion}" /tmp/delete_datasync_resources.sh

                    # create destination location (efs)
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Execute\tCreating destination location(efs) for ${!destinationEfsFilesystemId}" >> /tmp/datasync_setup.log 2>&1
                    destinationLocationArn=$(createDataSyncLocationEfs ${!destinationEfsFilesystemId} ${!destinationSubdirectory} ${!destinationSubnetId} ${!destinationSecurityGroupId} ${!destinationLocationRegion} ${!accountId} ${!stackName})
                    if [ "$?" != "0" ]
                    then
                        echo -e "$(date -u +%FT%T.%3N)\tDataSync: Error\t\tFailed to create destination location for ${!destinationEfsFilesystemId}" >> /tmp/datasync_setup.log 2>&1
                        exit 1
                    fi
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Success\tDestination location(efs) created: ${!destinationLocationArn}" >> /tmp/datasync_setup.log 2>&1
                    sed -i "1iaws datasync delete-location --location-arn ${!destinationLocationArn} --region ${!destinationLocationRegion}" /tmp/delete_datasync_resources.sh

                    # create task
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Execute\tCreating task between ${!sourceDnsName}${!sourceSubdirectory} and ${!destinationEfsFilesystemId}${!destinationSubdirectory}" >> /tmp/datasync_setup.log 2>&1
                    taskArn=$(createDataSyncTask ${!sourceLocationArn} ${!destinationLocationArn} ${!destinationLocationRegion} ${!agentIp} ${!stackName})
                    if [ "$?" != "0" ]
                    then
                        echo -e "$(date -u +%FT%T.%3N)\tDataSync: Error\t\tFailed to create task between ${!sourceDnsName}${!sourceSubdirectory} and ${!destinationEfsFilesystemId}${!destinationSubdirectory}" >> /tmp/datasync_setup.log 2>&1
                        exit 1
                    fi
                    echo -e "$(date -u +%FT%T.%3N)\tDataSync: Success\tTask created: ${!taskArn}" >> /tmp/datasync_setup.log 2>&1
                    sed -i "1iaws datasync delete-task --task-arn ${!taskArn} --region ${!destinationLocationRegion}" /tmp/delete_datasync_resources.sh

                  path: /tmp/create_datasync_resources.sh
                  permissions: 0777

                runcmd:
                - curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
                - unzip awscli-bundle.zip
                - ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws
                - export PATH=/usr/local/bin:$PATH
                - source /tmp/functions_datasync.sh
                - agentIp=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
                - sleep 60
                - echo /tmp/create_datasync_resources.sh ${!agentIp} ${SourceDnsName} ${SourceSubdirectory} ${DestinationEfsFilesystemId} ${DestinationSubdirectory} ${DestinationSubnetId} ${DestinationSecurityGroupId} ${DestinationLocationRegion} ${AWS::AccountId} ${AWS::StackName}
                - echo -e "$(date -u +%FT%T.%3N)\tDataSync":" Execute\tCreating datasync resources" >> /tmp/datasync_setup.log 2>&1
                - /tmp/create_datasync_resources.sh ${!agentIp} ${SourceDnsName} ${SourceSubdirectory} ${DestinationEfsFilesystemId} ${DestinationSubdirectory} ${DestinationSubnetId} ${DestinationSecurityGroupId} ${DestinationLocationRegion} ${AWS::AccountId} ${AWS::StackName}
                - if [ "$?" != "0" ]; then echo -e "$(date -u +%FT%T.%3N)\tDataSync":" Error\t\tFailed to create datasync resources" >> /tmp/datasync_setup.log 2>&1; else echo -e "$(date -u +%FT%T.%3N)\tDataSync":" Success\tCreated datasync resources" >> /tmp/datasync_setup.log 2>&1; fi
Outputs:
  ViewDataSyncSetupLog:
    Description: SSM command to view DataSync setup log
    Value:
      !Sub |
        commandId=$(aws ssm send-command --max-concurrency 1 --max-errors 0 --targets Key=tag:aws:ec2:fleet-id,Values=${AgentFleet} --document-name "AWS-RunShellScript" --parameters commands="sudo cat /tmp/datasync_setup.log" --query 'Command.CommandId' --output text --region ${AWS::Region}); instanceId=$(aws ec2 describe-fleet-instances --fleet-id ${AgentFleet} --query 'ActiveInstances[0].InstanceId' --output text --region ${AWS::Region}); aws ssm get-command-invocation --command-id ${!commandId} --instance-id ${!instanceId} --query 'StandardOutputContent' --output text --region ${AWS::Region}
  ViewDeleteDataSyncResourcesScript:
    Description: SSM command to view delete DataSync resources script
    Value:
      !Sub |
        commandId=$(aws ssm send-command --max-concurrency 1 --max-errors 0 --targets Key=tag:aws:ec2:fleet-id,Values=${AgentFleet} --document-name "AWS-RunShellScript" --parameters commands="sudo cat /tmp/delete_datasync_resources.sh" --query 'Command.CommandId' --output text --region ${AWS::Region}); instanceId=$(aws ec2 describe-fleet-instances --fleet-id ${AgentFleet} --query 'ActiveInstances[0].InstanceId' --output text --region ${AWS::Region}); aws ssm get-command-invocation --command-id ${!commandId} --instance-id ${!instanceId} --query 'StandardOutputContent' --output text --region ${AWS::Region}