Merge pull request #126 from RLOpenCatalyst/topic-release

1.15.0 build updates

Author: ravigurram8

cft-templates/pcluster.yaml

@@ -1,6 +1,6 @@
 AWSTemplateFormatVersion: 2010-09-09
 
-Description: Sets up an pcluster on AWS
+Description: Sets up an pcluster on AWS version 3.0 test
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -17,6 +17,7 @@ Metadata:
           - HeadNodeInstanceType
           - VpcId
           - HeadNodeSubnetId
+          - FileSystemType
           - FileSystemId
       - Label:
           default: Scheduler Configuration
@@ -113,9 +114,14 @@ Parameters:
     Type: String
     Description: Enter the AMI Id of a custom AMI if you wish to use a non-default AMI. e.g. ami-12345678. The AMI Id entered should exist in this project account and region
     Default: default
+  FileSystemType:
+    Type: String
+    Description: Choose the fileSystem type you want to mount on headnode leave it as NONE if don't want to mount any filesystem.
+    AllowedValues: [NONE,EFS, FSxForLustre]
+    Default: NONE  
   FileSystemId:
     Type: String
-    Description: Enter the File System Id of the FSx for Lustre file system which you want to use. e.g. fs-12345678. Leave this field to default if you do not want to use FSx for Lustre
+    Description: Enter the File System Id of the EFS/FSxForLustre file system which you want to use. e.g. fs-12345678. Leave this field to default if you do not want to use any file system
     Default: default
   QueueCapacityType:
     Description: The type of the compute resources used in the queue. Supported values are ONDEMAND or SPOT
@@ -216,6 +222,25 @@ Resources :
               - route53:*
               - ssm:*
               - fsx:*
+              - elasticfilesystem:Backup
+              - elasticfilesystem:CreateFileSystem
+              - elasticfilesystem:CreateMountTarget
+              - elasticfilesystem:CreateTags
+              - elasticfilesystem:CreateAccessPoint
+              - elasticfilesystem:CreateReplicationConfiguration
+              - elasticfilesystem:DeleteFileSystem
+              - elasticfilesystem:DeleteMountTarget
+              - elasticfilesystem:DeleteTags
+              - elasticfilesystem:DeleteAccessPoint
+              - elasticfilesystem:DeleteFileSystemPolicy
+              - elasticfilesystem:DeleteReplicationConfiguration
+              - elasticfilesystem:DescribeBackupPolicy
+              - elasticfilesystem:DescribeMountTargets
+              - elasticfilesystem:DescribeFileSystems
+              - elasticfilesystem:PutFileSystemPolicy
+              - elasticfilesystem:PutLifecycleConfiguration
+              - elasticfilesystem:PutBackupPolicy
+              - elasticfilesystem:DescribeMountTargetSecurityGroups
              Resource: "*"
     InstanceSecurityGroup:
       Type: AWS::EC2::SecurityGroup
@@ -261,7 +286,7 @@ Resources :
               trap '/opt/aws/bin/cfn-signal --exit-code 1 --resource EC2Instance --region ${AWS::Region} --stack ${AWS::StackName}' ERR    
               sudo yum update -y --security
               cd /home/ec2-user/parallel-update             
-              sudo -u ec2-user ./Provision-pcluster.sh ${Scheduler} ${AWS::Region} ${HeadNodeInstanceType} ${HeadNodeSubnetId} ${KeyPair} ${ComputeNodeInstanceType} ${ComputeEnvMinvCpus} ${ComputeEnvMaxvCpus} ${ComputeNodeSubnetId} ${ComputeEnvDesiredvCpus} ${SpotBidPercentage} ${AWS::StackName} ${CustomAMI} ${FileSystemId} ${QueueCapacityType} ${DisableSimultaneousMultithreading} $efaEnabled ${PlacementGroup}
+              sudo -u ec2-user ./Provision-pcluster.sh ${Scheduler} ${AWS::Region} ${HeadNodeInstanceType} ${HeadNodeSubnetId} ${KeyPair} ${ComputeNodeInstanceType} ${ComputeEnvMinvCpus} ${ComputeEnvMaxvCpus} ${ComputeNodeSubnetId} ${ComputeEnvDesiredvCpus} ${SpotBidPercentage} ${AWS::StackName} ${CustomAMI} ${FileSystemId} ${QueueCapacityType} ${DisableSimultaneousMultithreading} $efaEnabled ${PlacementGroup} ${FileSystemType}
               /opt/aws/bin/cfn-signal --exit-code 0 --resource EC2Instance --region ${AWS::Region} --stack ${AWS::StackName}
               sleep 10
               aws ec2 stop-instances --instance-ids `wget -qO- http://instance-data/latest/meta-data/instance-id`

cft-templates/sagemaker-template-with-url.yml

@@ -9,6 +9,14 @@ Parameters:
     AllowedValues:
       - ml.t2.medium
       - ml.t3.medium
+      - ml.m5.large
+      - ml.m5.xlarge
+      - ml.m5.2xlarge
+      - ml.m5.4xlarge
+      - ml.g4dn.large
+      - ml.g4dn.xlarge
+      - ml.g4dn.2xlarge
+      - ml.g4dn.4xlarge
     ConstraintDescription: Must select a valid notebook instance type.
     Default: ml.t3.medium
     Description: Select Instance type for the SageMaker Notebook. e.g.ml.t3.medium
@@ -35,12 +43,63 @@ Resources:
     Properties:
       InstanceType: !Ref NotebookInstanceType 
       RoleArn: !GetAtt SageMakerRole.Arn
-      LifecycleConfigName: 'Research-Portal-Setup'
+      PlatformIdentifier: notebook-al2-v2
+      LifecycleConfigName: !GetAtt NotebookLifeCycleConfig.NotebookInstanceLifecycleConfigName
       Tags:
         - Key: cost_resource
           Value: !Sub ${AWS::StackName}
         - Key: Name
-          Value: !Sub ${AWS::StackName}
+          Value: !Sub ${AWS::StackName}      
+      
+  NotebookLifeCycleConfig:
+    Type: AWS::SageMaker::NotebookInstanceLifecycleConfig
+    Properties: 
+      OnStart:
+        - Content:
+            Fn::Base64: !Sub |
+              #!/bin/bash
+
+              set -e
+
+              function get_tag()
+              {
+                  
+                  NOTEBOOK_ARN=$(jq '.ResourceArn' /opt/ml/metadata/resource-metadata.json --raw-output)
+                  TAG_NAME=$1
+                  local value=$(aws sagemaker list-tags --resource-arn $NOTEBOOK_ARN  | jq -r --arg TAG_NAME "$TAG_NAME" .'Tags[] | select(.Key == $TAG_NAME).Value' --raw-output)
+                  echo "$value"
+              }
+
+              # PARAMETERS
+              DATA_BUCKET=$(get_tag "DATA_BUCKET")
+
+              #Set Environment variables
+              if [[ $DATA_BUCKET  != '' ]]
+              then
+                  touch /etc/profile.d/jupyter-env.sh
+                  echo "export 'DATA_BUCKET'=$DATA_BUCKET" >> /etc/profile.d/jupyter-env.sh
+              fi
+
+
+              #Copy Sample notebook from S3
+
+              TEMPLATE_NOTEBOOK_BUCKET=$(get_tag "TEMPLATE_NOTEBOOK_BUCKET")
+              TEMPLATE_NOTEBOOK_DIRECTORY="/home/ec2-user/sample-notebooks/research_portal_templates/getting_started"
+
+              # Copy the files only when the bucket exists
+
+              if aws s3api head-bucket --bucket "$TEMPLATE_NOTEBOOK_BUCKET" 2>/dev/null; then
+                  mkdir -p $TEMPLATE_NOTEBOOK_DIRECTORY
+                  aws s3 cp s3://$TEMPLATE_NOTEBOOK_BUCKET/ $TEMPLATE_NOTEBOOK_DIRECTORY/ --recursive
+                  chown -R "ec2-user":"ec2-user" $TEMPLATE_NOTEBOOK_DIRECTORY
+              else
+                  echo "Template bucket not found. Skipping sample notebook creation"
+              fi
+
+              systemctl restart jupyter-server
+
+
+
 Outputs:
   SageMakerNotebookInstanceARN:
     Description: "ARN for the newly created SageMaker Notebook Instance"

docker-compose.yml

@@ -18,7 +18,7 @@ services:
       - sp2net
 
   cc-3102:
-    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.14.2_b1452
+    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.15.0_b1480
     secrets:
       - source: sp2prod-config.json
         target: /rlc/cc/server/app/config/config.json
@@ -42,7 +42,7 @@ services:
     command: /bin/bash start.sh
 
   scheduler-3102:
-    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.14.2_b1452
+    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.15.0_b1480
     secrets:
       - source: sp2prod-config.json
         target: /rlc/cc/server/app/config/config.json
@@ -62,7 +62,7 @@ services:
     command: node ./app/scheduler/app.js
 
   notificationsink:
-    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.14.0_b3
+    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.15.0_b1
     secrets:
       - source: sp2prod-config.json
         target: /opt/app/config/config.json

packer-rg.json

@@ -6,13 +6,13 @@
     "awsRegion": "",
     "RG_HOME": "/opt/deploy/sp2",
     "RG_SRC": "/home/ubuntu",
-    "amiName": "Ubuntu20.04_MongoDB3.6_Docker-ce20.10"
+    "amiName": "RG-AMI-1.15.0"
   },
   "builders": [
     {
       "type": "amazon-ebs",
       "profile": "{{user `awsProfile`}}",
-      "iam_instance_profile": "packer-role-test",
+      "iam_instance_profile": "packer-role1",
       "region": "{{user `awsRegion`}}",
       "vpc_id": "{{user `vpcId`}}",
       "subnet_id": "{{user `subnetId`}}",

products/PCluster/machine-images/config/infra/files/pcluster/Provision-pcluster.sh

@@ -46,6 +46,7 @@ QueueCapacityType=${15}
 disableSimultaneousMultithreading=${16}
 efa=${17}
 placementGroup=${18}
+FileSystemType=${19}
 
 IFS='-' read -ra TRIMMED <<< "$CustomAMI"
 CustomAMIStartsWith=${TRIMMED[0]}
@@ -54,8 +55,13 @@ if [ "$scheduler" == "slurm" ]; then
        if [ "$CustomAMIStartsWith" == "ami" ]; then
           yq -i ".Image.CustomAmi=\"$CustomAMI\"" slurm.yaml
        fi
-       if [ "$FileSystemId" != "default" ]; then
-          yq -i '.SharedStorage=[{"MountDir": "/fsx", "Name":"RG_Filesysytem", "StorageType": "FsxLustre", "FsxLustreSettings":{"FileSystemId":"'$FileSystemId'"}}]' slurm.yaml
+       if [ "$FileSystemType" == "FSxForLustre" ] && [ "$FileSystemId" != "default" ]; then
+          echo "Mounting $FileSystemId FSxForLustre filesystem to headnode"
+          yq -i '.SharedStorage=[{"MountDir": "/fsx", "Name":"RG_Fsx_Filesysytem", "StorageType": "FsxLustre", "FsxLustreSettings":{"FileSystemId":"'$FileSystemId'"}}]' slurm.yaml
+       fi
+       if [ "$FileSystemType" == "EFS" ] && [ "$FileSystemId" != "default" ]; then
+          echo "Mounting $FileSystemId EFS filesystem to headnode"
+          yq -i '.SharedStorage=[{"MountDir": "/efs", "Name":"RG_Efs_Filesysytem", "StorageType": "Efs", "EfsSettings":{"FileSystemId":"'$FileSystemId'"}}]' slurm.yaml
        fi
        yq -i ".Region=\"$Region\"" slurm.yaml
        yq -i ".HeadNode.InstanceType=\"$headnodeinstancetype\"" slurm.yaml
@@ -79,6 +85,10 @@ else
         if [ "$CustomAMIStartsWith" == "ami" ]; then
             yq -i ".Image.CustomAmi=\"$CustomAMI\"" batch.yaml
        fi
+       if [ "$FileSystemType" == "EFS" ] && [ "$FileSystemId" != "default" ]; then
+          echo "Mounting $FileSystemId EFS filesystem to headnode"
+          yq -i '.SharedStorage=[{"MountDir": "/efs", "Name":"RG_Efs_Filesysytem", "StorageType": "Efs", "EfsSettings":{"FileSystemId":"'$FileSystemId'"}}]' batch.yaml
+       fi
        yq -i ".Region=\"$Region\"" batch.yaml
        yq -i ".HeadNode.InstanceType=\"$headnodeinstancetype\"" batch.yaml
        yq -i ".HeadNode.Networking.SubnetId=\"$headnodesubnetId\"" batch.yaml

provisioners/provision-ecr.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -xe
 sudo docker login -u AWS -p $(aws ecr get-login-password --region us-east-2) 045938549113.dkr.ecr.us-east-2.amazonaws.com
-sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.14.2_b1452
+sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.15.0_b1480
 sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/nginx:latest
-sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.14.0_b3
+sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.15.0_b1