Merge pull request #127 from ravigurram8/topic-release-11

ravigurram8 · web-flow · commit 39b855030348 · 2023-01-09T15:13:53.000+05:30
1.16.0 build updates
diff --git a/cft-templates/Rstudio.yml b/cft-templates/Rstudio.yml
@@ -136,3 +136,6 @@ Outputs:
     Value: !Ref 'RstudioEC2Instance'
   ApplicationPort:
     Value: '443'
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created Rstudio EC2Instance
+    Value: !GetAtt [RstudioEC2Instance, AvailabilityZone]  
diff --git a/cft-templates/amazon-ebs.yml b/cft-templates/amazon-ebs.yml
@@ -0,0 +1,109 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Description: Amazon EBS volume is a durable, block-level storage device that you can attach to your instances. After you attach a volume to an instance, you can use it as you would use a physical hard drive.
+Metadata:
+  AWS::CloudFormation::Interface:    
+    ParameterGroups:
+      - Label:
+          default: EBS Configuration
+        Parameters:
+          - VolumeType
+          - VolumeSize
+          - Iops
+          - Throughput
+          - AvailabilityZone
+          - Encrypted
+          - KmsKeyId
+Parameters:
+  VolumeType:
+    Description: If the volume type is io1 or io2, you must specify the IOPS that the volume supports.
+    Type: String
+    AllowedValues: [gp2, gp3, io1, io2, sc1, st1, standard]
+    Default: standard
+  VolumeSize:
+    Description: >- 
+      The size of the volume, in GiBs. You must specify either a snapshot ID or a volume size. If you specify a snapshot, the default is the snapshot size. You can specify a volume size that is equal to or larger than the snapshot size.
+      The following are the supported volumes sizes for each volume type:
+      gp2 and gp3:1-16,384
+      io1 and io2: 4-16,384
+      st1 and sc1: 125-16,384
+      standard: 1-1,024
+    Type: Number
+    Default: 100
+  Iops:
+    Description: >-
+      The number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, this represents the number of IOPS that are provisioned for the volume.
+      applicable only if VolumeType selected is gp3,io1 or io2.
+      gp3: 3,000-16,000 IOPS
+      io1: 100-64,000 IOPS
+      io2: 100-64,000 IOPS
+    Type: Number
+    Default: 3000
+  Throughput:
+    Description: The throughput to provision for a gp3 volume, with a minimum of 125  MiB/s or a maximum of 1,000 MiB/s.
+    Type: Number
+    MaxValue: 1000
+    MinValue: 125
+    Default: 125
+  AvailabilityZone:
+    Description: choose the Availability Zone in which to create the volume. A volume can be attached only to  instances that is in the same Availability Zone.
+    Type: AWS::EC2::AvailabilityZone::Name    
+  Encrypted:
+    Description: Indicates whether the volume should be encrypted.
+    Type: String
+    AllowedValues: [true, false]
+    Default: false
+  KmsKeyId:
+    Description: >-
+      The ID of the AWS Key Management Service (AWS KMS) key used to encrypt Amazon Elastic Block Store (Amazon EBS) volume.
+      applicable only when AWS KMS key is available and Encryption is set to true.
+    Type: String
+    Default: NONE
+     
+  ResearcherName:
+    Description: Researcher name that will be added as tag to the resources.
+    Type: String
+  ProjectId:
+    Description: Project Id that will be added as tag to the resources.
+    Type: String
+  Namespace:
+    Description: An environment name that will be prefixed to resource names.
+    Type: String
+    
+
+Conditions:
+  KMSKeyNotProvided: !Equals [!Ref KmsKeyId, 'NONE']
+  IopsVolumeType: !Or [!Equals [!Ref VolumeType, gp3],!Equals [!Ref VolumeType, io1],!Equals [!Ref VolumeType, io2]]
+  ThroughputApplicable: !Equals [!Ref VolumeType, gp3]
+
+Resources:
+  VolumeId:
+    Type: AWS::EC2::Volume
+    Properties:
+      VolumeType: !Ref VolumeType
+      Size: !Ref VolumeSize
+      Iops: !If [IopsVolumeType, !Ref Iops, !Ref 'AWS::NoValue']
+      Throughput: !If [ThroughputApplicable, !Ref Throughput, !Ref 'AWS::NoValue']
+      AvailabilityZone: !Ref AvailabilityZone
+      Encrypted: !Ref Encrypted
+      KmsKeyId: !If [KMSKeyNotProvided, !Ref 'AWS::NoValue', !Ref KmsKeyId]
+      # AvailabilityZone: !Select [0, !GetAZs '']
+      
+      Tags: 
+        - Key: cost_resource
+          Value: !Sub ${AWS::StackName}
+        - Key: researcher_name
+          Value: !Ref ResearcherName
+        - Key: project_name
+          Value: !Ref ProjectId
+        - Key: Name
+          Value: !Join ['-', [Ref: Namespace, 'Volume']]
+    DeletionPolicy: Snapshot
+
+Outputs:
+  VolumeId:
+    Description: Volume id of newly created Ebs volume. 
+    Value: !Ref VolumeId
+  AvailabilityZone:   
+    Description: location of newly created Ebs volume.
+    Value: !Ref AvailabilityZone
diff --git a/cft-templates/cromwell-advanced.yaml b/cft-templates/cromwell-advanced.yaml
@@ -946,3 +946,6 @@ Outputs:
     Value: !Sub ${S3Bucket}
   BatchComputeEnvironment:
     Value: !Ref SpotComputeEnv
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/ec2-EIP.yml b/cft-templates/ec2-EIP.yml
@@ -123,3 +123,6 @@ Outputs:
   InstanceDNSName:
     Description: DNS name of the newly created EC2 instance
     Value: !GetAtt [EC2Instance, PublicDnsName]
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/ec2-dcv.yml b/cft-templates/ec2-dcv.yml
@@ -178,4 +178,7 @@ Outputs:
     Value: !GetAtt [EC2Instance, PublicDnsName]
   ApplicationPort:
     Description: The Port in which the application is running
-    Value: '8443'
+    Value: '8443'
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/ec2-linux-docker-mysql.yml b/cft-templates/ec2-linux-docker-mysql.yml
@@ -171,4 +171,7 @@ Outputs:
     Value: !GetAtt [EC2Instance, PublicDnsName]
   InstancePrivateIPAddress:
     Description: Private IP address of the newly created EC2 instance
-    Value: !GetAtt [EC2Instance, PrivateIp]
+    Value: !GetAtt [EC2Instance, PrivateIp]
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/ec2-linux-docker.yml b/cft-templates/ec2-linux-docker.yml
@@ -133,4 +133,7 @@ Outputs:
     Value: !GetAtt [EC2Instance, PublicIp]
   InstanceDNSName:
     Description: DNS name of the newly created EC2 instance
-    Value: !GetAtt [EC2Instance, PublicDnsName]
+    Value: !GetAtt [EC2Instance, PublicDnsName]
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/ec2-ubuntu-update.yaml b/cft-templates/ec2-ubuntu-update.yaml
@@ -127,3 +127,6 @@ Outputs:
   InstanceDNSName:
     Description: DNS name of the newly created EC2 instance
     Value: !GetAtt [EC2Instance, PublicDnsName]
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/ec2-ubuntu.yml b/cft-templates/ec2-ubuntu.yml
@@ -127,3 +127,6 @@ Outputs:
   InstanceDNSName:
     Description: DNS name of the newly created EC2 instance
     Value: !GetAtt [EC2Instance, PublicDnsName]
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/nextflow-advanced.yaml b/cft-templates/nextflow-advanced.yaml
@@ -735,3 +735,6 @@ Outputs:
     Value: !Sub ${DefaultQueue}
   BatchComputeEnvironment:
     Value: !Ref SpotComputeEnv
+  AvailabilityZone: 
+    Description: AvailabilityZone of newly created EC2 instance
+    Value: !GetAtt [EC2Instance, AvailabilityZone]  
diff --git a/cft-templates/pcluster.yaml b/cft-templates/pcluster.yaml
@@ -117,8 +117,8 @@ Parameters:
   FileSystemType:
     Type: String
     Description: Choose the fileSystem type you want to mount on headnode leave it as NONE if don't want to mount any filesystem.
-    AllowedValues: [NONE,EFS, FSxForLustre]
-    Default: NONE  
+    AllowedValues: [NONE,EFS, FSxForLustre,EBS]
+    Default: NONE
   FileSystemId:
     Type: String
     Description: Enter the File System Id of the EFS/FSxForLustre file system which you want to use. e.g. fs-12345678. Leave this field to default if you do not want to use any file system
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -39,8 +39,8 @@ services:
       - ${PWD}/catalog:/rlc/cc/tools/deploy/catalog
     networks:
       - sp2net
-    command: /bin/bash start.sh
-
+    
+    
   scheduler-3102:
     image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.15.0_b1480
     secrets:
@@ -59,7 +59,7 @@ services:
       - ${PWD}/topologies:/rlc/cc/server/dashboard/topologies
     networks:
       - sp2net
-    command: node ./app/scheduler/app.js
+    command: ["./app/scheduler/app.js"]
 
   notificationsink:
     image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.15.0_b1
diff --git a/dump/configs.json b/dump/configs.json
@@ -408,5 +408,20 @@
                 ]
         },
         "studySelectionCount" : 4
+},
+{
+  "_id" : { 
+    "$oid": "62b6c70a7dec087675b33788"
+  },
+  "key" : "ebsCompatibleProducts",
+  "value" : {
+          "productList" : [
+                  "ec2",
+                  "cromwell",
+                  "rstudio",
+                  "ec2-dcv",
+                  "nextflow"
+          ]
+  }
 }
 ]
diff --git a/dump/standardcatalogitems.json b/dump/standardcatalogitems.json
@@ -952,5 +952,65 @@
                   },
                   "cost_resource" : true
           }
+  },
+  {
+    "_id": {
+      "$oid": "60504955135663522b880cfa"
+    },
+    "name" : "Amazon EBS volume",
+    "description" : "Amazon EBS volume is a durable, block-level storage device that you can attach to your instances. After you attach a volume to an instance, you can use it as you would use a physical hard drive.",
+    "fileName" : "amazon-ebs.yml",
+    "tags" : [
+            {
+                    "Key" : "EstimatedTimeToProvision",
+                    "Value" : "10 Minutes"
+            },
+            {
+                    "Key" : "DetailsLink",
+                    "Value" : "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volumes.html"
+            },
+            {
+                    "Key" : "Service",
+                    "Value" : "AmazonEbsVolume"
+            },
+            {
+                    "Key" : "TypeOfProduct",
+                    "Value" : "Research"
+            }
+    ],
+    "owner" : "RL",
+    "portfolio" : "RGPortfolio",
+    "availableRegions" : [ ],
+    "assignedOU" : [
+            
+    ],
+    "metaData" : {
+            "pre_provisioning" : [
+                    {
+                            "code" : "CFT_PARAMS",
+                            "params" : [
+                                    {
+                                            "name" : "ProjectId",
+                                            "type" : "RL::SC::PARAM::HD"
+                                    },
+                                    {
+                                            "name" : "ResearcherName",
+                                            "type" : "RL::SC::PARAM::HD"
+                                    },
+                                    {
+                                            "name" : "Namespace",
+                                            "type" : "RL::SC::PARAM::HD"
+                                    }
+                            ]
+                    }
+            ],
+            "post_provisioning" : [ ],
+            "checks_before_assigning_product" : [ ],
+            "checks_after_assigning_product" : [ ],
+            "permission_required" : {
+
+            },
+            "cost_resource" : true
+    }
   }
 ]
diff --git a/products/PCluster/machine-images/config/infra/files/pcluster/Provision-pcluster.sh b/products/PCluster/machine-images/config/infra/files/pcluster/Provision-pcluster.sh
@@ -63,6 +63,10 @@ if [ "$scheduler" == "slurm" ]; then
           echo "Mounting $FileSystemId EFS filesystem to headnode"
           yq -i '.SharedStorage=[{"MountDir": "/efs", "Name":"RG_Efs_Filesysytem", "StorageType": "Efs", "EfsSettings":{"FileSystemId":"'$FileSystemId'"}}]' slurm.yaml
        fi
+       if [ "$FileSystemType" == "EBS" ] && [ "$FileSystemId" != "default" ]; then
+          echo "Mounting $FileSystemId EBS Volume to headnode"
+          yq -i '.SharedStorage=[{"MountDir": "/ebs", "Name":"RG_EBS_Volume", "StorageType": "Ebs", "EbsSettings":{"VolumeId":"'$FileSystemId'"}}]' slurm.yaml
+       fi
        yq -i ".Region=\"$Region\"" slurm.yaml
        yq -i ".HeadNode.InstanceType=\"$headnodeinstancetype\"" slurm.yaml
        yq -i ".HeadNode.Networking.SubnetId=\"$headnodesubnetId\"" slurm.yaml
@@ -89,6 +93,10 @@ else
           echo "Mounting $FileSystemId EFS filesystem to headnode"
           yq -i '.SharedStorage=[{"MountDir": "/efs", "Name":"RG_Efs_Filesysytem", "StorageType": "Efs", "EfsSettings":{"FileSystemId":"'$FileSystemId'"}}]' batch.yaml
        fi
+       if [ "$FileSystemType" == "EBS" ] && [ "$FileSystemId" != "default" ]; then
+          echo "Mounting $FileSystemId EBS Volume to headnode"
+          yq -i '.SharedStorage=[{"MountDir": "/ebs", "Name":"RG_EBS_Volume", "StorageType": "Ebs", "EbsSettings":{"VolumeId":"'$FileSystemId'"}}]' batch.yaml
+       fi
        yq -i ".Region=\"$Region\"" batch.yaml
        yq -i ".HeadNode.InstanceType=\"$headnodeinstancetype\"" batch.yaml
        yq -i ".HeadNode.Networking.SubnetId=\"$headnodesubnetId\"" batch.yaml
diff --git a/provisioners/provision-ecr.sh b/provisioners/provision-ecr.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -xe
 sudo docker login -u AWS -p $(aws ecr get-login-password --region us-east-2) 045938549113.dkr.ecr.us-east-2.amazonaws.com
-sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.15.0_b1480
+sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.16.0_b1600
 sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/nginx:latest
 sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.15.0_b1
