Skip to content

Commit 2b5dac1

Browse files
author
ravigurram8
committed
Merge remote-tracking branch 'upstream/topic-release' into topic-release-3
2 parents 33ed3d3 + c85ec84 commit 2b5dac1

16 files changed

Lines changed: 789 additions & 11 deletions

‎cft-templates/Rstudio.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,14 @@ Resources:
9898
SecurityGroups: [!Ref 'RstudioEC2SecurityGroup']
9999
KeyName: !Ref 'KeyPair'
100100
IamInstanceProfile: !Ref InstanceProfile
101+
PropagateTagsToVolumeOnCreation: true
101102
Tags:
102103
- Key: Name
103104
Value: !Join ['-', [Ref: Namespace, 'rstudio-server']]
104105
- Key: Description
105106
Value: EC2 linux based Rstudio server
107+
- Key: cost_resource
108+
Value: !Sub ${AWS::StackName}
106109
UserData:
107110
Fn::Base64: !Sub |
108111
#!/bin/bash

‎cft-templates/cromwell-advanced.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,16 @@ Resources:
127127
Value: !Ref ResearcherName
128128
- Key: project_name
129129
Value: !Ref ProjectId
130+
- ResourceType: volume
131+
Tags:
132+
- Key: cost_resource
133+
Value: !Sub ${AWS::StackName}
134+
- Key: Name
135+
Value: !Join ['-', [Ref: Namespace, 'cromwell-work-node']]
136+
- Key: researcher_name
137+
Value: !Ref ResearcherName
138+
- Key: project_name
139+
Value: !Ref ProjectId
130140
BlockDeviceMappings:
131141
- Ebs:
132142
DeleteOnTermination: True
@@ -867,6 +877,7 @@ Resources:
867877
ImageId: !Ref 'LatestAmiId'
868878
InstanceType: !Ref InstanceType
869879
IamInstanceProfile: !Ref EC2InstanceProfile
880+
PropagateTagsToVolumeOnCreation: true
870881
SecurityGroupIds:
871882
- { "Fn::GetAtt" : ["EC2SecurityGroup", "GroupId"] }
872883
KeyName: !Ref 'KeyPair'

‎cft-templates/ec2-EIP.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,11 +94,14 @@ Resources:
9494
KeyName: !Ref 'KeyPair'
9595
ImageId: !Ref 'LatestAmiId'
9696
IamInstanceProfile: !Ref InstanceProfile
97+
PropagateTagsToVolumeOnCreation: true
9798
Tags:
9899
- Key: Name
99100
Value: !Join ['-', [Ref: Namespace, 'ec2-linux']]
100101
- Key: Description
101102
Value: EC2 workspace instance
103+
- Key: cost_resource
104+
Value: !Sub ${AWS::StackName}
102105

103106
InstanceSecurityGroup:
104107
Type: AWS::EC2::SecurityGroup

‎cft-templates/ec2-linux-docker-mysql.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,11 +137,14 @@ Resources:
137137
KeyName: !Ref 'KeyPair'
138138
ImageId: !Ref 'LatestAmiId'
139139
IamInstanceProfile: !Ref InstanceProfile
140+
PropagateTagsToVolumeOnCreation: true
140141
Tags:
141142
- Key: Name
142143
Value: !Join ['-', [Ref: Namespace, 'ec2-linux']]
143144
- Key: Description
144145
Value: EC2 workspace instance
146+
- Key: cost_resource
147+
Value: !Sub ${AWS::StackName}
145148

146149
InstanceSecurityGroup:
147150
Type: AWS::EC2::SecurityGroup

‎cft-templates/ec2-linux-docker.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,15 @@ Resources:
104104
SecurityGroups: [!Ref 'InstanceSecurityGroup']
105105
KeyName: !Ref 'KeyPair'
106106
ImageId: !Ref 'LatestAmiId'
107+
PropagateTagsToVolumeOnCreation: true
107108
IamInstanceProfile: !Ref InstanceProfile
108109
Tags:
109110
- Key: Name
110111
Value: !Join ['-', [Ref: Namespace, 'ec2-linux']]
111112
- Key: Description
112113
Value: EC2 workspace instance
114+
- Key: cost_resource
115+
Value: !Sub ${AWS::StackName}
113116

114117
InstanceSecurityGroup:
115118
Type: AWS::EC2::SecurityGroup

‎cft-templates/ec2-ubuntu-update.yaml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
Metadata:
2+
License: Apache-2.0
3+
AWSTemplateFormatVersion: '2010-09-09'
4+
Description: 'AWS CloudFormation Template to create an EC2 instance
5+
**WARNING** This template creates an Amazon EC2 instance.
6+
You will be billed for the AWS resources used if you create a stack from this template.'
7+
8+
Parameters:
9+
Namespace:
10+
Type: String
11+
Description: An environment name that will be prefixed to resource names
12+
S3Mounts:
13+
Type: String
14+
Description: A JSON array of objects with name, bucket, and prefix properties used to mount data
15+
IamPolicyDocument:
16+
Type: String
17+
Description: The IAM policy to be associated with the launched workstation
18+
EnvironmentInstanceFiles:
19+
Type: String
20+
Description: >-
21+
An S3 URI (starting with "s3://") that specifies the location of files to be copied to
22+
the environment instance, including any bootstrap scripts
23+
InstanceType:
24+
Description: Choose the instance type for this instance. e.g. t2.small
25+
Type: String
26+
Default: t2.small
27+
AllowedValues: [t2.nano, t2.micro, t2.small, t2.medium]
28+
ConstraintDescription: must be a valid EC2 instance type.
29+
KeyPair:
30+
Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. If no key pairs exist, please create one from the button next to the dropdown. Please contact your Administrator if you are unable to create one.
31+
Type: AWS::EC2::KeyPair::KeyName
32+
ConstraintDescription: must be the name of an existing EC2 KeyPair.
33+
AllowedSSHLocation:
34+
Description: The IP address range that can be used to SSH to the EC2 instances
35+
Type: String
36+
MinLength: '9'
37+
MaxLength: '18'
38+
Default: 0.0.0.0/0
39+
AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})
40+
ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
41+
LatestAmiId:
42+
Description: Enter an AMI-Id e.g. ami-12345. The AMI should exist in the same region as the project. Leave the default string to use the latest Canonical Ubuntu 20.04 image.
43+
Type: String
44+
Default: '/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id'
45+
46+
Conditions:
47+
IamPolicyEmpty: !Equals [!Ref IamPolicyDocument, '{}']
48+
UserProvidedAMI: !Equals [ 'ami', !Select [0, !Split ["-", !Ref LatestAmiId]]]
49+
50+
Resources:
51+
IAMRole:
52+
Type: 'AWS::IAM::Role'
53+
Properties:
54+
RoleName: !Join ['-', [Ref: Namespace, 'ec2-role']]
55+
Path: '/'
56+
AssumeRolePolicyDocument:
57+
Version: '2012-10-17'
58+
Statement:
59+
- Effect: 'Allow'
60+
Principal:
61+
Service:
62+
- 'ec2.amazonaws.com'
63+
Action:
64+
- 'sts:AssumeRole'
65+
Policies:
66+
- !If
67+
- IamPolicyEmpty
68+
- !Ref 'AWS::NoValue'
69+
- PolicyName: !Join ['-', [Ref: Namespace, 's3-studydata-policy']]
70+
PolicyDocument: !Ref IamPolicyDocument
71+
72+
InstanceProfile:
73+
Type: 'AWS::IAM::InstanceProfile'
74+
Properties:
75+
InstanceProfileName: !Join ['-', [Ref: Namespace, 'ec2-profile']]
76+
Path: '/'
77+
Roles:
78+
- Ref: IAMRole
79+
80+
EC2Instance:
81+
Type: AWS::EC2::Instance
82+
Properties:
83+
UserData:
84+
Fn::Base64: !Sub |
85+
#!/usr/bin/env bash
86+
exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
87+
apt update
88+
apt install -y awscli jq
89+
# Download and execute bootstrap script
90+
aws s3 cp "${EnvironmentInstanceFiles}/get_bootstrap_ubuntu.sh" "/tmp"
91+
chmod 500 "/tmp/get_bootstrap_ubuntu.sh"
92+
/tmp/get_bootstrap_ubuntu.sh "${EnvironmentInstanceFiles}" '${S3Mounts}'
93+
94+
# Signal result to CloudFormation
95+
/opt/aws/bin/cfn-signal -e $? --stack "${AWS::StackName}" --resource "EC2Instance" --region "${AWS::Region}"
96+
InstanceType: !Ref 'InstanceType'
97+
SecurityGroups: [!Ref 'InstanceSecurityGroup']
98+
KeyName: !Ref 'KeyPair'
99+
ImageId: !If [ UserProvidedAMI,!Ref LatestAmiId,'{{resolve:ssm:/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id}}']
100+
IamInstanceProfile: !Ref InstanceProfile
101+
PropagateTagsToVolumeOnCreation: true
102+
Tags:
103+
- Key: Name
104+
Value: !Join ['-', [Ref: Namespace, 'ec2-linux']]
105+
- Key: Description
106+
Value: EC2 workspace instance
107+
- Key: cost_resource
108+
Value: !Sub ${AWS::StackName}
109+
110+
InstanceSecurityGroup:
111+
Type: AWS::EC2::SecurityGroup
112+
Properties:
113+
GroupDescription: Enable SSH access
114+
SecurityGroupIngress:
115+
- IpProtocol: tcp
116+
FromPort: '22'
117+
ToPort: '22'
118+
CidrIp: !Ref 'AllowedSSHLocation'
119+
120+
Outputs:
121+
InstanceId:
122+
Description: InstanceId of the newly created EC2 instance
123+
Value: !Ref 'EC2Instance'
124+
InstanceIPAddress:
125+
Description: IP address of the newly created EC2 instance
126+
Value: !GetAtt [EC2Instance, PublicIp]
127+
InstanceDNSName:
128+
Description: DNS name of the newly created EC2 instance
129+
Value: !GetAtt [EC2Instance, PublicDnsName]

‎cft-templates/ec2-ubuntu.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,14 @@ Resources:
9898
KeyName: !Ref 'KeyPair'
9999
ImageId: !If [ UserProvidedAMI,!Ref LatestAmiId,'{{resolve:ssm:/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id}}']
100100
IamInstanceProfile: !Ref InstanceProfile
101+
PropagateTagsToVolumeOnCreation: true
101102
Tags:
102103
- Key: Name
103104
Value: !Join ['-', [Ref: Namespace, 'ec2-linux']]
104105
- Key: Description
105106
Value: EC2 workspace instance
107+
- Key: cost_resource
108+
Value: !Sub ${AWS::StackName}
106109

107110
InstanceSecurityGroup:
108111
Type: AWS::EC2::SecurityGroup

‎cft-templates/fsx-for-lustre.yaml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
AWSTemplateFormatVersion: 2010-09-09
2+
3+
Description: Sets up an FSX for Lustre filesystem with PERSISTENT_1 deployment type using SSD storage.
4+
5+
Parameters:
6+
StorageCapacity:
7+
Description: >-
8+
The storage capacity of the file system that you're creating.
9+
The valid values are 1200 GiB, 2400 GiB, and increments of 2400 GiB.
10+
Default: 1200
11+
Type: Number
12+
ImportPath:
13+
Description: The path to the Amazon S3 bucket (including the optional prefix) that you're using as the data repository for your Amazon FSx for Lustre file system
14+
Default: NONE
15+
Type: String
16+
AllowedPattern: ^[^\u0000\u0085\u2028\u2029\r\n]{3,4357}$
17+
ExportPath:
18+
Description: The path in the Amazon S3 bucket where the root of your Amazon FSx file system is exported.
19+
Default: NONE
20+
Type: String
21+
AllowedPattern: ^[^\u0000\u0085\u2028\u2029\r\n]{3,4357}$
22+
VpcId:
23+
Description: The ID of the VPC that the file system will be accessible from
24+
Type: AWS::EC2::VPC::Id
25+
SubnetId:
26+
Description: The ID of the subnet that the file system will be accessible from. Make sure the subnet is part of the VPC selected above.
27+
Type: AWS::EC2::Subnet::Id
28+
KmsKeyId:
29+
Description: The ID of the AWS Key Management Service (AWS KMS) key used to encrypt Amazon FSx file system data.
30+
Default: NONE
31+
Type: String
32+
AutoImportPolicy:
33+
Description: Choose how Amazon FSx keeps your file and directory listings up to date as you add or modify objects in your linked S3 bucket
34+
Type: String
35+
Default: NONE
36+
AllowedValues: [NONE, NEW, NEW_CHANGED, NEW_CHANGED_DELETED]
37+
DataCompressionType:
38+
Description: Sets the data compression configuration for the file system.
39+
Type: String
40+
Default: NONE
41+
AllowedValues: [NONE, LZ4]
42+
PerUnitStorageThroughput:
43+
Description: The amount of read and write throughput for each 1 tebibyte (TiB) of file system storage capacity.
44+
Type: Number
45+
Default: 50
46+
AllowedValues: [50, 100, 200]
47+
ResearcherName:
48+
Type: String
49+
Description: Researcher name that will be added as tag to the resources
50+
ProjectId:
51+
Type: String
52+
Description: Project Id that will be added as tag to the resources
53+
Namespace:
54+
Type: String
55+
Description: An environment name that will be prefixed to resource names
56+
57+
Conditions:
58+
KMSKeyNotProvided: !Equals [!Ref KmsKeyId, 'NONE']
59+
ImportPathNotProvided: !Equals [!Ref ImportPath, 'NONE']
60+
ExportPathNotProvided: !Equals [!Ref ExportPath, 'NONE']
61+
62+
Resources:
63+
FSXSecurityGroup:
64+
Type: AWS::EC2::SecurityGroup
65+
Properties:
66+
VpcId: !Ref VpcId
67+
SecurityGroupIngress:
68+
- CidrIp: "0.0.0.0/0"
69+
FromPort: 988
70+
ToPort: 988
71+
IpProtocol: tcp
72+
- CidrIp: "0.0.0.0/0"
73+
FromPort: 1021
74+
ToPort: 1023
75+
IpProtocol: tcp
76+
GroupDescription: FSX Security Group
77+
78+
FileSystem:
79+
Type: AWS::FSx::FileSystem
80+
Properties:
81+
FileSystemType: LUSTRE
82+
FileSystemTypeVersion: 2.12
83+
KmsKeyId: !If [KMSKeyNotProvided, !Ref 'AWS::NoValue', !Ref KmsKeyId]
84+
LustreConfiguration:
85+
AutoImportPolicy: !If [ImportPathNotProvided, !Ref 'AWS::NoValue', !Ref AutoImportPolicy]
86+
CopyTagsToBackups: True
87+
DataCompressionType: !Ref DataCompressionType
88+
DeploymentType: PERSISTENT_1
89+
ImportPath: !If [ImportPathNotProvided, !Ref 'AWS::NoValue', !Ref ImportPath]
90+
ExportPath: !If [ExportPathNotProvided, !Ref 'AWS::NoValue', !Ref ExportPath]
91+
PerUnitStorageThroughput: !Ref PerUnitStorageThroughput
92+
SecurityGroupIds: [!GetAtt FSXSecurityGroup.GroupId]
93+
StorageCapacity: !Ref StorageCapacity
94+
StorageType: SSD
95+
SubnetIds:
96+
- !Ref SubnetId
97+
Tags:
98+
- Key: cost_resource
99+
Value: !Sub ${AWS::StackName}
100+
- Key: researcher_name
101+
Value: !Ref ResearcherName
102+
- Key: project_name
103+
Value: !Ref ProjectId
104+
- Key: Name
105+
Value: !Join ['-', [Ref: Namespace, 'file-system']]
106+
107+
Outputs:
108+
FileSystemId:
109+
Value: !Ref FileSystem

‎cft-templates/nextflow-advanced.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,16 @@ Resources:
194194
Value: !Ref ResearcherName
195195
- Key: project_name
196196
Value: !Ref ProjectId
197+
- ResourceType: volume
198+
Tags:
199+
- Key: cost_resource
200+
Value: !Sub ${AWS::StackName}
201+
- Key: Name
202+
Value: !Sub worker-node-${AWS::StackName}
203+
- Key: researcher_name
204+
Value: !Ref ResearcherName
205+
- Key: project_name
206+
Value: !Ref ProjectId
197207
BlockDeviceMappings:
198208
- Ebs:
199209
DeleteOnTermination: True
@@ -608,8 +618,15 @@ Resources:
608618
process.container = '${PipelineContainer}'
609619
process.executor = 'awsbatch'
610620
process.queue = '${DefaultQueue}'
611-
workDir = 's3://${S3Bucket}/${AWS::StackName}'
621+
workDir = 's3://${S3Bucket}/${AWS::StackName}'
622+
process.errorStrategy = {
623+
sleep( Math.pow( 2, task.attempt ) * 150 as long )
624+
return 'retry'
625+
}
626+
process.maxRetries = 5
612627
aws.batch.cliPath = '/home/ec2-user/miniconda/bin/aws'
628+
aws.batch.maxParallelTransfers = 5
629+
aws.batch.maxTransferAttempts = 5
613630
}
614631
}
615632

@@ -649,6 +666,7 @@ Resources:
649666
KeyName: !Ref 'KeyPair'
650667
ImageId: '{{resolve:ssm:/RL/RG/StandardCatalog/nextflow-latest}}'
651668
IamInstanceProfile: !Ref IamInstanceProfile
669+
PropagateTagsToVolumeOnCreation: true
652670
BlockDeviceMappings:
653671
- DeviceName: /dev/xvda
654672
Ebs:

0 commit comments

Comments
 (0)