Skip to content

Commit 7fe0168

Browse files
RLIndiaRLIndia
authored
Merge pull request #84 from RLOpenCatalyst/topic-release
1.12.0 build updates
2 parents e47ecaf + 9a532c2 commit 7fe0168

8 files changed

Lines changed: 422 additions & 16 deletions

File tree

‎cft-templates/cromwell-advanced.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -867,6 +867,7 @@ Resources:
867867
ImageId: !Ref 'LatestAmiId'
868868
InstanceType: !Ref InstanceType
869869
IamInstanceProfile: !Ref EC2InstanceProfile
870+
PropagateTagsToVolumeOnCreation: true
870871
SecurityGroupIds:
871872
- { "Fn::GetAtt" : ["EC2SecurityGroup", "GroupId"] }
872873
KeyName: !Ref 'KeyPair'

‎cft-templates/nextflow-advanced.yaml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -608,8 +608,15 @@ Resources:
608608
process.container = '${PipelineContainer}'
609609
process.executor = 'awsbatch'
610610
process.queue = '${DefaultQueue}'
611-
workDir = 's3://${S3Bucket}/${AWS::StackName}'
611+
workDir = 's3://${S3Bucket}/${AWS::StackName}'
612+
process.errorStrategy = {
613+
sleep( Math.pow( 2, task.attempt ) * 150 as long )
614+
return 'retry'
615+
}
616+
process.maxRetries = 5
612617
aws.batch.cliPath = '/home/ec2-user/miniconda/bin/aws'
618+
aws.batch.maxParallelTransfers = 5
619+
aws.batch.maxTransferAttempts = 5
613620
}
614621
}
615622

@@ -649,6 +656,7 @@ Resources:
649656
KeyName: !Ref 'KeyPair'
650657
ImageId: '{{resolve:ssm:/RL/RG/StandardCatalog/nextflow-latest}}'
651658
IamInstanceProfile: !Ref IamInstanceProfile
659+
PropagateTagsToVolumeOnCreation: true
652660
BlockDeviceMappings:
653661
- DeviceName: /dev/xvda
654662
Ebs:

‎cft-templates/pcluster.yaml‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
AWSTemplateFormatVersion: 2010-09-09
2+
3+
Description: Sets up an pcluster on AWS
4+
Metadata:
5+
AWS::CloudFormation::Interface:
6+
ParameterGroups:
7+
- Label:
8+
default: PCluster Launcher Node Configuration
9+
Parameters:
10+
- InstanceType
11+
- SSHLocation
12+
- KeyPair
13+
- CustomAMI
14+
- Label:
15+
default: Head Node Configuration
16+
Parameters:
17+
- HeadNodeInstanceType
18+
- VpcId
19+
- HeadNodeSubnetId
20+
- Label:
21+
default: Scheduler Configuration
22+
Parameters:
23+
- Scheduler
24+
- WorkerNodeInstanceType
25+
- WorkerNodeSubnetId
26+
- ComputeEnvMinvCpus
27+
- ComputeEnvMaxvCpus
28+
- ComputeEnvDesiredvCpus
29+
- SpotBidPercentage
30+
- ResearcherName
31+
- ProjectId
32+
ProductName: Pcluster
33+
34+
35+
Parameters:
36+
InstanceType:
37+
Type: String
38+
Description: Instance type for pcluster. Default is t2.micro.
39+
AllowedValues:
40+
- t2.micro
41+
- t2.small
42+
- t2.medium
43+
- t2.large
44+
- t3.micro
45+
- t3.small
46+
Description: The instance type to be used for the launcher node
47+
Default: t3.micro
48+
KeyPair:
49+
Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. If no key pairs exist, please create one from the button next to the dropdown. Please contact your Administrator if you are unable to create one.
50+
Type: "AWS::EC2::KeyPair::KeyName"
51+
SSHLocation:
52+
Description: The IP address range that can be used to SSH to the EC2 instances
53+
Type: String
54+
MinLength: '9'
55+
MaxLength: '18'
56+
Default: 0.0.0.0/0
57+
AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})
58+
ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
59+
ResearcherName:
60+
Type: String
61+
ProjectId:
62+
Type: String
63+
Scheduler:
64+
Description: Cluster scheduler
65+
Type: String
66+
Default: awsbatch
67+
ConstraintDescription: Must be a supported scheduler
68+
AllowedValues:
69+
- slurm
70+
- awsbatch
71+
VpcId:
72+
Type: AWS::EC2::VPC::Id
73+
Description: 'The VPC to create security groups and deploy AWS Batch or slurm to.'
74+
HeadNodeInstanceType:
75+
Description: Head Node EC2 instance type
76+
Type: String
77+
Default: t3.micro
78+
AllowedValues: [t2.nano, t2.micro, t2.small, t2.medium, t3.micro, t3.small, t3.medium, t3.large, t3.xlarge]
79+
ConstraintDescription: Must be a valid EC2 instance type.
80+
WorkerNodeInstanceType:
81+
Description: Specify the instance types to be used to carry out the computation.
82+
Type: String
83+
Default: c4.large
84+
AllowedValues: [c4.large, m4.large, r4.large, c4.4xlarge, m4.4xlarge, r4.4xlarge]
85+
HeadNodeSubnetId:
86+
Type: AWS::EC2::Subnet::Id
87+
Description: 'Subnet you want your Head Node to launch in. You must select a public subnet.'
88+
WorkerNodeSubnetId:
89+
Type: AWS::EC2::Subnet::Id
90+
Description: 'Subnet you want your Batch or slurm Worker Node to launch in. We recommend public subnets.'
91+
ComputeEnvDesiredvCpus:
92+
Description: The Desired number of CPUs for the default Batch Compute Environment
93+
Type: Number
94+
Default: 0
95+
ComputeEnvMinvCpus:
96+
Description: The minimum number of CPUs to be kept in running state for the Batch/slurm Worker Nodes. If you give a non-zero value, some worker nodes may stay in running state always and you may incur higher cost.
97+
Type: Number
98+
Default: 0
99+
ComputeEnvMaxvCpus:
100+
Description: The maximum number of CPUs for the default Batch or slurm Compute Environment
101+
Type: Number
102+
Default: 10
103+
SpotBidPercentage:
104+
Type: Number
105+
Description: The maximum percentage of On-Demand pricing you want to pay for Spot resources. You will always pay the lowest Spot market price and never more than your maximum percentage.
106+
Default: 100
107+
CustomAMI:
108+
Type: String
109+
Description: Enter the AMI Id of a custom AMI if you wish to use a non-default AMI. e.g. ami-12345678. The AMI Id entered should exist in this project account and region
110+
Default: default
111+
112+
113+
Resources :
114+
PortalRole:
115+
Type: AWS::IAM::Role
116+
Properties:
117+
Description: The role used by the EC2 Instance running the portal
118+
AssumeRolePolicyDocument:
119+
Statement:
120+
- Effect: Allow
121+
Principal:
122+
Service:
123+
- ec2.amazonaws.com
124+
Action:
125+
- sts:AssumeRole
126+
ManagedPolicyArns:
127+
- arn:aws:iam::aws:policy/AmazonS3FullAccess
128+
- arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
129+
- arn:aws:iam::aws:policy/AWSBatchFullAccess
130+
- arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole
131+
- arn:aws:iam::aws:policy/service-role/AWSAppRunnerServicePolicyForECRAccess
132+
- arn:aws:iam::aws:policy/AmazonElasticContainerRegistryPublicPowerUser
133+
- arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser
134+
- arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
135+
- arn:aws:iam::aws:policy/IAMFullAccess
136+
- Ref: GatewayRolePolicy
137+
Path: /
138+
GatewayInstanceProfile:
139+
Type: AWS::IAM::InstanceProfile
140+
Properties:
141+
Path: /
142+
Roles:
143+
- Ref: PortalRole
144+
GatewayRolePolicy:
145+
Type: AWS::IAM::ManagedPolicy
146+
Properties:
147+
ManagedPolicyName: !Sub ${AWS::StackName}
148+
PolicyDocument:
149+
Version: "2012-10-17"
150+
Statement:
151+
- Effect: Allow
152+
Action:
153+
- config:*
154+
- ec2:Describe*
155+
- ec2:CopyImage
156+
- ec2:*
157+
- ec2:ModifyImageAttribute
158+
- ec2:ModifyInstanceMetadataOptions
159+
- ec2:DescribeRegions
160+
- ec2:DescribeAvailabilityZones
161+
- iam:attachRolePolicy
162+
- iam:createPolicy
163+
- iam:createPolicyVersion
164+
- iam:createRole
165+
- iam:deletePolicy
166+
- iam:deletePolicyVersion
167+
- iam:detachRolePolicy
168+
- iam:updateAssumeRolePolicy
169+
- elasticloadbalancing:RegisterTargets
170+
- ses:*
171+
- sns:*
172+
- logs:*
173+
- lambda:*
174+
- events:*
175+
- ecr:CreateRepository
176+
- ecr:ReplicateImage
177+
- ecr:*
178+
- codebuild:*
179+
- iam:GetPolicy
180+
- iam:GetPolicyVersion
181+
- iam:GetRole
182+
- sts:AssumeRole
183+
- cloudformation:*
184+
- cloudwatch:*
185+
- dynamodb:*
186+
- route53:*
187+
- ssm:*
188+
Resource: "*"
189+
InstanceSecurityGroup:
190+
Type: AWS::EC2::SecurityGroup
191+
Properties:
192+
GroupDescription: Enable SSH access
193+
SecurityGroupIngress:
194+
- IpProtocol: tcp
195+
FromPort: '22'
196+
ToPort: '22'
197+
CidrIp: !Ref 'SSHLocation'
198+
EC2Instance :
199+
Type : AWS::EC2::Instance
200+
CreationPolicy:
201+
ResourceSignal:
202+
Timeout: PT40M
203+
Properties :
204+
KeyName: !Ref KeyPair
205+
ImageId: '{{resolve:ssm:/RL/RG/StandardCatalog/ParallelCluster-linux-ami}}'
206+
InstanceType: !Ref InstanceType
207+
SecurityGroups: [!Ref 'InstanceSecurityGroup']
208+
PropagateTagsToVolumeOnCreation: true
209+
IamInstanceProfile:
210+
Ref: GatewayInstanceProfile
211+
Tags:
212+
- Key: researcher_name
213+
Value: !Ref ResearcherName
214+
- Key: project_name
215+
Value: !Ref ProjectId
216+
- Key: cost_resource
217+
Value: !Sub ${AWS::StackName}
218+
UserData:
219+
Fn::Base64:
220+
!Sub |
221+
#!/bin/bash
222+
set -e
223+
trap '/opt/aws/bin/cfn-signal --exit-code 1 --resource EC2Instance --region ${AWS::Region} --stack ${AWS::StackName}' ERR
224+
sudo yum update -y --security
225+
cd /home/ec2-user/parallel-update
226+
sudo -u ec2-user ./Provision-pcluster.sh ${Scheduler} ${AWS::Region} ${HeadNodeInstanceType} ${HeadNodeSubnetId} ${KeyPair} ${WorkerNodeInstanceType} ${ComputeEnvMinvCpus} ${ComputeEnvMaxvCpus} ${WorkerNodeSubnetId} ${ComputeEnvDesiredvCpus} ${SpotBidPercentage} ${AWS::StackName} ${CustomAMI}
227+
/opt/aws/bin/cfn-signal --exit-code 0 --resource EC2Instance --region ${AWS::Region} --stack ${AWS::StackName}
228+
aws ec2 stop-instances --instance-ids `wget -qO- http://instance-data/latest/meta-data/instance-id`
229+
230+
Outputs:
231+
InstanceId:
232+
Description: InstanceId of the newly created EC2 instance
233+
Value: !Ref 'EC2Instance'
234+
InstanceIPAddress:
235+
Description: IP address of the newly created EC2 instance
236+
Value: !GetAtt EC2Instance.PublicIp
237+
ClusterName:
238+
Description: Name of the Plcuster created
239+
Value: !Join ['-', ["RG", "Pcluster", !Select [1, !Split ["-", !Ref EC2Instance ]]]]

‎config/access-policy.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,9 @@
106106
"imagebuilder:GetInfrastructureConfiguration",
107107
"imagebuilder:TagResource",
108108
"imagebuilder:UntagResource",
109-
"cognito-idp:DeleteUserPool"
109+
"cognito-idp:DeleteUserPool",
110+
"codebuild:DeleteProject",
111+
"route53:*"
110112
],
111113
"Resource": [
112114
"*"

‎config/settings-config.json‎

Lines changed: 74 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,6 @@
7777
}
7878
],
7979
"action": [
80-
{
81-
"action": "Share",
82-
"imageUrl": "../../assets/images/share@2x.png"
83-
},
8480
{
8581
"action": "Unshare",
8682
"imageUrl": "../../assets/images/unshare@2x.png"
@@ -107,10 +103,6 @@
107103
}
108104
],
109105
"action": [
110-
{
111-
"action": "Share",
112-
"imageUrl": "../../assets/images/share@2x.png"
113-
},
114106
{
115107
"action": "Upload",
116108
"imageUrl": "../../assets/images/cloud-computing@2x.png"
@@ -215,6 +207,78 @@
215207
"terminated"
216208
]
217209
},
210+
"pcluster": {
211+
"actions": {
212+
"running": {
213+
"connect": [
214+
{
215+
"menu": "SSH to Pcluster HeadNode",
216+
"imageUrl": "../../assets/images/technology@2x.png",
217+
"outputsRequired": []
218+
}
219+
],
220+
"action": [
221+
{
222+
"action": "Stop",
223+
"imageUrl": "../../assets/images/no-stopping@2x.png",
224+
"outputsRequired": []
225+
},
226+
{
227+
"action": "Reboot",
228+
"imageUrl": "../../assets/images/reset@2x.png",
229+
"outputsRequired": []
230+
}
231+
]
232+
},
233+
"stopped": {
234+
"connect": [],
235+
"action": [
236+
{
237+
"action": "Start",
238+
"imageUrl": "../../assets/images/play@2x.png",
239+
"outputsRequired": []
240+
}
241+
]
242+
},
243+
"default": {
244+
"connect": [
245+
{
246+
"menu": "SSH to Pcluster HeadNode",
247+
"imageUrl": "../../assets/images/technology@2x.png",
248+
"outputsRequired": []
249+
}
250+
],
251+
"action": [
252+
{
253+
"action": "Stop",
254+
"imageUrl": "../../assets/images/no-stopping@2x.png",
255+
"outputsRequired": []
256+
},
257+
{
258+
"action": "Reboot",
259+
"imageUrl": "../../assets/images/reset@2x.png",
260+
"outputsRequired": []
261+
}
262+
]
263+
}
264+
},
265+
"transient_states": [
266+
"pending",
267+
"shutting-down",
268+
"stopping",
269+
"deleting"
270+
],
271+
"active_states": [
272+
"running",
273+
"active"
274+
],
275+
"stopped_states": [
276+
"stopped"
277+
],
278+
"terminated_states": [
279+
"terminated"
280+
]
281+
},
218282
"rstudio": {
219283
"actions": {
220284
"running": {
@@ -745,7 +809,8 @@
745809
"rds",
746810
"rstudio",
747811
"nextflow",
748-
"cromwell"
812+
"cromwell",
813+
"pcluster"
749814
],
750815
"cft_active_states": [
751816
"available",

0 commit comments

Comments
 (0)