Skip to content

Commit c85ec84

Browse files
RLIndiaRLIndia
authored
Merge pull request #96 from manikandan-thangavelu-rl/topic-release
Pcluster and Fsx changes for 1.12.1
2 parents ab9f841 + b154264 commit c85ec84

4 files changed

Lines changed: 176 additions & 2 deletions

File tree

‎cft-templates/fsx-for-lustre.yaml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
AWSTemplateFormatVersion: 2010-09-09
2+
3+
Description: Sets up an FSX for Lustre filesystem with PERSISTENT_1 deployment type using SSD storage.
4+
5+
Parameters:
6+
StorageCapacity:
7+
Description: >-
8+
The storage capacity of the file system that you're creating.
9+
The valid values are 1200 GiB, 2400 GiB, and increments of 2400 GiB.
10+
Default: 1200
11+
Type: Number
12+
ImportPath:
13+
Description: The path to the Amazon S3 bucket (including the optional prefix) that you're using as the data repository for your Amazon FSx for Lustre file system
14+
Default: NONE
15+
Type: String
16+
AllowedPattern: ^[^\u0000\u0085\u2028\u2029\r\n]{3,4357}$
17+
ExportPath:
18+
Description: The path in the Amazon S3 bucket where the root of your Amazon FSx file system is exported.
19+
Default: NONE
20+
Type: String
21+
AllowedPattern: ^[^\u0000\u0085\u2028\u2029\r\n]{3,4357}$
22+
VpcId:
23+
Description: The ID of the VPC that the file system will be accessible from
24+
Type: AWS::EC2::VPC::Id
25+
SubnetId:
26+
Description: The ID of the subnet that the file system will be accessible from. Make sure the subnet is part of the VPC selected above.
27+
Type: AWS::EC2::Subnet::Id
28+
KmsKeyId:
29+
Description: The ID of the AWS Key Management Service (AWS KMS) key used to encrypt Amazon FSx file system data.
30+
Default: NONE
31+
Type: String
32+
AutoImportPolicy:
33+
Description: Choose how Amazon FSx keeps your file and directory listings up to date as you add or modify objects in your linked S3 bucket
34+
Type: String
35+
Default: NONE
36+
AllowedValues: [NONE, NEW, NEW_CHANGED, NEW_CHANGED_DELETED]
37+
DataCompressionType:
38+
Description: Sets the data compression configuration for the file system.
39+
Type: String
40+
Default: NONE
41+
AllowedValues: [NONE, LZ4]
42+
PerUnitStorageThroughput:
43+
Description: The amount of read and write throughput for each 1 tebibyte (TiB) of file system storage capacity.
44+
Type: Number
45+
Default: 50
46+
AllowedValues: [50, 100, 200]
47+
ResearcherName:
48+
Type: String
49+
Description: Researcher name that will be added as tag to the resources
50+
ProjectId:
51+
Type: String
52+
Description: Project Id that will be added as tag to the resources
53+
Namespace:
54+
Type: String
55+
Description: An environment name that will be prefixed to resource names
56+
57+
Conditions:
58+
KMSKeyNotProvided: !Equals [!Ref KmsKeyId, 'NONE']
59+
ImportPathNotProvided: !Equals [!Ref ImportPath, 'NONE']
60+
ExportPathNotProvided: !Equals [!Ref ExportPath, 'NONE']
61+
62+
Resources:
63+
FSXSecurityGroup:
64+
Type: AWS::EC2::SecurityGroup
65+
Properties:
66+
VpcId: !Ref VpcId
67+
SecurityGroupIngress:
68+
- CidrIp: "0.0.0.0/0"
69+
FromPort: 988
70+
ToPort: 988
71+
IpProtocol: tcp
72+
- CidrIp: "0.0.0.0/0"
73+
FromPort: 1021
74+
ToPort: 1023
75+
IpProtocol: tcp
76+
GroupDescription: FSX Security Group
77+
78+
FileSystem:
79+
Type: AWS::FSx::FileSystem
80+
Properties:
81+
FileSystemType: LUSTRE
82+
FileSystemTypeVersion: 2.12
83+
KmsKeyId: !If [KMSKeyNotProvided, !Ref 'AWS::NoValue', !Ref KmsKeyId]
84+
LustreConfiguration:
85+
AutoImportPolicy: !If [ImportPathNotProvided, !Ref 'AWS::NoValue', !Ref AutoImportPolicy]
86+
CopyTagsToBackups: True
87+
DataCompressionType: !Ref DataCompressionType
88+
DeploymentType: PERSISTENT_1
89+
ImportPath: !If [ImportPathNotProvided, !Ref 'AWS::NoValue', !Ref ImportPath]
90+
ExportPath: !If [ExportPathNotProvided, !Ref 'AWS::NoValue', !Ref ExportPath]
91+
PerUnitStorageThroughput: !Ref PerUnitStorageThroughput
92+
SecurityGroupIds: [!GetAtt FSXSecurityGroup.GroupId]
93+
StorageCapacity: !Ref StorageCapacity
94+
StorageType: SSD
95+
SubnetIds:
96+
- !Ref SubnetId
97+
Tags:
98+
- Key: cost_resource
99+
Value: !Sub ${AWS::StackName}
100+
- Key: researcher_name
101+
Value: !Ref ResearcherName
102+
- Key: project_name
103+
Value: !Ref ProjectId
104+
- Key: Name
105+
Value: !Join ['-', [Ref: Namespace, 'file-system']]
106+
107+
Outputs:
108+
FileSystemId:
109+
Value: !Ref FileSystem

‎cft-templates/pcluster.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Metadata:
1717
- HeadNodeInstanceType
1818
- VpcId
1919
- HeadNodeSubnetId
20+
- FileSystemId
2021
- Label:
2122
default: Scheduler Configuration
2223
Parameters:
@@ -107,6 +108,10 @@ Parameters:
107108
CustomAMI:
108109
Type: String
109110
Description: Enter the AMI Id of a custom AMI if you wish to use a non-default AMI. e.g. ami-12345678. The AMI Id entered should exist in this project account and region
111+
Default: default
112+
FileSystemId:
113+
Type: String
114+
Description: Enter the File System Id of the FSx for Lustre file system which you want to use. e.g. fs-12345678. Leave this field to default if you do not want to use FSx for Lustre
110115
Default: default
111116

112117

@@ -185,6 +190,7 @@ Resources :
185190
- dynamodb:*
186191
- route53:*
187192
- ssm:*
193+
- fsx:*
188194
Resource: "*"
189195
InstanceSecurityGroup:
190196
Type: AWS::EC2::SecurityGroup
@@ -223,8 +229,9 @@ Resources :
223229
trap '/opt/aws/bin/cfn-signal --exit-code 1 --resource EC2Instance --region ${AWS::Region} --stack ${AWS::StackName}' ERR
224230
sudo yum update -y --security
225231
cd /home/ec2-user/parallel-update
226-
sudo -u ec2-user ./Provision-pcluster.sh ${Scheduler} ${AWS::Region} ${HeadNodeInstanceType} ${HeadNodeSubnetId} ${KeyPair} ${WorkerNodeInstanceType} ${ComputeEnvMinvCpus} ${ComputeEnvMaxvCpus} ${WorkerNodeSubnetId} ${ComputeEnvDesiredvCpus} ${SpotBidPercentage} ${AWS::StackName} ${CustomAMI}
232+
sudo -u ec2-user ./Provision-pcluster.sh ${Scheduler} ${AWS::Region} ${HeadNodeInstanceType} ${HeadNodeSubnetId} ${KeyPair} ${WorkerNodeInstanceType} ${ComputeEnvMinvCpus} ${ComputeEnvMaxvCpus} ${WorkerNodeSubnetId} ${ComputeEnvDesiredvCpus} ${SpotBidPercentage} ${AWS::StackName} ${CustomAMI} ${FileSystemId}
227233
/opt/aws/bin/cfn-signal --exit-code 0 --resource EC2Instance --region ${AWS::Region} --stack ${AWS::StackName}
234+
sleep 10
228235
aws ec2 stop-instances --instance-ids `wget -qO- http://instance-data/latest/meta-data/instance-id`
229236

230237
Outputs:

‎config/access-policy.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
"iam:GetRolePolicy",
4747
"iam:GetPolicy",
4848
"iam:ListPolicyVersions",
49+
"iam:CreateServiceLinkedRole",
4950
"batch:*",
5051
"sts:AssumeRole",
5152
"sts:GetFederationToken",
@@ -108,7 +109,8 @@
108109
"imagebuilder:UntagResource",
109110
"cognito-idp:DeleteUserPool",
110111
"codebuild:DeleteProject",
111-
"route53:*"
112+
"route53:*",
113+
"fsx:*"
112114
],
113115
"Resource": [
114116
"*"

‎dump/standardcatalogitems.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,5 +760,61 @@
760760

761761
}
762762
}
763+
},
764+
{
765+
"_id": {
766+
"$oid": "60504955135663522b880cfd"
767+
},
768+
"name": "FSx For Lustre",
769+
"description": "Sets up an FSX for Lustre filesystem with PERSISTENT_1 deployment type using SSD storage",
770+
"fileName": "fsx-for-lustre.yaml",
771+
"tags": [
772+
{
773+
"Key": "EstimatedTimeToProvision",
774+
"Value": "20 Minutes"
775+
},
776+
{
777+
"Key": "DetailsLink",
778+
"Value": "https://docs.aws.amazon.com/fsx/latest/LustreGuide/what-is.html"
779+
},
780+
{
781+
"Key": "Service",
782+
"Value": "FSXForLustre"
783+
},
784+
{
785+
"Key": "TypeOfProduct",
786+
"Value": "Research"
787+
}
788+
],
789+
"owner": "RL",
790+
"portfolio": "RGPortfolio",
791+
"availableRegions": [],
792+
"assignedOU": [],
793+
"metaData": {
794+
"pre_provisioning": [
795+
{
796+
"code": "CFT_PARAMS",
797+
"params": [
798+
{
799+
"name": "ProjectId",
800+
"type": "RL::SC::PARAM::HD"
801+
},
802+
{
803+
"name": "ResearcherName",
804+
"type": "RL::SC::PARAM::HD"
805+
},
806+
{
807+
"name": "Namespace",
808+
"type": "RL::SC::PARAM::HD"
809+
}
810+
]
811+
}
812+
],
813+
"post_provisioning": [],
814+
"checks_before_assigning_product": [],
815+
"checks_after_assigning_product": [],
816+
"permission_required": {},
817+
"cost_resource": true
818+
}
763819
}
764820
]

0 commit comments

Comments
 (0)