Skip to content

Commit dba4b76

Browse files
ravigurram8ravigurram8
authored
Merge pull request #113 from RLOpenCatalyst/topic-release
1.13.2 Release updates
2 parents d6bfdc2 + 06d9a1b commit dba4b76

16 files changed

Lines changed: 687 additions & 8 deletions

File tree

‎cft-templates/EFS.yaml‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
AWSTemplateFormatVersion: 2010-09-09
2+
3+
Description: Sets up EFS on AWS
4+
Metadata:
5+
AWS::CloudFormation::Interface:
6+
ParameterGroups:
7+
- Label:
8+
default: File system settings
9+
Parameters:
10+
- BackupPolicy
11+
- VpcId
12+
- SubnetId
13+
- CidrIp
14+
- Label:
15+
default: Lifecycle management
16+
Parameters:
17+
- TransitionIntoIA
18+
- TransitionToPrimaryStorageClass
19+
- PerformanceMode
20+
- ThroughputMode
21+
- ProvisionedThroughputInMibps
22+
- Label:
23+
default: Encryption
24+
Parameters:
25+
- Encrypted
26+
- KmsKeyId
27+
28+
Parameters:
29+
BackupPolicy:
30+
Description: Automatically backup your file system data with AWS Backup using recommended settings. Additional pricing applies.
31+
Type: String
32+
Default: DISABLED
33+
AllowedValues: [ENABLED,DISABLED]
34+
VpcId:
35+
Description: The ID of the VPC that the file system will be accessible from
36+
Type: AWS::EC2::VPC::Id
37+
SubnetId:
38+
Description: The subnet where to launch the service
39+
Type: AWS::EC2::Subnet::Id
40+
CidrIp:
41+
Description: The IP address range that can be used to access the file system.
42+
Type: String
43+
MinLength: '9'
44+
MaxLength: '18'
45+
Default: 0.0.0.0/0
46+
AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})
47+
ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
48+
TransitionIntoIA:
49+
Description: Migrates files that have not been accessed for a set period of time to the EFS Standard–Infrequent Access (Standard-IA) or One Zone–Infrequent Access (One Zone-IA) storage class, depending on your file system.
50+
Type: String
51+
Default: AFTER_60_DAYS
52+
AllowedValues: [AFTER_7_DAYS,AFTER_14_DAYS,AFTER_60_DAYS,AFTER_90_DAYS]
53+
TransitionToPrimaryStorageClass:
54+
Description: Automatically moves files back to the EFS Standard or EFS One Zone storage classes when the TransitionToPrimaryStorageClass lifecycle policy is set to On first access. This helps to eliminate the risk of unbounded access charges, while providing consistent low latencies.
55+
Type: String
56+
Default: NONE
57+
AllowedValues: [AFTER_1_ACCESS, NONE]
58+
PerformanceMode:
59+
Description: >-
60+
Amazon EFS delivers more than 10 gibibytes per second (GiBps) of throughput over 500,000 IOPS, and sub-millisecond or low single digit millisecond latencies.
61+
General Purpose mode supports up to 35,000 IOPS and has the lowest per-operation latency. File systems with EFS One Zone storage classes always use General Purpose performance mode.
62+
Max I/O mode supports 500,000+ IOPS and has higher per-operation latencies when compared to General Purpose mode.
63+
Type: String
64+
Default: generalPurpose
65+
AllowedValues: [generalPurpose, maxIO]
66+
ThroughputMode:
67+
Description: >-
68+
determines the throughput available to your file system.
69+
Bursting Throughput mode is a good fit for traditional applications that have a bursty throughput pattern.
70+
In Provisioned Throughput mode, you specify a level of throughput that the file system can drive independent of the file system's size or burst credit balance.
71+
Type: String
72+
Default: bursting
73+
AllowedValues: [bursting, provisioned]
74+
ProvisionedThroughputInMibps:
75+
Description: >-
76+
The amount of provisioned throughput, measured in MiB/s, for the file system. Valid for file systems using ThroughputMode set to provisioned.
77+
Valid range is 1-1024 MiB/s
78+
Type: Number
79+
Default: 1
80+
Encrypted:
81+
Description: Choose to enable encryption of your file system's data at rest. Uses the AWS KMS service key (aws/elasticfilesystem) by default
82+
Type: String
83+
Default: false
84+
AllowedValues: [true, false]
85+
KmsKeyId:
86+
Description: The ID of the AWS Key Management Service (AWS KMS) key used to encrypt Amazon EFS file system data.
87+
Type: String
88+
Default: NONE
89+
ResearcherName:
90+
Description: Researcher name that will be added as tag to the resources
91+
Type: String
92+
ProjectId:
93+
Description: Project Id that will be added as tag to the resources
94+
Type: String
95+
Namespace:
96+
Description: Name your file system. Name can include letters, numbers, and +-=._:/ symbols, up to 256 characters.
97+
Type: String
98+
99+
Conditions:
100+
burstingThroughputMode: !Equals [!Ref ThroughputMode ,bursting]
101+
KMSKeyNotProvided: !Equals [!Ref KmsKeyId, 'NONE']
102+
103+
Resources:
104+
EFSSecurityGroup:
105+
Type: "AWS::EC2::SecurityGroup"
106+
Properties:
107+
VpcId: !Ref VpcId
108+
SecurityGroupIngress:
109+
- CidrIp: !Ref CidrIp
110+
FromPort: 2049
111+
ToPort: 2049
112+
IpProtocol: tcp
113+
GroupDescription: EFS Security Group
114+
115+
FileSystem:
116+
Type: AWS::EFS::FileSystem
117+
Properties:
118+
BackupPolicy:
119+
Status: !Ref BackupPolicy
120+
Encrypted: !Ref Encrypted
121+
KmsKeyId: !If [KMSKeyNotProvided, !Ref 'AWS::NoValue', !Ref KmsKeyId]
122+
LifecyclePolicies:
123+
- TransitionToIA: !Ref TransitionIntoIA
124+
- TransitionToPrimaryStorageClass: AFTER_1_ACCESS
125+
PerformanceMode: !Ref PerformanceMode
126+
ThroughputMode: !Ref ThroughputMode
127+
ProvisionedThroughputInMibps: !If [burstingThroughputMode, !Ref 'AWS::NoValue', !Ref ProvisionedThroughputInMibps]
128+
129+
FileSystemTags:
130+
- Key: Name
131+
Value: !Ref Namespace
132+
- Key: cost_resource
133+
Value: !Sub ${AWS::StackName}
134+
- Key: researcher_name
135+
Value: !Ref ResearcherName
136+
- Key: project_name
137+
Value: !Ref ProjectId
138+
- Key: Name
139+
Value: !Join ['-', [Ref: Namespace, 'file-system']]
140+
141+
MountTarget:
142+
Type: AWS::EFS::MountTarget
143+
Properties:
144+
FileSystemId: !Ref FileSystem
145+
SubnetId: !Ref SubnetId
146+
SecurityGroups:
147+
- !Ref EFSSecurityGroup
148+
Outputs:
149+
EFS:
150+
Description: The created EFS filesystem ID
151+
Value: !Ref FileSystem
152+
MountTarget:
153+
Description: MountTarget for the created EFS filesystem.
154+
Value: !Ref MountTarget

‎config/access-policy.json‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,25 @@
110110
"cognito-idp:DeleteUserPool",
111111
"codebuild:DeleteProject",
112112
"route53:*",
113-
"fsx:*"
113+
"fsx:*",
114+
"elasticfilesystem:Backup",
115+
"elasticfilesystem:CreateFileSystem",
116+
"elasticfilesystem:CreateMountTarget",
117+
"elasticfilesystem:CreateTags",
118+
"elasticfilesystem:CreateAccessPoint",
119+
"elasticfilesystem:CreateReplicationConfiguration",
120+
"elasticfilesystem:DeleteFileSystem",
121+
"elasticfilesystem:DeleteMountTarget",
122+
"elasticfilesystem:DeleteTags",
123+
"elasticfilesystem:DeleteAccessPoint",
124+
"elasticfilesystem:DeleteFileSystemPolicy",
125+
"elasticfilesystem:DeleteReplicationConfiguration",
126+
"elasticfilesystem:DescribeBackupPolicy",
127+
"elasticfilesystem:DescribeMountTargets",
128+
"elasticfilesystem:DescribeFileSystems",
129+
"elasticfilesystem:PutFileSystemPolicy",
130+
"elasticfilesystem:PutLifecycleConfiguration",
131+
"elasticfilesystem:PutBackupPolicy"
114132
],
115133
"Resource": [
116134
"*"

‎deploy.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,15 @@ rm -f nextflow-advanced.tar.gz
271271
tar -czf rstudio.tar.gz RStudio/*
272272
aws s3 cp ./rstudio.tar.gz s3://"$bucketname/"
273273
rm -f rstudio.tar.gz
274+
cp ./PCluster/machine-images/config/infra/files/pcluster/slurm-main.yaml ./PCluster/machine-images/config/infra/files/pcluster/slurm.yaml
275+
cp ./PCluster/machine-images/config/infra/files/pcluster/batch-main.yaml ./PCluster/machine-images/config/infra/files/pcluster/batch.yaml
276+
sed -i "s/tempbucket/$bucketname/g" ./PCluster/machine-images/config/infra/files/pcluster/slurm.yaml
277+
sed -i "s/tempbucket/$bucketname/g" ./PCluster/machine-images/config/infra/files/pcluster/batch.yaml
278+
tar -czf PCluster.tar.gz PCluster/*
279+
aws s3 cp ./PCluster.tar.gz s3://"$bucketname/"
280+
rm -f ./PCluster/machine-images/config/infra/files/pcluster/slurm.yaml
281+
rm -f ./PCluster/machine-images/config/infra/files/pcluster/batch.yaml
282+
rm -f PCluster.tar.gz
274283

275284
echo "Copying Database seed-data files to new bucket"
276285
cd "$localhome" || exit

‎docker-compose.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ services:
1818
- sp2net
1919

2020
cc-3102:
21-
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.13.0_b1319
21+
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.13.2_b1352
2222
secrets:
2323
- source: sp2prod-config.json
2424
target: /rlc/cc/server/app/config/config.json
@@ -42,7 +42,7 @@ services:
4242
command: /bin/bash start.sh
4343

4444
scheduler-3102:
45-
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.13.0_b1319
45+
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.13.2_b1352
4646
secrets:
4747
- source: sp2prod-config.json
4848
target: /rlc/cc/server/app/config/config.json
@@ -62,7 +62,7 @@ services:
6262
command: node ./app/scheduler/app.js
6363

6464
notificationsink:
65-
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.12.1_b1
65+
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.13.2_b3
6666
secrets:
6767
- source: sp2prod-config.json
6868
target: /opt/app/config/config.json

‎dump/configs.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,8 @@
343343
"standardCatalogLink" : "https://researchgateway.readthedocs.io/en/latest/",
344344
"ownCatalogLink" : "https://researchgateway.readthedocs.io/en/latest/",
345345
"ownCatalogWithTagLink" : "https://researchgateway.readthedocs.io/en/latest/",
346-
"isProductDailyCostMigration_v1_13_0_Required" : true
346+
"isProductDailyCostMigration_v1_13_0_Required" : true,
347+
"projectStorage" : "https://researchgateway.readthedocs.io/en/latest/GettingStarted.html#project-storage"
347348
}
348349
},
349350
{

‎dump/standardcatalogitems.json‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,5 +894,63 @@
894894
},
895895
"cost_resource" : true
896896
}
897+
},
898+
{
899+
"_id": {
900+
"$oid": "60504955135663522b880abd"
901+
},
902+
"name" : "Amazon EFS",
903+
"description" : "Amazon Elastic File System (Amazon EFS) provides a simple, serverless, set-and-forget elastic file system for use with AWS Cloud services and on-premises resources.",
904+
"fileName" : "EFS.yaml",
905+
"tags" : [
906+
{
907+
"Key" : "EstimatedTimeToProvision",
908+
"Value" : "10 Minutes"
909+
},
910+
{
911+
"Key" : "DetailsLink",
912+
"Value" : "https://docs.aws.amazon.com/efs/latest/ug/whatisefs.html"
913+
},
914+
{
915+
"Key" : "Service",
916+
"Value" : "EFS"
917+
},
918+
{
919+
"Key" : "TypeOfProduct",
920+
"Value" : "Research"
921+
}
922+
],
923+
"owner" : "RL",
924+
"portfolio" : "RGPortfolio",
925+
"availableRegions" : [ ],
926+
"assignedOU" : [ ],
927+
"metaData" : {
928+
"pre_provisioning" : [
929+
{
930+
"code" : "CFT_PARAMS",
931+
"params" : [
932+
{
933+
"name" : "ProjectId",
934+
"type" : "RL::SC::PARAM::HD"
935+
},
936+
{
937+
"name" : "ResearcherName",
938+
"type" : "RL::SC::PARAM::HD"
939+
},
940+
{
941+
"name" : "Namespace",
942+
"type" : "RL::SC::PARAM::HD"
943+
}
944+
]
945+
}
946+
],
947+
"post_provisioning" : [ ],
948+
"checks_before_assigning_product" : [ ],
949+
"checks_after_assigning_product" : [ ],
950+
"permission_required" : {
951+
952+
},
953+
"cost_resource" : true
954+
}
897955
}
898956
]

‎products/PCluster/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Pipelines for RLCatalyst Research Gateway based on Nextflow
2+
3+
Use the below command to start the packer script after providing necessary details in [configuration.json](machine-images/config/infra/configuration.json)
4+
5+
> cd products/Nextflow-Advanced/machine-images/config/infra
6+
7+
> sudo packer build -var-file=configuration.json packer-ec2-nextflow-workspace.json

‎products/PCluster/machine-images/config/infra/configuration.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"aws_access_key": "",
3+
"aws_secret_key": "",
4+
"awsRegion": "",
5+
"vpcId": "",
6+
"subnetId": "",
7+
"amiName": "",
8+
"awsProfile": ""
9+
}

0 commit comments

Comments
 (0)