Skip to content

Commit 9aa085e

Browse files
RLIndiaRLIndia
authored
Merge pull request #66 from RLOpenCatalyst/topic-release
Topic release
2 parents 13cdd25 + ae19aea commit 9aa085e

11 files changed

Lines changed: 220 additions & 27 deletions

File tree

‎cft-templates/nextflow-advanced.yaml‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ Parameters:
8888
Description: Head Node EC2 instance type
8989
Type: String
9090
Default: t2.small
91-
AllowedValues: [t2.nano, t2.micro, t2.small, t2.medium]
91+
AllowedValues: [t2.nano, t2.micro, t2.small, t2.medium, t2.large, m5.8xlarge]
9292
ConstraintDescription: Must be a valid EC2 instance type.
9393
KeyPair:
9494
Description: Name of an existing EC2 KeyPair to enable SSH access to the Head Node. If no key pairs exist, please create one from the button next to the dropdown. Please contact your Administrator if you are unable to create one.
@@ -110,7 +110,7 @@ Parameters:
110110
Description: Specify the instance types to be used to carry out the computation. You can specify one or more family or instance type. The option 'optimal' chooses the best fit of M4, C4, and R4 instance types available in the region.
111111
Type: String
112112
Default: optimal
113-
AllowedValues: [optimal, c4.large, m4.large, r4.large, c4.4xlarge, m4.4xlarge, r4.4xlarge]
113+
AllowedValues: [optimal, c4.large, m4.large, r4.large, c4.4xlarge, m4.4xlarge, r4.4xlarge, r5.8xlarge, r5.12xlarge, r5.16xlarge, r5.24xlarge, c6i.32xlarge]
114114
WorkerNodeEBSVolumeSize:
115115
Description: The initial size of the volume (in GBs) Worker Node EBS will use for storage.
116116
Type: Number
@@ -505,8 +505,7 @@ Resources:
505505
LaunchTemplateId: !Ref EC2LaunchTemplate
506506
Version: $Latest
507507
InstanceRole: !GetAtt BatchInstanceProfile.Arn
508-
InstanceTypes:
509-
- !Ref WorkerNodeInstanceType
508+
InstanceTypes: [c5, c5a, c5ad, c5d, i3, m5, r5, c6i ]
510509
MinvCpus: !Ref ComputeEnvMinvCpus
511510
MaxvCpus: !Ref ComputeEnvMaxvCpus
512511
SecurityGroupIds:
@@ -522,11 +521,8 @@ Resources:
522521
Type: AWS::Batch::JobQueue
523522
Properties:
524523
Tags:
525-
Name: !Sub worker-node-queue-${AWS::StackName}
526-
JobQueueName:
527-
!Sub
528-
- 'job-queue-${RandomGUID}'
529-
- { RandomGUID: !Select [0, !Split ["-", !Select [2, !Split ["/", !Ref AWS::StackId ]]]] }
524+
Name: !Sub worker-node-queue-${Namespace}
525+
JobQueueName: !Sub worker-node-queue-${Namespace}
530526
Priority: 1
531527
State: ENABLED
532528
ComputeEnvironmentOrder:
@@ -545,7 +541,9 @@ Resources:
545541
exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
546542

547543
cd /usr/bin
548-
curl -s https://get.nextflow.io | bash
544+
sudo wget https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm
545+
sudo rpm -ivh jdk-17_linux-x64_bin.rpm
546+
sudo wget -qO- https://github.com/nextflow-io/nextflow/releases/download/v21.11.0-edge/nextflow-21.11.0-edge-all | sudo bash
549547

550548
sudo yum update -y --security
551549
cd /root/
@@ -583,7 +581,10 @@ Resources:
583581
echo $InputDataLocation
584582
fi
585583

586-
cd .nextflow/
584+
sudo chown -R ec2-user:ec2-user /usr/bin/nextflow
585+
sudo cp -prf /root/.nextflow/ /home/ec2-user/
586+
cd /home/ec2-user/.nextflow/
587+
587588
PROJECT_NF_CONFIG=config
588589

589590
cat << EOF >> $PROJECT_NF_CONFIG
@@ -606,8 +607,15 @@ Resources:
606607
process.container = '${PipelineContainer}'
607608
process.executor = 'awsbatch'
608609
process.queue = '${DefaultQueue}'
609-
workDir = 's3://${S3Bucket}/${AWS::StackName}'
610+
workDir = 's3://${S3Bucket}/${AWS::StackName}'
611+
process.errorStrategy = {
612+
sleep( Math.pow( 2, task.attempt ) * 150 as long )
613+
return 'retry'
614+
}
615+
process.maxRetries = 5
610616
aws.batch.cliPath = '/home/ec2-user/miniconda/bin/aws'
617+
aws.batch.maxParallelTransfers = 5
618+
aws.batch.maxTransferAttempts = 5
611619
}
612620
}
613621

@@ -640,7 +648,7 @@ Resources:
640648
chmod 500 "/tmp/get_bootstrap.sh"
641649
/tmp/get_bootstrap.sh "${EnvironmentInstanceFiles}" '${S3Mounts}'
642650
/opt/aws/bin/cfn-signal --exit-code 0 --resource EC2Instance --region ${AWS::Region} --stack ${AWS::StackName}
643-
651+
644652
InstanceType: !Ref 'InstanceType'
645653
SecurityGroupIds:
646654
- { "Fn::GetAtt" : ["InstanceSecurityGroup", "GroupId"] }
@@ -711,3 +719,5 @@ Outputs:
711719
Value: !Ref EC2LaunchTemplate
712720
WorkDataLocation:
713721
Value: !Sub ${S3Bucket}
722+
BatchQueue:
723+
Value: !Sub ${DefaultQueue}

‎cft-templates/s3.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,21 @@
11
AWSTemplateFormatVersion: '2010-09-09'
22
Description: create a single S3 bucket
33

4+
Parameters:
5+
Namespace:
6+
Type: String
7+
Description: An environment name that will be prefixed to resource names
8+
49
Resources:
510
SampleBucket:
611
Type: AWS::S3::Bucket
12+
Properties:
13+
BucketName: !Ref Namespace
714

815
Outputs:
916
BucketArn:
1017
Value: !GetAtt [SampleBucket, Arn]
1118
Description: URL for website hosted on S3
1219
BucketName:
1320
Value: !Ref SampleBucket
14-
Description: Name of S3 bucket
21+
Description: Name of S3 bucket

‎cft-templates/users.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
email,first_name,last_name,role,userTags
2+
john.doe@example.com,Student,1,0,"tag1,tag2,tag3"

‎config/config.json‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@
104104
"BucketName": "REPLACE_WITH_S3_BUCKET_NAME_CONTAINING_CFT_TEMPLATES",
105105
"cftTemplateURL": "REPLACE_WITH_S3_BUCKET_URL_CONTAINING_CFT_TEMPLATES",
106106
"RGUser": "RG-PortalUser",
107-
"DescribeStackRetry": 2,
107+
"DescribeStackRetry": 20,
108108
"DefaultCatalogAssignedToOrg": "Research",
109109
"userSetting": {
110110
"userRoleMapping": {
@@ -193,5 +193,14 @@
193193
"defaultLimit":10,
194194
"linksForRg": {
195195
"termsAndConditions": "REPLACE_WITH_TERMS_CONDITIONS_DOC_URL"
196-
}
196+
},
197+
"s3BucketLimit" : 100,
198+
"eodReportTime":{
199+
"startTime": "20:00:00",
200+
"endTime": "21:10:00"
201+
},
202+
"cronJobForEodSync" : "*/60 * * * *",
203+
"sampleCSVForUser": "users.csv",
204+
"sampleCSVBucketRegion": "REPLACE_WITH_REGION",
205+
"sessionMaxAge" : 900000
197206
}

‎config/settings-config.json‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,79 @@
400400
"terminated"
401401
]
402402
},
403+
"cromwell": {
404+
"actions": {
405+
"running": {
406+
"connect": [
407+
{
408+
"menu": "SSH/RDP",
409+
"imageUrl": "../../assets/images/technology@2x.png"
410+
},
411+
{
412+
"menu": "View Outputs",
413+
"imageUrl": "../../assets/images/link@2x.png",
414+
"outputsRequired": [
415+
"InstanceIPAddress",
416+
"InstanceId"
417+
]
418+
}
419+
],
420+
"action": [
421+
{
422+
"action": "Stop",
423+
"imageUrl": "../../assets/images/stop.png"
424+
},
425+
{
426+
"action": "Reboot",
427+
"imageUrl": "../../assets/images/reset@2x.png"
428+
}
429+
]
430+
},
431+
"stopped": {
432+
"connect": [],
433+
"action": [
434+
{
435+
"action": "Start",
436+
"imageUrl": "../../assets/images/play@2x.png"
437+
}
438+
]
439+
},
440+
"default": {
441+
"connect": [
442+
{
443+
"menu": "SSH/RDP",
444+
"imageUrl": "../../assets/images/technology@2x.png"
445+
}
446+
],
447+
"action": [
448+
{
449+
"action": "Stop",
450+
"imageUrl": "../../assets/images/stop.png"
451+
},
452+
{
453+
"action": "Reboot",
454+
"imageUrl": "../../assets/images/reset@2x.png"
455+
}
456+
]
457+
}
458+
},
459+
"transient_states": [
460+
"pending",
461+
"shutting-down",
462+
"stopping",
463+
"deleting"
464+
],
465+
"active_states": [
466+
"running",
467+
"active"
468+
],
469+
"stopped_states": [
470+
"stopped"
471+
],
472+
"terminated_states": [
473+
"terminated"
474+
]
475+
},
403476
"wordpress": {
404477
"actions": {
405478
"running": {
@@ -651,7 +724,8 @@
651724
"emr",
652725
"rds",
653726
"rstudio",
654-
"nextflow"
727+
"nextflow",
728+
"cromwell"
655729
],
656730
"cft_active_states": [
657731
"available",

‎config/snsConfig.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,5 +51,6 @@
5151
"snsProtocol": "REPLACE_WITH_HTTP_HTTPS",
5252
"endpoint": "notificationsink/productProvisioningUpdates",
5353
"notificationSinkCleanUpRoute": "unlinkAccount",
54-
"snsUrlKey": "snsUrl"
54+
"snsUrlKey": "snsUrl",
55+
"sendEmailRoute":"sendEmail"
5556
}

‎docker-compose.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ services:
1818
- sp2net
1919

2020
cc-3102:
21-
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.11.1_b1113
21+
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.11.3_b1152
2222
secrets:
2323
- source: sp2prod-config.json
2424
target: /rlc/cc/server/app/config/config.json
@@ -42,7 +42,7 @@ services:
4242
command: /bin/bash start.sh
4343

4444
scheduler-3102:
45-
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.11.1_b1113
45+
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.11.3_b1152
4646
secrets:
4747
- source: sp2prod-config.json
4848
target: /rlc/cc/server/app/config/config.json
@@ -62,7 +62,7 @@ services:
6262
command: node ./app/scheduler/app.js
6363

6464
notificationsink:
65-
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.11.0_b4
65+
image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.11.2_b2
6666
secrets:
6767
- source: sp2prod-config.json
6868
target: /opt/app/config/config.json

‎dump/configs.json‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,11 @@
113113
"value": "Example : This project uses remotely sensed data products and measurements for studying processes.",
114114
"tags": ["project_desc"]
115115
},
116+
{
117+
"key" : "How can I add more than 1 project at a time?",
118+
"value" : "We have a Project Copies field using which we can create N number of projects at a time.",
119+
"tags" : ["project_copies"]
120+
},
116121
{
117122
"key": "Budget Available : Budget should be greater then 0",
118123
"value": "Example : 1000",
@@ -137,6 +142,16 @@
137142
"key": "What products can I get from the Standard Catalog",
138143
"value": "The RLCatalyst Research Gateway standard catalog brings in many of the products commonly used in scientific research computing. The products span many different categories like compute, storage, databases etc. For a full list click https://researchgateway.readthedocs.io/en/latest/Appendices.html#appendix-b",
139144
"tags": ["add_products"]
145+
},
146+
{
147+
"key" : "What products can I get from the Bring all catalog items?",
148+
"value" : "This option will sync all the products which do not have a project tag that matches any of the projects in this account. For BYOC to work, the role assumed by the Portal in the project account must be added to the portfolio manually",
149+
"tags" : ["ownCatalog"]
150+
},
151+
{
152+
"key" : "What products can I get from the Bring specific catalog items?",
153+
"value" : "This option will pull in any products that are tagged with the same tag which user provides in the UI. For BYOC to work, the role assumed by the Portal in the project account must be added to the portfolio manually",
154+
"tags" : ["ownCatalogWithTag"]
140155
}
141156
],
142157
"configType": "create_project_helpText"
@@ -209,7 +224,7 @@
209224
},
210225
"key": "projectStorage",
211226
"value": {
212-
"productList": ["ec2", "rstudio", "nextflow"],
227+
"productList": ["ec2", "rstudio", "nextflow", "cromwell"],
213228
"s3Mounts": {
214229
"ProjectStorage": {
215230
"id": "ProjectStorage",
@@ -314,7 +329,68 @@
314329
"defaultAllowProjectActionsOverride": true,
315330
"projectPauseThresholdPercentage": 80,
316331
"projectStopThresholdPercentage": 90,
317-
"costControlReadMoreLink": "https://researchgateway.readthedocs.io/en/latest/"
332+
"costControlReadMoreLink": "https://researchgateway.readthedocs.io/en/latest/",
333+
"budgetMaxAllowedValue": 50000
334+
}
335+
},
336+
{
337+
"_id" : {
338+
"$oid": "626fb82256aaa6ec4fe242e9"
339+
},
340+
"key" : "projectConfig",
341+
"value" : {
342+
"maxProjectCount" : "10",
343+
"standardCatalogLink" : "https://researchgateway.readthedocs.io/en/latest/",
344+
"ownCatalogLink" : "https://researchgateway.readthedocs.io/en/latest/",
345+
"ownCatalogWithTagLink" : "https://researchgateway.readthedocs.io/en/latest/"
318346
}
347+
},
348+
{
349+
"_id" : {
350+
"$oid": "62b6c70a7dec087675b33797"
351+
},
352+
"key" : "usersConfig",
353+
"value" : {
354+
"maxBulkUserCount" : 20,
355+
"csvHeaderValues" : [
356+
"email",
357+
"first_name",
358+
"last_name",
359+
"role",
360+
"userTags"
361+
]
362+
}
363+
},
364+
{
365+
"_id" : {
366+
"$oid": "628b094788660e80ceb36ba3"
367+
},
368+
"key": "cromwellPipeline",
369+
"value": [
370+
{
371+
"pipeline_name": "seq-format-validation",
372+
"pipeline_git_url": "https://github.com/gatk-workflows/seq-format-validation.git"
373+
},
374+
{
375+
"pipeline_name": "seq-format-conversion",
376+
"pipeline_git_url": "https://github.com/gatk-workflows/seq-format-conversion.git"
377+
},
378+
{
379+
"pipeline_name": "gatk4-rnaseq-germline-snps-indels",
380+
"pipeline_git_url": "https://github.com/gatk-workflows/gatk4-rnaseq-germline-snps-indels.git"
381+
},
382+
{
383+
"pipeline_name": "gatk4-germline-snps-indels",
384+
"pipeline_git_url": "https://github.com/gatk-workflows/gatk4-germline-snps-indels.git"
385+
},
386+
{
387+
"pipeline_name": "gatk4-data-processing",
388+
"pipeline_git_url": "https://github.com/gatk-workflows/gatk4-data-processing.git"
389+
},
390+
{
391+
"pipeline_name": "gatk4-basic-joint-genotyping",
392+
"pipeline_git_url": "https://github.com/gatk-workflows/gatk4-basic-joint-genotyping.git"
393+
}
394+
]
319395
}
320396
]

0 commit comments

Comments
 (0)