Merge branch 'topic-release' into main

Author: RLIndia

cft-templates/nextflow-advanced.yaml

@@ -16,7 +16,8 @@ Metadata:
         Parameters:
           - PipelineName
           - PipelineContainer
-          - InputDataLocation 
+          - InputDataLocation
+          - InputDataPattern 
           - OutputDataLocation
       - Label:
           default: Head Node Configuration
@@ -58,7 +59,12 @@ Parameters:
     Description: >-
       An S3 bucket which holds input data for the Nextflow pipeline. The bucket name must respect the S3 bucket naming conventions
       (can contain lowercase letters, numbers, periods and hyphens).
-    Type: String 
+    Type: String
+  InputDataPattern:
+    Description: >-
+      The pattern to match samples to be processed as inputs to the pipeline. E.g. data/ggal/*_{1,2}.fq. It can also point to a csv or tsv file that contains details of the files to be processed.
+    Type: String
+    Default: samplesheet.csv 
   OutputDataLocation:
     Description: >-
       The full path on the local disk where outputs of the pipeline should be stored. The default path above will enable you to view the outputs via the browser. The path should be accessible to the user ec2-user.
@@ -539,13 +545,15 @@ Resources:
           exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 
           sudo yum update -y --security
-          cd /root/ 
-  
-          getFirstChar=`echo $OutputDataLocation | cut -c1-1`
+          cd /root/  
+          
+          getOutputData=${OutputDataLocation}
+          getFirstChar=`echo $getOutputData | cut -c1-1`
 
           if [ "$getFirstChar" == "/" ] 
           then :
-              echo "Ngnix URL"
+              echo "Ngnix URL" 
+              OutputDataLocation=${OutputDataLocation}
               echo $OutputDataLocation
           else
               echo "S3 Path"
@@ -558,13 +566,13 @@ Resources:
 
           cat << EOF >> $PROJECT_NF_CONFIG                                                                                                                                                                                                                                                                                                                                                                
           process.container = '${PipelineContainer}'
-          docker.runOptions='-u 0:0'
           plugins {
             id 'nf-amazon'
           }
           docker.enabled = true
-          params.outdir='${OutputDataLocation}'
-          params.reads='s3://${InputDataLocation}'
+          params.outdir='$OutputDataLocation'
+          params.reads='s3://${InputDataLocation}/${InputDataPattern}'
+          params.input='s3://${InputDataLocation}/${InputDataPattern}'
           profiles {
             standard {
               process.container = '${PipelineContainer}'
@@ -589,10 +597,10 @@ Resources:
 
           NEXTFLOW_CONFIG=nextflow.config
 
-          NEXTFLOW_OUTPUT_PARAMS='\nparams.outdir="${OutputDataLocation}"'
-          
-          echo -e $NEXTFLOW_OUTPUT_PARAMS >>$NEXTFLOW_CONFIG
- 
+          cat << EOF >> $NEXTFLOW_CONFIG
+          params.outdir='$OutputDataLocation'
+          EOF
+  
           docker pull ${PipelineContainer}
           chown -R ec2-user:ec2-user /home/ec2-user/
           chown -R ec2-user:ec2-user /home/ec2-user/.[^.]*

config/snsConfig.json

@@ -50,6 +50,6 @@
   "snsTopicName": "Research-Product-Events",
   "snsProtocol": "REPLACE_WITH_HTTP_HTTPS",
   "endpoint": "notificationsink/productProvisioningUpdates",
-  "notificationSinkCleanUpRoute": "notificationsink/unlinkAccount",
+  "notificationSinkCleanUpRoute": "unlinkAccount",
   "snsUrlKey": "snsUrl"
 }

docker-compose.yml

@@ -18,7 +18,7 @@ services:
       - sp2net
 
   cc-3102:
-    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.9.0_b954
+    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.9.1_b982
     secrets:
       - source: sp2prod-config.json
         target: /rlc/cc/server/app/config/config.json
@@ -32,6 +32,7 @@ services:
       - APM_ENABLE=false
       - APM_URL=APM URL where it has been placed
       - CC_HOST=cc-3102:3002
+      - NS_HOST=notificationsink:3005
     volumes:
       - ${PWD}/config:/rlc/cc/server/app/config
       - ${PWD}/logs:/rlc/cc/server/app/logs
@@ -42,7 +43,7 @@ services:
     command: /bin/bash start.sh 
 
   scheduler-3102:
-    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.9.0_b954
+    image: 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.9.1_b982
     secrets:
       - source: sp2prod-config.json
         target: /rlc/cc/server/app/config/config.json

dump/configs.json

@@ -318,3 +318,33 @@
     }
   }
 }
+{
+  "_id": {
+    "$oid": "61fa15c4bd4d148afe2155dc"
+  },
+  "key": "readTheDocsUrl",
+  "value": "https://researchgateway.readthedocs.io/"
+}
+{
+  "_id": {
+    "$oid": "61fbd4239fab988868a9d00e"
+  },
+  "key": "rgSetupVideo",
+  "value": {
+    "fullVideo": {
+      "link": "https://youtu.be/zDksIb1n9zI"
+    },
+    "addAccount": {
+      "link": "https://youtu.be/zDksIb1n9zI?t=60"
+    },
+    "addProject": {
+      "link": "https://youtu.be/zDksIb1n9zI?t=100"
+    },
+    "assignProducts": {
+      "link": "https://youtu.be/zDksIb1n9zI?t=100"
+    },
+    "users": {
+      "link": "https://youtu.be/zDksIb1n9zI?t=100"
+    }
+  }
+}

provisioners/provision-ecr.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -xe
 sudo docker login -u AWS -p $(aws ecr get-login-password --region us-east-2) 045938549113.dkr.ecr.us-east-2.amazonaws.com
-sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.8.5_b896
+sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/researchportal:_fd_1.9.1_b982
 sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/nginx:latest
-sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.8.1_b4
+sudo docker pull 045938549113.dkr.ecr.us-east-2.amazonaws.com/notificationsink:1.9.0_b2