Skip to content

Commit 3d69a91

Browse files
ravigurram8ravigurram8
authored
Merge pull request #109 from ravigurram8/topic-release-8
PCluster Added
2 parents ae51330 + d715cca commit 3d69a91

7 files changed

Lines changed: 299 additions & 0 deletions

File tree

‎products/PCluster/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Pipelines for RLCatalyst Research Gateway based on Nextflow
2+
3+
Use the below command to start the packer script after providing necessary details in [configuration.json](machine-images/config/infra/configuration.json)
4+
5+
> cd products/Nextflow-Advanced/machine-images/config/infra
6+
7+
> sudo packer build -var-file=configuration.json packer-ec2-nextflow-workspace.json

‎products/PCluster/machine-images/config/infra/configuration.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"aws_access_key": "",
3+
"aws_secret_key": "",
4+
"awsRegion": "",
5+
"vpcId": "",
6+
"subnetId": "",
7+
"amiName": "",
8+
"awsProfile": ""
9+
}

‎products/PCluster/machine-images/config/infra/files/pcluster/Provision-pcluster.sh‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/bin/bash
2+
launcherstackname=${12}
3+
Region=$2
4+
trap '/opt/aws/bin/cfn-signal --exit-code 1 --resource EC2Instance --region ${Region} --stack ${launcherstackname}' ERR
5+
function wait_stack_create() {
6+
CLUSTER_STACK_NAME=$1
7+
LAUNCHER_STACK_NAME=$2
8+
REGION=$3
9+
echo "Waiting for [$CLUSTER_STACK_NAME] stack creation."
10+
aws cloudformation wait stack-create-complete --region ${REGION} --stack-name ${CLUSTER_STACK_NAME}
11+
status=$?
12+
if [[ ${status} -ne 0 ]] ; then
13+
# Waiter encountered a failure state.
14+
echo "Stack [${CLUSTER_STACK_NAME}] creation failed. AWS error code is ${status}."
15+
/opt/aws/bin/cfn-signal --exit-code 1 --resource EC2Instance --region ${REGION} --stack ${LAUNCHER_STACK_NAME}
16+
exit 1
17+
fi
18+
}
19+
source ~/apc-ve/bin/activate
20+
echo "Activated Virtual Environment"
21+
echo "Retrieving Tags from Running Instance"
22+
INSTANCE_ID=`wget -qO- http://instance-data/latest/meta-data/instance-id`
23+
IFS='-' read -ra TRIMMED <<< "$INSTANCE_ID"
24+
CLUSTER_NAME=RG-Pcluster-${TRIMMED[1]}
25+
REGION=`wget -qO- http://instance-data/latest/meta-data/placement/availability-zone | sed 's/.$//'`
26+
aws ec2 describe-tags --region $REGION --filter "Name=resource-id,Values=$INSTANCE_ID" --query 'Tags[*].{Key:Key,Value:Value}' | jq -r '.[] | select( .Key as $a | ["cost_resource", "project_name","researcher_name"] | index($a) )' >> out.json
27+
jq -s '.' out.json >> valid.json
28+
echo "valid.json file is updated with Tags"
29+
yq eval -P valid.json > valid.yaml
30+
echo "Json file is converted to yaml"
31+
sed -i '1 i\Tags:' valid.yaml
32+
scheduler=$1
33+
Region=$2
34+
headnodeinstancetype=$3
35+
headnodesubnetId=$4
36+
keyname=$5
37+
computenodeinstancetype=$6
38+
minvpc=$7
39+
maxvpc=$8
40+
computenodesubnetId=$9
41+
desiredvpc=${10}
42+
spotbid=${11}
43+
CustomAMI=${13}
44+
FileSystemId=${14}
45+
QueueCapacityType=${15}
46+
disableSimultaneousMultithreading=${16}
47+
efa=${17}
48+
placementGroup=${18}
49+
50+
IFS='-' read -ra TRIMMED <<< "$CustomAMI"
51+
CustomAMIStartsWith=${TRIMMED[0]}
52+
if [ "$scheduler" == "slurm" ]; then
53+
echo "slurm.yaml exists"
54+
if [ "$CustomAMIStartsWith" == "ami" ]; then
55+
yq -i ".Image.CustomAmi=\"$CustomAMI\"" slurm.yaml
56+
fi
57+
if [ "$FileSystemId" != "default" ]; then
58+
yq -i '.SharedStorage=[{"MountDir": "/fsx", "Name":"RG_Filesysytem", "StorageType": "FsxLustre", "FsxLustreSettings":{"FileSystemId":"'$FileSystemId'"}}]' slurm.yaml
59+
fi
60+
yq -i ".Region=\"$Region\"" slurm.yaml
61+
yq -i ".HeadNode.InstanceType=\"$headnodeinstancetype\"" slurm.yaml
62+
yq -i ".HeadNode.Networking.SubnetId=\"$headnodesubnetId\"" slurm.yaml
63+
yq -i ".HeadNode.Ssh.KeyName=\"$keyname\"" slurm.yaml
64+
yq -i ".Scheduling.SlurmQueues[0].CapacityType=\"$QueueCapacityType\"" slurm.yaml
65+
yq -i ".Scheduling.SlurmQueues[0].ComputeResources[0].InstanceType=\"$computenodeinstancetype\"" slurm.yaml
66+
yq -i ".Scheduling.SlurmQueues[0].ComputeResources[0].MinCount=\"$minvpc\"" slurm.yaml
67+
yq -i ".Scheduling.SlurmQueues[0].ComputeResources[0].MaxCount=\"$maxvpc\"" slurm.yaml
68+
yq -i ".Scheduling.SlurmQueues[0].ComputeResources[0].DisableSimultaneousMultithreading=\"$disableSimultaneousMultithreading\"" slurm.yaml
69+
yq -i ".Scheduling.SlurmQueues[0].ComputeResources[0].Efa.Enabled=\"$efa\"" slurm.yaml
70+
yq -i ".Scheduling.SlurmQueues[0].Networking.PlacementGroup.Enabled=\"$placementGroup\"" slurm.yaml
71+
yq -i ".Scheduling.SlurmQueues[0].Networking.SubnetIds[0]=\"$computenodesubnetId\"" slurm.yaml
72+
sed -i 's/\"//g' slurm.yaml
73+
yq eval-all "select(fileIndex == 1) *+ select(fileIndex == 0)" valid.yaml slurm.yaml >> cluster-config-slurm.yaml
74+
echo "valid.yaml file and cluster-config.yaml file is merged into cluster-config-slurm.yaml"
75+
echo "Modified cluster-config-slurm.yaml with Tags"
76+
pcluster create-cluster --cluster-name $CLUSTER_NAME --cluster-configuration cluster-config-slurm.yaml
77+
else
78+
echo "batch.yaml exists"
79+
if [ "$CustomAMIStartsWith" == "ami" ]; then
80+
yq -i ".Image.CustomAmi=\"$CustomAMI\"" batch.yaml
81+
fi
82+
yq -i ".Region=\"$Region\"" batch.yaml
83+
yq -i ".HeadNode.InstanceType=\"$headnodeinstancetype\"" batch.yaml
84+
yq -i ".HeadNode.Networking.SubnetId=\"$headnodesubnetId\"" batch.yaml
85+
yq -i ".HeadNode.Ssh.KeyName=\"$keyname\"" batch.yaml
86+
yq -i ".Scheduling.AwsBatchQueues[0].CapacityType=\"$QueueCapacityType\"" batch.yaml
87+
yq -i ".Scheduling.AwsBatchQueues[0].ComputeResources[0].InstanceTypes[0]=\"$computenodeinstancetype\"" batch.yaml
88+
yq -i ".Scheduling.AwsBatchQueues[0].ComputeResources[0].MinvCpus=\"$minvpc\"" batch.yaml
89+
yq -i ".Scheduling.AwsBatchQueues[0].ComputeResources[0].MaxvCpus=\"$maxvpc\"" batch.yaml
90+
yq -i ".Scheduling.AwsBatchQueues[0].ComputeResources[0].DesiredvCpus=\"$desiredvpc\"" batch.yaml
91+
yq -i ".Scheduling.AwsBatchQueues[0].ComputeResources[0].SpotBidPercentage=\"$spotbid\"" batch.yaml
92+
yq -i ".Scheduling.AwsBatchQueues[0].Networking.SubnetIds[0]=\"$computenodesubnetId\"" batch.yaml
93+
sed -i 's/\"//g' batch.yaml
94+
yq eval-all "select(fileIndex == 1) *+ select(fileIndex == 0)" valid.yaml batch.yaml >> cluster-config-batch.yaml
95+
echo "valid.yaml file and cluster-config.yaml file is merged into cluster-config-batch.yaml"
96+
echo "Modified cluster-config-batch.yaml with Tags"
97+
pcluster create-cluster --cluster-name $CLUSTER_NAME --cluster-configuration cluster-config-batch.yaml
98+
99+
fi
100+
101+
wait_stack_create $CLUSTER_NAME $launcherstackname $REGION
102+
HEAD_INSTANCE_ID=`pcluster describe-cluster -n $CLUSTER_NAME -r $REGION --query headNode | jq -r '.instanceId'`
103+
# PRIVATE_IP_ADDRESS=`pcluster describe-cluster -n $1 --query headNode.privateIpAddress`
104+
PARAMETER_NAME="/rg/pcluster/headnode-instance-id/${CLUSTER_NAME}"
105+
aws ssm put-parameter --name "${PARAMETER_NAME}" --type "String" --value "${HEAD_INSTANCE_ID}" --region $REGION --overwrite
106+
echo "Instance id of the head node is stored on ${PARAMETER_NAME}"
107+
echo "Instance id is : ${HEAD_INSTANCE_ID}"

‎products/PCluster/machine-images/config/infra/files/pcluster/batch-main.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
Region:
2+
Image:
3+
Os: alinux2
4+
HeadNode:
5+
InstanceType:
6+
Networking:
7+
SubnetId:
8+
Ssh:
9+
KeyName:
10+
Dcv:
11+
Enabled: true
12+
Iam:
13+
S3Access:
14+
- EnableWriteAccess: true
15+
BucketName: '*'
16+
AdditionalIamPolicies:
17+
- Policy: arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
18+
CustomActions:
19+
OnNodeStart:
20+
Script: s3://tempbucket/bootstrap-scripts/pcluster-on-node-start.sh
21+
Imds:
22+
Secured: false
23+
Scheduling:
24+
Scheduler: awsbatch
25+
AwsBatchQueues:
26+
- Name: queue1
27+
CapacityType: SPOT
28+
ComputeResources:
29+
- Name: optimal
30+
InstanceTypes:
31+
-
32+
MinvCpus:
33+
DesiredvCpus:
34+
MaxvCpus:
35+
SpotBidPercentage:
36+
Networking:
37+
SubnetIds:
38+
-

‎products/PCluster/machine-images/config/infra/files/pcluster/slurm-main.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
Region:
2+
Image:
3+
Os: alinux2
4+
HeadNode:
5+
InstanceType:
6+
Networking:
7+
SubnetId:
8+
Ssh:
9+
KeyName:
10+
Dcv:
11+
Enabled: true
12+
Iam:
13+
S3Access:
14+
- EnableWriteAccess: true
15+
BucketName: '*'
16+
AdditionalIamPolicies:
17+
- Policy: arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
18+
CustomActions:
19+
OnNodeStart:
20+
Script: s3://tempbucket/bootstrap-scripts/pcluster-on-node-start.sh
21+
Scheduling:
22+
Scheduler: slurm
23+
SlurmQueues:
24+
- Name: queue1
25+
CapacityType: SPOT
26+
ComputeResources:
27+
- Name: t2micro
28+
InstanceType:
29+
MinCount:
30+
MaxCount:
31+
DisableSimultaneousMultithreading:
32+
Efa:
33+
Enabled:
34+
Networking:
35+
PlacementGroup:
36+
Enabled:
37+
SubnetIds:
38+
-
39+
Iam:
40+
S3Access:
41+
- EnableWriteAccess: true
42+
BucketName: '*'

‎products/PCluster/machine-images/config/infra/packer-ec2-pcluster.json‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
{
2+
"variables": {
3+
"aws_access_key": "",
4+
"aws_secret_key": "",
5+
"vpcId": "",
6+
"subnetId": "",
7+
"awsProfile": "",
8+
"awsRegion": "",
9+
"amiName": ""
10+
},
11+
"builders": [
12+
{
13+
"type": "amazon-ebs",
14+
"profile": "{{user `awsProfile`}}",
15+
"access_key": "{{user `aws_access_key`}}",
16+
"secret_key": "{{user `aws_secret_key`}}",
17+
"region": "{{user `awsRegion`}}",
18+
"vpc_id": "{{user `vpcId`}}",
19+
"subnet_id": "{{user `subnetId`}}",
20+
"source_ami_filter": {
21+
"filters": {
22+
"virtualization-type": "hvm",
23+
"name": "amzn2-ami-hvm-*-x86_64-gp2",
24+
25+
"root-device-type": "ebs"
26+
},
27+
"owners": [
28+
"amazon"
29+
],
30+
"most_recent": true
31+
},
32+
"instance_type": "t2.micro",
33+
"ssh_username": "ec2-user",
34+
"ami_name": "{{user `amiName`}}",
35+
"associate_public_ip_address": "true",
36+
"tags": {
37+
"Schedule": "Bangalore-office-hours"
38+
}
39+
}
40+
],
41+
"provisioners": [
42+
{
43+
"type": "file",
44+
"source": "files/pcluster/batch.yaml",
45+
"destination": "/home/ec2-user/"
46+
},
47+
{
48+
"type": "file",
49+
"source": "files/pcluster/slurm.yaml",
50+
"destination": "/home/ec2-user/"
51+
},
52+
{
53+
"type": "file",
54+
"source": "files/pcluster/Provision-pcluster.sh",
55+
"destination": "/home/ec2-user/"
56+
},
57+
{
58+
"type": "shell",
59+
"script": "provisioners/launch.sh"
60+
}
61+
]
62+
}

‎products/PCluster/machine-images/config/infra/provisioners/provision-launch.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
cd /home/ec2-user
3+
p=$(pwd)
4+
echo "You are currently in: $p"
5+
sudo yum update -y
6+
sudo yum install -y jq git gcc-c++ make curl awscli unzip
7+
# Install yq
8+
sudo wget -qO /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
9+
sudo chmod a+x /usr/bin/yq
10+
# Install Virtual Env
11+
python3 -m pip install --upgrade pip
12+
python3 -m pip install --user --upgrade virtualenv
13+
python3 -m virtualenv /home/ec2-user/apc-ve
14+
source /home/ec2-user/apc-ve/bin/activate
15+
# Install Node
16+
curl -sL https://rpm.nodesource.com/setup_16.x | sudo -E bash -
17+
sudo yum install -y nodejs
18+
# Install PCluster
19+
python3 -m pip install --upgrade "aws-parallelcluster"
20+
# Install AWS CLI
21+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
22+
unzip awscliv2.zip
23+
sudo ./aws/install
24+
p=$(pwd)
25+
echo "You are currently in: $p"
26+
cd /home/ec2-user
27+
mkdir -p /home/ec2-user/parallel-update
28+
cp /tmp/cluster/* /home/ec2-user/parallel-update/
29+
sudo chown -R ec2-user:ec2-user /home/ec2-user/*
30+
chmod 775 parallel-update/slurm.yaml
31+
chmod 775 parallel-update/batch.yaml
32+
chmod 775 parallel-update/Provision-pcluster.sh
33+
rm -rf /tmp/PCluster
34+
rm -rf /tmp/cluster

0 commit comments

Comments
 (0)