Concourse pipeline

README.md

@@ -325,6 +325,48 @@ terraform destroy -var-file=env/dev/dev.tfvars
 
 **Please Note:** Make sure to use the correct `.tfbackend` and `.tfvars` files for your environment.
 
+### Deployments with Concourse
+
+#### Allowlisting your IP
+
+To setup the deployment pipeline with concourse, you must first allowlist your IP address on the Concourse
+server. IP addresses are flushed everyday at 00:00 so this must be done at the beginning of every working day
+whenever the deployment pipeline needs to be used. Follow the instructions on the Confluence page (SDP Homepage > SDP Concourse > Concourse Login) to
+login. All our pipelines run on sdp-pipeline-prod, whereas sdp-pipeline-dev is the account used for
+changes to Concourse instance itself. Make sure to export all necessary environment variables from sdp-pipeline-prod (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN).
+
+#### Setting up a pipeline
+
+When setting up our pipelines, we use ecs-infra-user on sdp-dev to be able to interact with our infrastructure on AWS. The credentials for this are stored on
+AWS Secrets Manager so you do not need to set up anything yourself.
+
+To set the pipeline, run the following script:
+
+```bash
+chmod u+x ./concourse/scripts/set_pipeline.sh
+./concourse/scripts/set_pipeline.sh github-repo-archive-script
+```
+
+Note that you only have to run chmod the first time running the script in order to give permissions.
+This script will set the branch and pipeline name to whatever branch you are currently on. It will also set the image tag on ECR to the current commit hash at the time of setting the pipeline.
+
+The pipeline name itself will usually follow a pattern as follows: `<repo-name>-<branch-name>`
+If you wish to set a pipeline for another branch without checking out, you can run the following:
+
+```bash
+./concourse/scripts/set_pipeline.sh github-repo-archive-script <branch_name>
+```
+
+If the branch you are deploying is "main" or "master", it will trigger a deployment to the sdp-prod environment. To set the ECR image tag, you must draft a Github release pointing to the latest release of the main/master branch that has a tag in the form of vX.Y.Z. Drafting up a release will automatically deploy the latest version of the main/master branch with the associated release tag, but you can also manually trigger a build through the Concourse UI or the terminal prompt.
+
+#### Triggering a pipeline
+
+Once the pipeline has been set, you can manually trigger a build on the Concourse UI, or run the following command:
+
+```bash
+fly -t aws-sdp trigger-job -j github-repo-archive-script-<branch-name>/build-and-push
+```
+
 ## Linting and Testing
 
 ### GitHub Actions

concourse/ci.yml

@@ -0,0 +1,79 @@
+---
+resources:
+  - name: resource-repo
+    type: git
+    icon: github
+    source:
+      uri: https://github.com/ONS-Innovation/github-repository-archive-script
+      username: ((github_access_token))
+      password: x-oauth-basic
+      branch: ((branch))
+
+jobs:
+  - name: build-and-push
+    public: true
+    plan:
+      - get: resource-repo
+        timeout: 5m
+      - task: build-image
+        privileged: true
+        config:
+          platform: linux
+          image_resource:
+            type: docker-image
+            source:
+              repository: hashicorp/terraform
+          inputs:
+            - name: resource-repo
+          params:
+            aws_account_id: ((aws_account_sdp_((env))))
+            aws_role_arn: arn:aws:iam::((aws_account_sdp_((env)))):role/sdp-concourse-((env))
+            secrets: ((sdp_((env))_github_repo_archive_secrets))
+          run: # binary used to build the image
+            path: sh
+            args:
+              - -cx
+              - |
+                apk add --no-cache aws-cli podman jq iptables curl
+
+                if [[ "((env))" == "prod" ]]; then
+                  tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-repository-archive-script/releases" | jq -r '.[0].tag_name')
+                  export tag
+                else
+                  export tag=((tag))
+                fi
+                git rev-parse --abbrev-ref HEAD
+                chmod u+x ./resource-repo/concourse/scripts/assume_role.sh
+                chmod u+x ./resource-repo/concourse/scripts/build_image.sh
+                source ./resource-repo/concourse/scripts/assume_role.sh
+                ./resource-repo/concourse/scripts/build_image.sh
+        timeout: 10m
+      - task: terraform
+        privileged: true
+        config:
+          platform: linux
+          image_resource:
+            type: docker-image
+            source: { repository: hashicorp/terraform }
+          inputs:
+            - name: resource-repo
+          params:
+            secrets: ((sdp_((env))_github_repo_archive_secrets))
+            github_access_token: ((github_access_token))
+          run:
+            path: sh
+            args:
+              - -cx
+              - |
+                apk add --no-cache jq curl
+
+                if [[ "((env))" == "prod" ]]; then
+                  tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-repository-archive-script/releases" | jq -r '.[0].tag_name')
+                  export tag
+                else
+                  export tag=((tag))
+                fi
+                chmod u+x ./resource-repo/concourse/scripts/terraform_infra.sh
+                export env=((env))
+                ./resource-repo/concourse/scripts/terraform_infra.sh
+        timeout: 30m

concourse/scripts/assume_role.sh

@@ -0,0 +1,19 @@
+# shellcheck disable=SC3040,SC2154,SC2148
+
+set -euo pipefail
+
+# shellcheck disable=SC3040,SC2154
+aws sts assume-role --output text \
+    --role-arn "${aws_role_arn}" \
+    --role-session-name concourse-pipeline-run \
+    --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
+    | awk -F '\t' '{print $1 > ("AccessKeyId")}{print $2 > ("SecretAccessKey")}{print $3 > ("SessionToken")}'
+
+# shellcheck disable=SC3040,SC2154
+AWS_ACCESS_KEY_ID="$(cat AccessKeyId)"
+AWS_SECRET_ACCESS_KEY="$(cat SecretAccessKey)"
+AWS_SESSION_TOKEN="$(cat SessionToken)"
+
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+export AWS_SESSION_TOKEN

concourse/scripts/build_image.sh

@@ -0,0 +1,20 @@
+# shellcheck disable=SC3040,SC2154,SC2148
+set -euo pipefail
+
+export STORAGE_DRIVER=vfs
+export PODMAN_SYSTEMD_UNIT=concourse-task
+
+# shellcheck disable=SC2154
+container_image=$(echo "$secrets" | jq -r .container_image)
+
+# shellcheck disable=SC2154
+aws ecr get-login-password --region eu-west-2 | podman --storage-driver=vfs login --username AWS --password-stdin "${aws_account_id}".dkr.ecr.eu-west-2.amazonaws.com
+
+# shellcheck disable=SC2154
+podman build -t "${container_image}":"${tag}" resource-repo
+
+# shellcheck disable=SC2154
+podman tag "${container_image}":"${tag}" "${aws_account_id}".dkr.ecr.eu-west-2.amazonaws.com/"${container_image}":"${tag}"
+
+# shellcheck disable=SC2154
+podman push "${aws_account_id}".dkr.ecr.eu-west-2.amazonaws.com/"${container_image}":"${tag}"

concourse/scripts/set_pipeline.sh

@@ -0,0 +1,29 @@
+# shellcheck disable=SC3040,SC2154,SC2148
+
+repo_name=${1}
+
+if [[ $# -gt 1 ]]; then
+    branch=${2}
+    git rev-parse --verify "${branch}"
+    # shellcheck disable=SC2181
+    if [[ $? -ne 0 ]]; then
+        echo "Branch \"${branch}\" does not exist"
+        exit 1
+    fi
+else
+    branch=$(git rev-parse --abbrev-ref HEAD)
+fi
+
+if [[ ${branch} == "main" || ${branch} == "master" ]]; then
+    env="prod"
+else
+    env="dev"
+fi
+
+if [[ ${env} == "dev" ]]; then
+    tag=$(git rev-parse HEAD)
+else
+    tag=$(git tag | tail -n 1)
+fi
+
+fly -t aws-sdp set-pipeline -c concourse/ci.yml -p "${repo_name}"-"${branch}" -v branch="${branch}" -v tag="${tag}" -v env="${env}"

concourse/scripts/terraform_infra.sh

@@ -0,0 +1,57 @@
+# shellcheck disable=SC3040,SC2154,SC2148
+
+set -euo pipefail
+
+# shellcheck disable=SC2154
+aws_account_id=$(echo "$secrets" | jq -r .aws_account_id)
+aws_access_key_id=$(echo "$secrets" | jq -r .aws_access_key_id)
+
+aws_secret_access_key=$(echo "$secrets" | jq -r .aws_secret_access_key)
+aws_secret_name=$(echo "$secrets" | jq -r .aws_secret_name)
+
+env_name=$(echo "$secrets" | jq -r .env_name)
+lambda_name=$(echo "$secrets" | jq -r .lambda_name)
+
+github_app_client_id=$(echo "$secrets" | jq -r .github_app_client_id)
+lambda_arch=$(echo "$secrets" | jq -r .lambda_arch)
+
+github_org=$(echo "$secrets" | jq -r .github_org)
+
+schedule=$(echo "$secrets" | jq -r .schedule)
+lambda_timeout=$(echo "$secrets" | jq -r .lambda_timeout)
+
+lambda_memory=$(echo "$secrets" | jq -r .lambda_memory)
+
+AWS_ACCESS_KEY_ID=$aws_access_key_id
+AWS_SECRET_ACCESS_KEY=$aws_secret_access_key
+
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+
+# shellcheck disable=SC2086
+git config --global url."https://x-access-token:$github_access_token@github.com/".insteadOf "https://github.com/"
+
+if [[ ${env} != "prod" ]]; then
+    env="dev"
+fi
+
+cd resource-repo/terraform/service
+
+terraform init -backend-config=env/"${env}"/backend-"${env}".tfbackend -reconfigure
+
+# shellcheck disable=SC2154
+terraform apply \
+-var "aws_account_id=$aws_account_id" \
+-var "aws_access_key_id=$aws_access_key_id" \
+-var "aws_secret_access_key=$aws_secret_access_key" \
+-var "aws_secret_name=$aws_secret_name" \
+-var "env_name=$env_name" \
+-var "lambda_version=${tag}" \
+-var "lambda_name=$lambda_name" \
+-var "lambda_arch=$lambda_arch" \
+-var "lambda_timeout=$lambda_timeout" \
+-var "github_app_client_id=$github_app_client_id" \
+-var "github_org=$github_org" \
+-var "lambda_memory=$lambda_memory" \
+-var "schedule=$schedule" \
+-auto-approve

terraform/service/providers.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 5.0"
+      version = "~> 6.2.0"
     }
   }
 }