V2.9.0+deploy

.github/actions/combine-deploy-update/action.yml

@@ -1,70 +1,88 @@
 name: "Combine Deploy Updates"
-description: "Deploy updated images for The Combine to the Kubernetes cluster"
+description: "Deploy updated Helm charts for The Combine to the Kubernetes cluster"
 inputs:
+  aws_access_key_id:
+    description: "AWS Access Key ID"
+    required: true
+  aws_account:
+    description: "AWS Account ID"
+    required: true
+  aws_default_region:
+    description: "AWS Default Region"
+    required: true
+  aws_secret_access_key:
+    description: "AWS Secret Access Key"
+    required: true
+  combine_captcha_secret_key:
+    description: "Cloudflare Turnstile secret key for The Combine"
+    required: true
+  combine_jwt_secret_key:
+    description: "JWT secret key for The Combine"
+    required: true
+  combine_smtp_password:
+    description: "SMTP password for sending emails from The Combine"
+    required: true
+  combine_smtp_username:
+    description: "SMTP username for sending emails from The Combine"
+    required: true
+  helm_version:
+    description: "Version of helm to install on the runner"
+    required: true
   image_registry:
     description: "Docker Image Registry"
     required: true
-  image_registry_alias:
-    description: >
-      Custom or default alias for AWS Public ECR registries.  Blank for AWS ECR Private registries.
-      If the image_registry_alias is specified, include the '/' at the beginning of the string,
-      e.g. /thecombine
-    required: false
-    default: ""
   image_tag:
     description: "Image Tag for docker images"
     required: true
   kube_context:
-    description: "Context for the kubectl commands"
+    description: "Context for the kubectl/helm commands"
     required: true
   kubectl_version:
     description: "Version of kubectl to install on the runner"
     required: true
-  update_cert_proxy:
-    description: "Specification for whether the combine-cert-proxy should be updated."
+  target:
+    description: "Deployment target (qa or prod)"
     required: true
+
 runs:
   using: "composite"
   steps:
-    - name: Install pre-requisites
+    - name: Install Helm
+      run: |
+        echo "Installing Helm version ${{ inputs.helm_version }}"
+        curl https://raw.githubusercontent.com/helm/helm/${{ inputs.helm_version }}/scripts/get-helm-3 | DESIRED_VERSION=${{ inputs.helm_version }} bash
+        helm version
+      shell: bash
+
+    - name: Install kubectl
       run: |
         cd
         echo "Installing kubectl version ${{ inputs.kubectl_version }}"
         curl -LO https://dl.k8s.io/release/${{ inputs.kubectl_version }}/bin/linux/amd64/kubectl
         sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
         kubectl version --output=yaml
       shell: bash
-    - name: Deploy updated images
-      run: echo "Update images with version ${{ inputs.image_tag }}"
-      shell: bash
-    - name: Update database
-      run: kubectl --context ${{ inputs.kube_context }}
-        --namespace thecombine
-        set image deployment/database
-        database="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_database:${{ inputs.image_tag }}"
-      shell: bash
-    - name: Update frontend
-      run: kubectl --context ${{ inputs.kube_context }}
-        --namespace thecombine
-        set image deployment/frontend
-        frontend="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_frontend:${{ inputs.image_tag }}"
-      shell: bash
-    - name: Update backend
-      run: kubectl --context ${{ inputs.kube_context }}
-        --namespace thecombine
-        set image deployment/backend
-        backend="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_backend:${{ inputs.image_tag }}"
-      shell: bash
-    - name: Update maintenance
-      run: kubectl --context ${{ inputs.kube_context }}
-        --namespace thecombine
-        set image deployment/maintenance
-        maintenance="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_maint:${{ inputs.image_tag }}"
+
+    - name: Install Python requirements
+      run: python3 -m pip install -r deploy/requirements.txt
       shell: bash
-    - name: Update Cert Proxy Server
-      if: ${{ inputs.update_cert_proxy == 'true' }}
-      run: kubectl --context ${{ inputs.kube_context }}
-        --namespace combine-cert-proxy
-        set image deployment/combine-cert-proxy
-        combine-cert-proxy="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_maint:${{ inputs.image_tag }}"
+
+    - name: Deploy with Helm
+      run: >
+        python3
+        deploy/scripts/setup_combine.py
+        --context ${{ inputs.kube_context }}
+        --non-interactive
+        --repo ${{ inputs.image_registry }}
+        --tag ${{ inputs.image_tag }}
+        --target ${{ inputs.target }}
       shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        AWS_ACCOUNT: ${{ inputs.aws_account }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws_default_region }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+        COMBINE_CAPTCHA_SECRET_KEY: ${{ inputs.combine_captcha_secret_key }}
+        COMBINE_JWT_SECRET_KEY: ${{ inputs.combine_jwt_secret_key }}
+        COMBINE_SMTP_PASSWORD: ${{ inputs.combine_smtp_password }}
+        COMBINE_SMTP_USERNAME: ${{ inputs.combine_smtp_username }}

.github/workflows/combine_deploy_image.yml

@@ -31,6 +31,7 @@ jobs:
             get.helm.sh:443
             github.com:443
             production.cloudflare.docker.com:443
+            production.cloudfront.docker.com:443
             public.ecr.aws:443
             pypi.org:443
             raw.githubusercontent.com:443

.github/workflows/deploy_qa.yml

@@ -1,6 +1,12 @@
 name: "Deploy Update to QA Server"
 
 on:
+  workflow_dispatch:
+    inputs:
+      confirm_qa_deploy:
+        description: "Type QA to confirm QA deployment"
+        required: true
+        default: ""
   push:
     branches: [master]
 
@@ -45,6 +51,7 @@ jobs:
             github.com:443
             mcr.microsoft.com:443
             production.cloudflare.docker.com:443
+            production.cloudfront.docker.com:443
             public.ecr.aws:443
             pypi.org:443
             registry-1.docker.io:443
@@ -53,7 +60,8 @@ jobs:
             security.ubuntu.com:80
             storage.googleapis.com:443
             sts.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com:443
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 0
       - name: Build The Combine
@@ -65,6 +73,7 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
           build_component: ${{ matrix.component }}
+
   clean_ecr_repo:
     needs: build
     runs-on: ubuntu-latest
@@ -81,7 +90,8 @@ jobs:
             api.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com:443
             github.com:443
             sts.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com:443
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
         with:
@@ -101,23 +111,32 @@ jobs:
               else REM="v$((VA - 1))\..*"; fi
               scripts/clean_aws_repo.py combine_frontend combine_backend combine_maint combine_database --remove "${REM}" --verbose
           fi
+
   deploy_update:
     needs: build
-    # Only push to the QA server when built on the master branch
-    if: github.ref_name == 'master'
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.confirm_qa_deploy == 'QA')
     runs-on: [self-hosted, thecombine]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
         with:
           egress-policy: audit
-
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
       - name: Deploy The Combine Update
         uses: ./.github/actions/combine-deploy-update
         with:
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_account: ${{ secrets.AWS_ACCOUNT }}
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          combine_captcha_secret_key: ${{ secrets.COMBINE_CAPTCHA_SECRET_KEY }}
+          combine_jwt_secret_key: ${{ secrets.COMBINE_JWT_SECRET_KEY }}
+          combine_smtp_password: ${{ secrets.COMBINE_SMTP_PASSWORD }}
+          combine_smtp_username: ${{ secrets.COMBINE_SMTP_USERNAME }}
+          helm_version: ${{ vars.HELM_VERSION }}
           image_registry: ${{ secrets.AWS_ACCOUNT }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
           image_tag: ${{ needs.build.outputs.image_tag }}
           kube_context: ${{ secrets.LTOPS_K8S_STAGING_CONTEXT }}
           kubectl_version: ${{ vars.KUBECTL_VERSION }}
-          update_cert_proxy: false
+          target: qa

.github/workflows/deploy_release.yml

@@ -38,6 +38,7 @@ jobs:
             github.com:443
             mcr.microsoft.com:443
             production.cloudflare.docker.com:443
+            production.cloudfront.docker.com:443
             public.ecr.aws:443
             pypi.org:443
             registry-1.docker.io:443
@@ -46,7 +47,8 @@ jobs:
             security.ubuntu.com:80
             storage.googleapis.com:443
             sts.us-east-1.amazonaws.com:443
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
       - name: Build The Combine
         id: build_combine
         uses: ./.github/actions/combine-build
@@ -60,6 +62,7 @@ jobs:
           # https://docs.aws.amazon.com/AmazonECR/latest/public/public-registry-auth.html#public-registry-auth-token
           aws_default_region: us-east-1
           build_component: ${{ matrix.component }}
+
   deploy_update:
     needs: build
     runs-on: [self-hosted, thecombine]
@@ -68,25 +71,41 @@ jobs:
         uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
         with:
           egress-policy: audit
-
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 0
       - name: Deploy The Combine Update to QA
         uses: ./.github/actions/combine-deploy-update
         with:
-          image_registry: public.ecr.aws
-          image_registry_alias: "/thecombine"
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_account: ${{ secrets.AWS_ACCOUNT }}
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          combine_captcha_secret_key: ${{ secrets.COMBINE_CAPTCHA_SECRET_KEY }}
+          combine_jwt_secret_key: ${{ secrets.COMBINE_JWT_SECRET_KEY }}
+          combine_smtp_password: ${{ secrets.COMBINE_SMTP_PASSWORD }}
+          combine_smtp_username: ${{ secrets.COMBINE_SMTP_USERNAME }}
+          helm_version: ${{ vars.HELM_VERSION }}
+          image_registry: public.ecr.aws/thecombine
           image_tag: ${{ needs.build.outputs.image_tag }}
           kube_context: ${{ secrets.LTOPS_K8S_STAGING_CONTEXT }}
           kubectl_version: ${{ vars.KUBECTL_VERSION }}
-          update_cert_proxy: false
+          target: qa
       - name: Deploy The Combine Update to Production
         uses: ./.github/actions/combine-deploy-update
         with:
-          image_registry: public.ecr.aws
-          image_registry_alias: "/thecombine"
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_account: ${{ secrets.AWS_ACCOUNT }}
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          combine_captcha_secret_key: ${{ secrets.COMBINE_CAPTCHA_SECRET_KEY }}
+          combine_jwt_secret_key: ${{ secrets.COMBINE_JWT_SECRET_KEY }}
+          combine_smtp_password: ${{ secrets.COMBINE_SMTP_PASSWORD }}
+          combine_smtp_username: ${{ secrets.COMBINE_SMTP_USERNAME }}
+          helm_version: ${{ vars.HELM_VERSION }}
+          image_registry: public.ecr.aws/thecombine
           image_tag: ${{ needs.build.outputs.image_tag }}
           kube_context: ${{ secrets.LTOPS_K8S_PRODUCTION_CONTEXT }}
           kubectl_version: ${{ vars.KUBECTL_VERSION }}
-          update_cert_proxy: true
+          target: prod

.github/workflows/installer_release.yml

@@ -2,6 +2,11 @@ name: installer_release
 
 on:
   workflow_dispatch:
+    inputs:
+      confirm_upload:
+        description: "Type UP to confirm installer upload"
+        required: true
+        default: ""
   workflow_run:
     workflows: ["Deploy Update to Live Server"]
     types:
@@ -13,7 +18,7 @@ permissions:
 jobs:
   make_installer:
     runs-on: ubuntu-latest
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success'
     outputs:
       release_tag: ${{ steps.release.outputs.tag }}
     steps:
@@ -74,6 +79,7 @@ jobs:
 
   upload_installer:
     needs: make_installer
+    if: github.event_name == 'workflow_run' || (github.event_name == 'workflow_dispatch' && github.event.inputs.confirm_upload == 'UP')
     runs-on: ubuntu-latest
     steps:
       # See https://docs.stepsecurity.io/harden-runner/getting-started/ for instructions on

.github/workflows/maintenance.yml

@@ -28,13 +28,10 @@ jobs:
           allowed-endpoints: >
             *.cloudfront.net:443
             archive.ubuntu.com:80
-            auth.docker.io:443
             files.pythonhosted.org:443
             github.com:443
-            production.cloudflare.docker.com:443
             public.ecr.aws:443
             pypi.org:443
-            registry-1.docker.io:443
             security.ubuntu.com:80
       # For subfolders, currently a full checkout is required.
       # See: https://github.com/marketplace/actions/build-and-push-docker-images#path-context

README.md

@@ -746,6 +746,8 @@ Notes:
 
 - Run the script with the `--help` option to see possible options for the script.
 
+- Run the script with `--non-interactive` in CI/CD to avoid user prompts.
+
 - The setup assumes `amd64` architecture. If the target architecture is `arm64`, add `--set global.cpuArch=arm64`.
 
 When the script completes, the resources will be installed on the specified cluster. It may take a few moments before

deploy/ansible/vars/k3s_versions.yml

@@ -1,6 +1,6 @@
 ---
 k3s_version: "v1.30.1%2Bk3s1"
-# kubectl_version should match GitHub repo variable KUBECTL_VERSION found at:
+# should match GitHub repo variables KUBECTL_VERSION and HELM_VERSION found at:
 # Settings > Secrets and variables > Actions > Variables > Repository variables
 kubectl_version: "v1.30.2"
 helm_version: "v3.15.2"

deploy/helm/aws-login/values.yaml

@@ -23,7 +23,7 @@ awsEcr:
   cronJobName: ecr-cred-helper-cron
   dockerEmail: noreply@thecombine.app
   image: "public.ecr.aws/thecombine/aws-kubectl"
-  imageVersion: "0.4.0"
+  imageVersion: "0.4.1"
   jobName: ecr-cred-helper
   schedule: "0 */8 * * *"
   secretsName: aws-ecr-credentials

deploy/helm/create-admin-user/templates/job-create-admin-user.yaml

@@ -6,7 +6,7 @@ metadata:
   annotations:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
-    "helm.sh/hook": post-install, post-upgrade
+    "helm.sh/hook": post-install
     "helm.sh/hook-delete-policy": before-hook-creation
 spec:
   # keep completed jobs for 24 hrs so that logs are

deploy/helm/thecombine/charts/maintenance/values.yaml

@@ -37,7 +37,7 @@ serviceAccount:
 
 awsEcr:
   image: "public.ecr.aws/thecombine/aws-kubectl"
-  imageVersion: "0.4.0"
+  imageVersion: "0.4.1"
 
 #######################################
 # Variables controlling backups

deploy/helm/thecombine/templates/letsencrypt-prod.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.certManager.enabled }}
+{{- if and .Values.certManager.enabled (eq .Values.certManager.certIssuer "letsencrypt-prod") }}
 apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata: