diff --git a/.github/actions/combine-deploy-update/action.yml b/.github/actions/combine-deploy-update/action.yml index df0c0fe2b8..0ca4f935c5 100644 --- a/.github/actions/combine-deploy-update/action.yml +++ b/.github/actions/combine-deploy-update/action.yml @@ -1,32 +1,60 @@ name: "Combine Deploy Updates" -description: "Deploy updated images for The Combine to the Kubernetes cluster" +description: "Deploy updated Helm charts for The Combine to the Kubernetes cluster" inputs: + aws_access_key_id: + description: "AWS Access Key ID" + required: true + aws_account: + description: "AWS Account ID" + required: true + aws_default_region: + description: "AWS Default Region" + required: true + aws_secret_access_key: + description: "AWS Secret Access Key" + required: true + combine_captcha_secret_key: + description: "Cloudflare Turnstile secret key for The Combine" + required: true + combine_jwt_secret_key: + description: "JWT secret key for The Combine" + required: true + combine_smtp_password: + description: "SMTP password for sending emails from The Combine" + required: true + combine_smtp_username: + description: "SMTP username for sending emails from The Combine" + required: true + helm_version: + description: "Version of helm to install on the runner" + required: true image_registry: description: "Docker Image Registry" required: true - image_registry_alias: - description: > - Custom or default alias for AWS Public ECR registries. Blank for AWS ECR Private registries. - If the image_registry_alias is specified, include the '/' at the beginning of the string, - e.g. /thecombine - required: false - default: "" image_tag: description: "Image Tag for docker images" required: true kube_context: - description: "Context for the kubectl commands" + description: "Context for the kubectl/helm commands" required: true kubectl_version: description: "Version of kubectl to install on the runner" required: true - update_cert_proxy: - description: "Specification for whether the combine-cert-proxy should be updated." + target: + description: "Deployment target (qa or prod)" required: true + runs: using: "composite" steps: - - name: Install pre-requisites + - name: Install Helm + run: | + echo "Installing Helm version ${{ inputs.helm_version }}" + curl https://raw.githubusercontent.com/helm/helm/${{ inputs.helm_version }}/scripts/get-helm-3 | DESIRED_VERSION=${{ inputs.helm_version }} bash + helm version + shell: bash + + - name: Install kubectl run: | cd echo "Installing kubectl version ${{ inputs.kubectl_version }}" @@ -34,37 +62,27 @@ runs: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl kubectl version --output=yaml shell: bash - - name: Deploy updated images - run: echo "Update images with version ${{ inputs.image_tag }}" - shell: bash - - name: Update database - run: kubectl --context ${{ inputs.kube_context }} - --namespace thecombine - set image deployment/database - database="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_database:${{ inputs.image_tag }}" - shell: bash - - name: Update frontend - run: kubectl --context ${{ inputs.kube_context }} - --namespace thecombine - set image deployment/frontend - frontend="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_frontend:${{ inputs.image_tag }}" - shell: bash - - name: Update backend - run: kubectl --context ${{ inputs.kube_context }} - --namespace thecombine - set image deployment/backend - backend="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_backend:${{ inputs.image_tag }}" - shell: bash - - name: Update maintenance - run: kubectl --context ${{ inputs.kube_context }} - --namespace thecombine - set image deployment/maintenance - maintenance="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_maint:${{ inputs.image_tag }}" + + - name: Install Python requirements + run: python3 -m pip install -r deploy/requirements.txt shell: bash - - name: Update Cert Proxy Server - if: ${{ inputs.update_cert_proxy == 'true' }} - run: kubectl --context ${{ inputs.kube_context }} - --namespace combine-cert-proxy - set image deployment/combine-cert-proxy - combine-cert-proxy="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_maint:${{ inputs.image_tag }}" + + - name: Deploy with Helm + run: > + python3 + deploy/scripts/setup_combine.py + --context ${{ inputs.kube_context }} + --non-interactive + --repo ${{ inputs.image_registry }} + --tag ${{ inputs.image_tag }} + --target ${{ inputs.target }} shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} + AWS_ACCOUNT: ${{ inputs.aws_account }} + AWS_DEFAULT_REGION: ${{ inputs.aws_default_region }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} + COMBINE_CAPTCHA_SECRET_KEY: ${{ inputs.combine_captcha_secret_key }} + COMBINE_JWT_SECRET_KEY: ${{ inputs.combine_jwt_secret_key }} + COMBINE_SMTP_PASSWORD: ${{ inputs.combine_smtp_password }} + COMBINE_SMTP_USERNAME: ${{ inputs.combine_smtp_username }} diff --git a/.github/workflows/combine_deploy_image.yml b/.github/workflows/combine_deploy_image.yml index 84761aed4f..a1c84d1052 100644 --- a/.github/workflows/combine_deploy_image.yml +++ b/.github/workflows/combine_deploy_image.yml @@ -31,6 +31,7 @@ jobs: get.helm.sh:443 github.com:443 production.cloudflare.docker.com:443 + production.cloudfront.docker.com:443 public.ecr.aws:443 pypi.org:443 raw.githubusercontent.com:443 diff --git a/.github/workflows/deploy_qa.yml b/.github/workflows/deploy_qa.yml index 5f6b0f037d..ba8fbbbe21 100644 --- a/.github/workflows/deploy_qa.yml +++ b/.github/workflows/deploy_qa.yml @@ -1,6 +1,12 @@ name: "Deploy Update to QA Server" on: + workflow_dispatch: + inputs: + confirm_qa_deploy: + description: "Type QA to confirm QA deployment" + required: true + default: "" push: branches: [master] @@ -45,6 +51,7 @@ jobs: github.com:443 mcr.microsoft.com:443 production.cloudflare.docker.com:443 + production.cloudfront.docker.com:443 public.ecr.aws:443 pypi.org:443 registry-1.docker.io:443 @@ -53,7 +60,8 @@ jobs: security.ubuntu.com:80 storage.googleapis.com:443 sts.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com:443 - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - name: Build The Combine @@ -65,6 +73,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} build_component: ${{ matrix.component }} + clean_ecr_repo: needs: build runs-on: ubuntu-latest @@ -81,7 +90,8 @@ jobs: api.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com:443 github.com:443 sts.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com:443 - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: @@ -101,23 +111,32 @@ jobs: else REM="v$((VA - 1))\..*"; fi scripts/clean_aws_repo.py combine_frontend combine_backend combine_maint combine_database --remove "${REM}" --verbose fi + deploy_update: needs: build - # Only push to the QA server when built on the master branch - if: github.ref_name == 'master' + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.confirm_qa_deploy == 'QA') runs-on: [self-hosted, thecombine] steps: - name: Harden Runner uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: egress-policy: audit - - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Deploy The Combine Update uses: ./.github/actions/combine-deploy-update with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_account: ${{ secrets.AWS_ACCOUNT }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + combine_captcha_secret_key: ${{ secrets.COMBINE_CAPTCHA_SECRET_KEY }} + combine_jwt_secret_key: ${{ secrets.COMBINE_JWT_SECRET_KEY }} + combine_smtp_password: ${{ secrets.COMBINE_SMTP_PASSWORD }} + combine_smtp_username: ${{ secrets.COMBINE_SMTP_USERNAME }} + helm_version: ${{ vars.HELM_VERSION }} image_registry: ${{ secrets.AWS_ACCOUNT }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com image_tag: ${{ needs.build.outputs.image_tag }} kube_context: ${{ secrets.LTOPS_K8S_STAGING_CONTEXT }} kubectl_version: ${{ vars.KUBECTL_VERSION }} - update_cert_proxy: false + target: qa diff --git a/.github/workflows/deploy_release.yml b/.github/workflows/deploy_release.yml index a259406e19..2b61257718 100644 --- a/.github/workflows/deploy_release.yml +++ b/.github/workflows/deploy_release.yml @@ -38,6 +38,7 @@ jobs: github.com:443 mcr.microsoft.com:443 production.cloudflare.docker.com:443 + production.cloudfront.docker.com:443 public.ecr.aws:443 pypi.org:443 registry-1.docker.io:443 @@ -46,7 +47,8 @@ jobs: security.ubuntu.com:80 storage.googleapis.com:443 sts.us-east-1.amazonaws.com:443 - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Build The Combine id: build_combine uses: ./.github/actions/combine-build @@ -60,6 +62,7 @@ jobs: # https://docs.aws.amazon.com/AmazonECR/latest/public/public-registry-auth.html#public-registry-auth-token aws_default_region: us-east-1 build_component: ${{ matrix.component }} + deploy_update: needs: build runs-on: [self-hosted, thecombine] @@ -68,25 +71,41 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: egress-policy: audit - - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - name: Deploy The Combine Update to QA uses: ./.github/actions/combine-deploy-update with: - image_registry: public.ecr.aws - image_registry_alias: "/thecombine" + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_account: ${{ secrets.AWS_ACCOUNT }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + combine_captcha_secret_key: ${{ secrets.COMBINE_CAPTCHA_SECRET_KEY }} + combine_jwt_secret_key: ${{ secrets.COMBINE_JWT_SECRET_KEY }} + combine_smtp_password: ${{ secrets.COMBINE_SMTP_PASSWORD }} + combine_smtp_username: ${{ secrets.COMBINE_SMTP_USERNAME }} + helm_version: ${{ vars.HELM_VERSION }} + image_registry: public.ecr.aws/thecombine image_tag: ${{ needs.build.outputs.image_tag }} kube_context: ${{ secrets.LTOPS_K8S_STAGING_CONTEXT }} kubectl_version: ${{ vars.KUBECTL_VERSION }} - update_cert_proxy: false + target: qa - name: Deploy The Combine Update to Production uses: ./.github/actions/combine-deploy-update with: - image_registry: public.ecr.aws - image_registry_alias: "/thecombine" + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_account: ${{ secrets.AWS_ACCOUNT }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + combine_captcha_secret_key: ${{ secrets.COMBINE_CAPTCHA_SECRET_KEY }} + combine_jwt_secret_key: ${{ secrets.COMBINE_JWT_SECRET_KEY }} + combine_smtp_password: ${{ secrets.COMBINE_SMTP_PASSWORD }} + combine_smtp_username: ${{ secrets.COMBINE_SMTP_USERNAME }} + helm_version: ${{ vars.HELM_VERSION }} + image_registry: public.ecr.aws/thecombine image_tag: ${{ needs.build.outputs.image_tag }} kube_context: ${{ secrets.LTOPS_K8S_PRODUCTION_CONTEXT }} kubectl_version: ${{ vars.KUBECTL_VERSION }} - update_cert_proxy: true + target: prod diff --git a/.github/workflows/installer_release.yml b/.github/workflows/installer_release.yml index 6365d248da..44807d5659 100644 --- a/.github/workflows/installer_release.yml +++ b/.github/workflows/installer_release.yml @@ -2,6 +2,11 @@ name: installer_release on: workflow_dispatch: + inputs: + confirm_upload: + description: "Type UP to confirm installer upload" + required: true + default: "" workflow_run: workflows: ["Deploy Update to Live Server"] types: @@ -13,7 +18,7 @@ permissions: jobs: make_installer: runs-on: ubuntu-latest - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' outputs: release_tag: ${{ steps.release.outputs.tag }} steps: @@ -74,6 +79,7 @@ jobs: upload_installer: needs: make_installer + if: github.event_name == 'workflow_run' || (github.event_name == 'workflow_dispatch' && github.event.inputs.confirm_upload == 'UP') runs-on: ubuntu-latest steps: # See https://docs.stepsecurity.io/harden-runner/getting-started/ for instructions on diff --git a/.github/workflows/maintenance.yml b/.github/workflows/maintenance.yml index 2912b72927..9b6f426a95 100644 --- a/.github/workflows/maintenance.yml +++ b/.github/workflows/maintenance.yml @@ -28,13 +28,10 @@ jobs: allowed-endpoints: > *.cloudfront.net:443 archive.ubuntu.com:80 - auth.docker.io:443 files.pythonhosted.org:443 github.com:443 - production.cloudflare.docker.com:443 public.ecr.aws:443 pypi.org:443 - registry-1.docker.io:443 security.ubuntu.com:80 # For subfolders, currently a full checkout is required. # See: https://github.com/marketplace/actions/build-and-push-docker-images#path-context diff --git a/README.md b/README.md index 3d49a57918..7610ddde0e 100644 --- a/README.md +++ b/README.md @@ -746,6 +746,8 @@ Notes: - Run the script with the `--help` option to see possible options for the script. +- Run the script with `--non-interactive` in CI/CD to avoid user prompts. + - The setup assumes `amd64` architecture. If the target architecture is `arm64`, add `--set global.cpuArch=arm64`. When the script completes, the resources will be installed on the specified cluster. It may take a few moments before diff --git a/deploy/ansible/vars/k3s_versions.yml b/deploy/ansible/vars/k3s_versions.yml index d1c9c27d33..952167853b 100644 --- a/deploy/ansible/vars/k3s_versions.yml +++ b/deploy/ansible/vars/k3s_versions.yml @@ -1,6 +1,6 @@ --- k3s_version: "v1.30.1%2Bk3s1" -# kubectl_version should match GitHub repo variable KUBECTL_VERSION found at: +# should match GitHub repo variables KUBECTL_VERSION and HELM_VERSION found at: # Settings > Secrets and variables > Actions > Variables > Repository variables kubectl_version: "v1.30.2" helm_version: "v3.15.2" diff --git a/deploy/helm/aws-login/values.yaml b/deploy/helm/aws-login/values.yaml index 86348a7fd6..45d3cbfd71 100644 --- a/deploy/helm/aws-login/values.yaml +++ b/deploy/helm/aws-login/values.yaml @@ -23,7 +23,7 @@ awsEcr: cronJobName: ecr-cred-helper-cron dockerEmail: noreply@thecombine.app image: "public.ecr.aws/thecombine/aws-kubectl" - imageVersion: "0.4.0" + imageVersion: "0.4.1" jobName: ecr-cred-helper schedule: "0 */8 * * *" secretsName: aws-ecr-credentials diff --git a/deploy/helm/create-admin-user/templates/job-create-admin-user.yaml b/deploy/helm/create-admin-user/templates/job-create-admin-user.yaml index eaf79685fb..de7a2b1b4a 100644 --- a/deploy/helm/create-admin-user/templates/job-create-admin-user.yaml +++ b/deploy/helm/create-admin-user/templates/job-create-admin-user.yaml @@ -6,7 +6,7 @@ metadata: annotations: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. - "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook": post-install "helm.sh/hook-delete-policy": before-hook-creation spec: # keep completed jobs for 24 hrs so that logs are diff --git a/deploy/helm/thecombine/charts/maintenance/values.yaml b/deploy/helm/thecombine/charts/maintenance/values.yaml index f1cbe8e0b5..f3eb287571 100644 --- a/deploy/helm/thecombine/charts/maintenance/values.yaml +++ b/deploy/helm/thecombine/charts/maintenance/values.yaml @@ -37,7 +37,7 @@ serviceAccount: awsEcr: image: "public.ecr.aws/thecombine/aws-kubectl" - imageVersion: "0.4.0" + imageVersion: "0.4.1" ####################################### # Variables controlling backups diff --git a/deploy/helm/thecombine/templates/letsencrypt-prod.yaml b/deploy/helm/thecombine/templates/letsencrypt-prod.yaml index 5a0976ee0f..7f22cd47c4 100644 --- a/deploy/helm/thecombine/templates/letsencrypt-prod.yaml +++ b/deploy/helm/thecombine/templates/letsencrypt-prod.yaml @@ -1,4 +1,4 @@ -{{- if .Values.certManager.enabled }} +{{- if and .Values.certManager.enabled (eq .Values.certManager.certIssuer "letsencrypt-prod") }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: diff --git a/deploy/helm/thecombine/templates/letsencrypt-staging.yaml b/deploy/helm/thecombine/templates/letsencrypt-staging.yaml index d5a0d6f0f7..2dee8b2d70 100644 --- a/deploy/helm/thecombine/templates/letsencrypt-staging.yaml +++ b/deploy/helm/thecombine/templates/letsencrypt-staging.yaml @@ -1,4 +1,4 @@ -{{- if .Values.certManager.enabled }} +{{- if and .Values.certManager.enabled (eq .Values.certManager.certIssuer "letsencrypt-staging") }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: diff --git a/deploy/helm/thecombine/templates/self-signed.yaml b/deploy/helm/thecombine/templates/self-signed.yaml index 4676ac173b..ab2e024f57 100644 --- a/deploy/helm/thecombine/templates/self-signed.yaml +++ b/deploy/helm/thecombine/templates/self-signed.yaml @@ -1,4 +1,4 @@ -{{- if .Values.certManager.enabled }} +{{- if and .Values.certManager.enabled (eq .Values.certManager.certIssuer "self-signed") }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: diff --git a/deploy/scripts/enum_types.py b/deploy/scripts/enum_types.py index 9fcb4521c4..9adc12371c 100644 --- a/deploy/scripts/enum_types.py +++ b/deploy/scripts/enum_types.py @@ -3,16 +3,6 @@ from enum import Enum, unique -@unique -class HelmAction(Enum): - """Enumerate helm commands for maintaining The Combine on the target system.""" - - INSTALL = "install" - """INSTALL is used when the chart is not already installed on the target.""" - UPGRADE = "upgrade" - """UPGRADE is used when the chart is already installed.""" - - @unique class ExitStatus(Enum): SUCCESS = 0 diff --git a/deploy/scripts/helm_utils.py b/deploy/scripts/helm_utils.py index 8da82a4854..8241901691 100644 --- a/deploy/scripts/helm_utils.py +++ b/deploy/scripts/helm_utils.py @@ -45,7 +45,7 @@ def create_secrets( def get_installed_charts(helm_cmd: List[str], helm_namespace: str) -> List[str]: """Create a list of the helm charts that are already installed on the target.""" - lookup_results = run_cmd(helm_cmd + ["list", "-a", "-n", helm_namespace, "-o", "yaml"]) + lookup_results = run_cmd(helm_cmd + ["list", "-n", helm_namespace, "-o", "yaml"]) chart_info: List[Dict[str, str]] = yaml.safe_load(lookup_results.stdout) chart_list: List[str] = [] for chart in chart_info: diff --git a/deploy/scripts/setup_cluster.py b/deploy/scripts/setup_cluster.py index efe1dae54a..c3069ceae8 100755 --- a/deploy/scripts/setup_cluster.py +++ b/deploy/scripts/setup_cluster.py @@ -11,7 +11,7 @@ import tempfile from typing import Any, Dict, List -from enum_types import ExitStatus, HelmAction +from enum_types import ExitStatus from kube_env import KubernetesEnvironment, add_helm_opts, add_kube_opts from utils import init_logging, run_cmd import yaml @@ -102,12 +102,6 @@ def main() -> None: # existing repos. run_cmd(["helm", "repo", "update"], print_cmd=not args.quiet, print_output=not args.quiet) - # List current charts - chart_list_results = run_cmd(["helm", "list", "-A", "-o", "yaml"]) - curr_charts: List[str] = [] - for chart in yaml.safe_load(chart_list_results.stdout): - curr_charts.append(chart["name"]) - # Add the current script directory to the OS Environment variables os.environ["SCRIPTS_DIR"] = str(scripts_dir) # Add an empty analytics key if not defined in the OS Environment variables @@ -121,15 +115,13 @@ def main() -> None: chart_spec = config[chart_descr]["chart"] # install the chart helm_cmd = kube_env.get_helm_cmd() - if chart_spec["name"] in curr_charts: - helm_action = HelmAction.UPGRADE - else: - helm_action = HelmAction.INSTALL helm_cmd.extend( [ "-n", chart_spec["namespace"], - helm_action.value, + "upgrade", + "--install", + "--create-namespace", chart_spec["name"], ] ) @@ -146,16 +138,14 @@ def main() -> None: # chart is a *.tgz file chart_files = list((Path(args.chart_dir).resolve() / chart_spec["name"]).glob("*.tgz")) if not chart_files: - logging.error(f"No chart file for {chart['name']} in {args.chart_dir}.") - sys.exit(1) + logging.error(f"No chart file for {chart_spec['name']} in {args.chart_dir}.") + sys.exit(ExitStatus.FAILURE.value) if len(chart_files) > 1: logging.warning( - f"Expecting 1 chart file for {chart['name']}, found {len(chart_files)}" + f"Expecting 1 chart file for {chart_spec['name']}, found {len(chart_files)}" ) helm_cmd.append(str(chart_files[0])) - if helm_action == HelmAction.INSTALL: - helm_cmd.append("--create-namespace") if ("wait" in chart_spec and chart_spec["wait"]) or args.timeout is not None: helm_cmd.append("--wait") if args.timeout is not None: @@ -177,7 +167,7 @@ def main() -> None: # command is running exit_status = os.waitstatus_to_exitcode(os.system(helm_cmd_str)) logging.info( - f'helm {helm_action.value} of {chart_spec["name"]} ' + f'helm install/upgrade of {chart_spec["name"]} ' + f"returned exit status {hex(exit_status)}" ) if exit_status != 0: diff --git a/deploy/scripts/setup_combine.py b/deploy/scripts/setup_combine.py index f51bf63f8d..7cf5403432 100755 --- a/deploy/scripts/setup_combine.py +++ b/deploy/scripts/setup_combine.py @@ -28,7 +28,7 @@ from app_release import get_release from aws_env import init_aws_environment import combine_charts -from enum_types import ExitStatus, HelmAction +from enum_types import ExitStatus from helm_utils import ( add_language_overrides, add_override_values, @@ -77,6 +77,12 @@ def parse_args() -> argparse.Namespace: action="store_true", help="List the available targets and exit.", ) + parser.add_argument( + "--non-interactive", + "-n", + action="store_true", + help="Disable interactive prompts (for CI/CD use).", + ) parser.add_argument( "--profile", "-p", @@ -132,6 +138,9 @@ def main() -> None: sys.exit(ExitStatus.SUCCESS.value) target = args.target + if args.non_interactive and target not in config["targets"]: + logging.error(f"Target '{target}' not found in configuration") + sys.exit(ExitStatus.FAILURE.value) while target not in config["targets"]: target = get_target(config) @@ -143,11 +152,13 @@ def main() -> None: profile = args.profile # Verify the Kubernetes/Helm environment - kube_env = KubernetesEnvironment(args) + kube_env = KubernetesEnvironment(args, prompt_for_context=not args.non_interactive) # Cache helm command used to alter the target cluster helm_cmd = kube_env.get_helm_cmd() + # Check AWS Environment Variables - init_aws_environment() + if not args.non_interactive: + init_aws_environment() # Create list of target specific variable values target_vars = [f"global.imageTag={args.image_tag}"] @@ -184,15 +195,20 @@ def main() -> None: installed_charts = get_installed_charts(helm_cmd, chart_namespace) logging.debug(f"Installed charts: {installed_charts}") - # Set helm_action based on whether chart is already installed - helm_action = HelmAction.INSTALL + namespace_cmd = helm_cmd + ["--namespace", chart_namespace] + if chart in installed_charts: + # Delete existing chart if --clean specified if args.clean: - # Delete existing chart if --clean specified - delete_cmd = helm_cmd + [f"--namespace={chart_namespace}", "delete", chart] + delete_cmd = namespace_cmd + ["delete", chart] + if args.dry_run: + delete_cmd.append("--dry-run") run_cmd(delete_cmd, print_cmd=not args.quiet, print_output=True) - else: - helm_action = HelmAction.UPGRADE + + # Skip existing install-only chart unless --clean specified + elif config["charts"][chart].get("install_only", False): + logging.info(f"Skipping install-only chart '{chart}' (already installed)") + continue # Build the secrets file secrets_file = Path(secrets_dir).resolve() / f"secrets_{chart}.yaml" @@ -202,17 +218,14 @@ def main() -> None: env_vars_req=this_config["env_vars_required"], ) - # Create the base helm install command - chart_dir = helm_dir / chart - helm_install_cmd = helm_cmd + [ + # Create the base helm install/upgrade command + helm_install_cmd = namespace_cmd + [ + "upgrade", "--dependency-update", - f"--namespace={chart_namespace}", - helm_action.value, + "--install", chart, - str(chart_dir), + str(helm_dir / chart), ] - - # Set the dry-run option if desired if args.dry_run: helm_install_cmd.append("--dry-run") diff --git a/deploy/scripts/setup_files/combine_config.yaml b/deploy/scripts/setup_files/combine_config.yaml index 25755f0e17..e9e4743e4f 100644 --- a/deploy/scripts/setup_files/combine_config.yaml +++ b/deploy/scripts/setup_files/combine_config.yaml @@ -156,6 +156,7 @@ charts: create-admin-user: namespace: thecombine install_langs: false + install_only: true secrets: - config_item: awsAccount env_var: AWS_ACCOUNT diff --git a/docs/deploy/README.md b/docs/deploy/README.md index b9cc53ce25..a38e222b9b 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -25,10 +25,11 @@ This document describes how to deploy _The Combine_ to a target Kubernetes clust 2. [Setup Environment](#setup-environment) 7. [Install Helm Charts Required by _The Combine_](#install-helm-charts-required-by-the-combine) 8. [Install _The Combine_](#install-the-combine) -9. [Maintenance](#maintenance) - 1. [Maintenance Scripts for Kubernetes](#maintenance-scripts-for-kubernetes) - 2. [Checking Certificate Expiration](#checking-certificate-expiration) - 3. [Creating your own Configurations](#creating-your-own-configurations) +9. [GitHub Actions CI/CD Setup](#github-actions-cicd-setup) +10. [Maintenance](#maintenance) + 1. [Maintenance Scripts for Kubernetes](#maintenance-scripts-for-kubernetes) + 2. [Checking Certificate Expiration](#checking-certificate-expiration) + 3. [Creating your own Configurations](#creating-your-own-configurations) ## System Design @@ -418,6 +419,21 @@ Notes: kubectl -n thecombine exec deployment/database -- /docker-entrypoint-initdb.d/update-semantic-domains.sh ``` +## GitHub Actions CI/CD Setup + +The repository uses two GitHub Actions workflows to automatically deploy _The Combine_ to the QA and Production servers +using `helm upgrade` via the `deploy/scripts/setup_combine.py` script. + +### CI/CD Workflow Overview + +- **`deploy_qa.yml`**: Triggered on every push to the `master` branch. Builds new images from the latest code and + deploys them to the QA server using the private AWS ECR registry. +- **`deploy_release.yml`**: Triggered when a new GitHub release is published. Deploys the released images (from the + public AWS ECR registry) to both the QA and Production servers. + +Both workflows run the deploy step on a `[self-hosted, thecombine]` runner that has network access to the Kubernetes +clusters (e.g. via WireGuard). The runner must have its `~/.kube/config` pre-configured with the cluster contexts. + ## Maintenance ### Maintenance Scripts for Kubernetes diff --git a/maintenance/Dockerfile b/maintenance/Dockerfile index 3096ff14d0..1281e0e6cf 100644 --- a/maintenance/Dockerfile +++ b/maintenance/Dockerfile @@ -16,7 +16,7 @@ # - ARM 64-bit ############################################################ -FROM public.ecr.aws/thecombine/aws-kubectl:0.4.0-$TARGETARCH +FROM public.ecr.aws/thecombine/aws-kubectl:0.4.1-$TARGETARCH USER root diff --git a/maintenance/scripts/aws_backup.py b/maintenance/scripts/aws_backup.py index 2fe65894a3..5238110f90 100644 --- a/maintenance/scripts/aws_backup.py +++ b/maintenance/scripts/aws_backup.py @@ -21,12 +21,12 @@ def __init__(self, *, bucket: str) -> None: def push(self, src: Path, dest: str) -> subprocess.CompletedProcess[str]: """Push a file to the AWS S3 bucket.""" s3_uri = f"{self.bucket}/{dest}" - return run_cmd(["aws", "s3", "cp", str(src), s3_uri]) + return run_cmd(["aws", "s3", "cp", "--no-progress", str(src), s3_uri]) def pull(self, src: str, dest: Path) -> subprocess.CompletedProcess[str]: - """Push a file to the AWS S3 bucket.""" + """Pull a file from the AWS S3 bucket.""" s3_uri = f"{self.bucket}/{src}" - return run_cmd(["aws", "s3", "cp", s3_uri, str(dest)]) + return run_cmd(["aws", "s3", "cp", "--no-progress", s3_uri, str(dest)]) def list(self) -> subprocess.CompletedProcess[str]: """List the objects in the S3 bucket.""" diff --git a/maintenance/scripts/combine_restore.py b/maintenance/scripts/combine_restore.py index 4e937b7fb2..c16f7a805b 100755 --- a/maintenance/scripts/combine_restore.py +++ b/maintenance/scripts/combine_restore.py @@ -63,9 +63,9 @@ def main() -> None: """Restore The Combine from a backup stored in the AWS S3 service.""" args = parse_args() if args.verbose: - logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) + logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG) else: - logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.WARNING) + logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) # Look up the required environment variables aws_bucket, db_files_subdir, backend_files_subdir = check_env_vars( ["aws_bucket", "db_files_subdir", "backend_files_subdir"] @@ -82,10 +82,10 @@ def main() -> None: backup = args.file else: # Get the list of backups - backup_list_output = aws.list().stdout.strip().split("\n") + backup_list_output = [line for line in aws.list().stdout.strip().split("\n") if line] if len(backup_list_output) == 0: - print(f"No backups available from {aws_bucket}") + logging.warning(f"No backups available from {aws_bucket}") sys.exit(0) # Convert the list of backups to a more useful structure @@ -120,7 +120,9 @@ def main() -> None: step.print(f"Fetch the selected backup, {backup}.") - aws.pull(backup, Path(restore_dir) / restore_file) + aws_proc = aws.pull(backup, Path(restore_dir) / restore_file) + logging.debug(f"stderr:\n{aws_proc.stderr.strip()}") + logging.debug(f"stdout:\n{aws_proc.stdout.strip()}") step.print("Unpack the backup.") os.chdir(restore_dir) @@ -154,54 +156,54 @@ def safe_extract( step.print("Restore the database.") db_pod = combine.get_pod_id(CombineApp.Component.Database) if not db_pod: - print("Cannot find the database container.", file=sys.stderr) + logging.error("Cannot find the database container.") sys.exit(1) - combine.kubectl( - [ - "cp", - db_files_subdir, - f"{db_pod}:/", - ] - ) - combine.exec( - db_pod, - [ - "mongorestore", - "--drop", - "--gzip", - "--quiet", - f"--dir=/{db_files_subdir}", - ], - ) - combine.exec( - db_pod, - [ - "rm", - "-rf", - f"/{db_files_subdir}", - ], + logging.debug(f"Copying {db_files_subdir} to {db_pod} ...") + cp_proc = combine.kubectl(["cp", db_files_subdir, f"{db_pod}:/"]) + logging.debug(f"stderr:\n{cp_proc.stderr.strip()}") + logging.debug(f"stdout:\n{cp_proc.stdout.strip()}") + + logging.debug(f"Running mongorestore on {db_pod} ...") + mongorestore_proc = combine.exec( + db_pod, ["mongorestore", "--drop", "--gzip", f"--dir=/{db_files_subdir}"] ) + logging.debug(f"stderr:\n{mongorestore_proc.stderr.strip()}") + logging.debug(f"stdout:\n{mongorestore_proc.stdout.strip()}") + + logging.debug(f"Removing {db_files_subdir} from {db_pod} ...") + rm_proc = combine.exec(db_pod, ["rm", "-rf", f"/{db_files_subdir}"]) + logging.debug(f"stderr:\n{rm_proc.stderr.strip()}") + logging.debug(f"stdout:\n{rm_proc.stdout.strip()}") step.print("Copy the backend files.") backend_pod = combine.get_pod_id(CombineApp.Component.Backend) if not backend_pod: - print("Cannot find the backend container.", file=sys.stderr) + logging.error("Cannot find the backend container.") sys.exit(1) + # if --clean option was used, delete the existing backend files if args.clean: + logging.info(f"Cleaning out backend files in {backend_pod} ...") # we run the rm command inside a bash shell so that the shell will do wildcard # expansion - combine.exec( + clean_proc = combine.exec( backend_pod, - [ - "/bin/bash", - "-c", - f"rm -rf /home/app/{backend_files_subdir}/*", - ], + ["/bin/bash", "-c", f"rm -rf /home/app/{backend_files_subdir}/*"], ) - - combine.kubectl(["cp", backend_files_subdir, f"{backend_pod}:/home/app", "--no-preserve"]) + logging.debug(f"stderr:\n{clean_proc.stderr.strip()}") + logging.debug(f"stdout:\n{clean_proc.stdout.strip()}") + + # Iterate through every item in the backend subdirectory + remote_subdir = f"{backend_pod}:/home/app/{backend_files_subdir}/" + logging.debug(f"Copying contents of {backend_files_subdir} to {remote_subdir} ...") + for item in os.listdir(backend_files_subdir): + if item.startswith(".") or item == "lost+found": + logging.debug(f"Skipping {item} ...") + continue + logging.debug(f"Copying {item} ...") + local_item = os.path.join(backend_files_subdir, item) + combine.kubectl(["cp", local_item, remote_subdir, "--no-preserve"]) if __name__ == "__main__":