ci.yml

---
name: CI
# Build and run tests in containers
# Store test and prod images to GitHub Container Registry (GHCR)
# Build prod image and push to AWS ECR
# Run tests and quality checks on test image
# Run security scans on code and prod images
# Use matrix for multiple versions of Elixir, OTP, and OS

# on: push
on:
  push:
    branches:
      - main
    # tags: [prod]
    paths-ignore:
      - README.md
      # - '.github/**'
      - .vscode
      - .gitignore
      - "terraform/*"
      # Handled by otel.yml GitHub Action
      - .github/workflows/otel.yml
      - deploy/aws-otel-collector.Dockerfile
      - 'otel/*'
      # Handled by newman.yml GitHub Action
      - deploy/newman.Dockerfile

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
env:
  # Name of image
  IMAGE_NAME: foo-app
  # Name of org in GHCR Docker repository (must be lowercase)
  IMAGE_OWNER: ${{ github.repository_owner }}
  # IMAGE_OWNER: cogini
  # AWS ECR Docker repo "org" name (may be blank, otherwise must have trailing slash)
  ECR_IMAGE_OWNER: cogini/
  # ECR_IMAGE_OWNER: ''
  # Tag for release images, used to find the latest deployed image.
  IMAGE_TAG: latest
  IMAGE_VER: ${{ github.sha }}
  # Registry to pull internal images from
  REGISTRY: ghcr.io/
  # Registry for public images, default (blank) is docker.io
  # PUBLIC_REGISTRY: ''
  # When base image has been synced to local registry
  PUBLIC_REGISTRY: ghcr.io/
  REPO_ORG: ${{ github.repository_owner }}
  REPO_ORG_AMAZON: ${{ github.repository_owner }}
  REPO_ORG_ELIXIR: ${{ github.repository_owner }}
  REPO_ORG_KAFKA: ${{ github.repository_owner }}
  REPO_ORG_POSTGRES: ${{ github.repository_owner }}
  # Git "main" branch. This might be "master" for old repos
  MAIN_BRANCH: main
  # GitHub Environment secrets and variables
  # Docker Hub credentials to pull base images without rate limits
  #   secrets.DOCKERHUB_USERNAME
  #   secrets.DOCKERHUB_TOKEN
  # AWS Account
  #   secrets.AWS_ACCOUNT_ID
  # AWS default region
  #   vars.AWS_REGION
  # AWS role allowing GitHub Actions to access resources and deploy
  #   secrets.AWS_ROLE_TO_ASSUME: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/foo-${{ environment }}-github-action
  # S3 bucket where assets are deployed, e.g., for use with CloudFront CDN
  #   vars.S3_BUCKET_ASSETS: cogini-foo-app-dev-app-assets
  # AWS CloudFront distribution ID to invalidate cache after deploy
  #   secrets.CLOUDFRONT_CDN_DISTRIBUTION_ID
  # S3 bucket with data for testing
  #   vars.S3_BUCKET_CI: cogini-prod-foo-ci
  # SSH key to access private repos during build
  #   secrets.SSH_PRIVATE_KEY
  # GitHub access token to access other repositories during build
  #   secrets.DEVOPS_ACCESS_TOKEN
  # AWS ECS deployment role names to put in task definition
  #   secrets.TASK_ROLE_ARN: "arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/foo-app"
  #   secrets.EXECUTION_ROLE_ARN: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/foo-ecs-task-execution-role
  # AWS Systems Manager Parameter Store prefix for config keys
  #   vars.AWS_PS_PREFIX: cogini/foo/prod
  # AppSignal API key for error reporting
  #   secrets.APPSIGNAL_PUSH_API_KEY
  # DataDog API key for reporting test results
  #   secrets.ACTIONS_DD_API_KEY
  #   secrets.DD_API_KEY
  # Oban Pro license
  #   secrets.OBAN_KEY_FINGERPRINT
  #   secrets.OBAN_LICENSE_KEY
  # Sentry.io
  #   secrets.SENTRY_AUTH_TOKEN
  #   vars.SENTRY_ORG: cogini
  #   vars.SENTRY_PROJECT: phoenix-container-example
  # secrets.GITLEAKS_LICENSE
  # secrets.SNYK_TOKEN
  # Target port for prod image tests
  APP_PORT: 4000
  # Elixir module, used in health checks
  ELIXIR_MODULE: PhoenixContainerExample
  # Elixir module Erlang app name
  ELIXIR_APP: phoenix_container_example
  APPSIGNAL_APP_NAME: phoenix-container-example
  # Name of environment for resources created by Terraform
  # TERRAFORM_ENV: dev
  # Docker
  DOCKER_BUILDKIT: '1'
  COMPOSE_DOCKER_CLI_BUILD: '1'
  COMPOSE_FILE: docker-compose.gha.yml
  DOCKER_FILE: deploy/debian.Dockerfile
jobs:
  config:
    name: Configure build
    runs-on: ubuntu-latest
    outputs:
      # Combinations of Elixir/Erlang/OS to run internal tests on
      # Generally this will match prod
      test-matrix: ${{ steps.common-matrix.outputs.result }}
      # Combinations to tests on in parallel, i.e., mix test with partitions
      test-matrix-parallel: ${{ steps.test-matrix-parallel.outputs.result }}
      # Combinations for prod images
      # Used for the final deploy image as well as external tests
      prod-matrix: ${{ steps.prod-matrix.outputs.result }}
      # Combinations to deploy (to prod or another environment)
      # Usually only a single variant
      deploy-matrix: ${{ steps.deploy-matrix.outputs.result }}
      # Combinations to sync images to GHCR for
      sync-matrix: ${{ steps.common-matrix.outputs.result }}

      # Select environment to deploy to based on git branch/tag
      environment: ${{ (github.ref_name == 'main' && 'staging') || (github.ref_name == 'prod' && 'production') }}

      # Whether to deploy. Only standard tags/branches deploy, not other dev branches.
      deploy: ${{ contains(fromJson('["main", "staging", "qa", "prod"]'), github.ref_name) }}

      # Sync 3rd-party images to GHCR
      # sync-images: '${{ steps.get-sync-images.outputs.result }}'
      sync-images: '1'

      # Sync 3rd-party images to ECR
      # sync-images-ecr: '${{ steps.get-sync-images.outputs.result }}'
      sync-images-ecr: '0'

      # Enable AWS interactions in the build, including deploys.
      aws-enabled: '1'

      # Run unit tests
      test: '1'

      # Run dialyzer
      test-dialyzer: '0'

      # Security scan prod image
      scan-prod-image: '0'

      # Security scan code image
      scan-code: '0'

      # Use specific scanning tools
      scan-gitleaks: '0'
      scan-grype: '1'
      scan-snyk: '0'
      scan-trivy: '0'

      # Use GitHub Advanced Security, free for open source, otherwise a paid feature
      # https://docs.github.com/en/get-started/learning-about-github/about-github-advanced-security
      # https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning
      # https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github
      github-advanced-security: '0'

      # Upload test results to Datadog
      test-datadog: '0'

      # Upload assets to S3
      upload-assets: '0'

      # Create package for deployment, e.g., Erlang release
      package: '1'
      package-ansible: '1'
      package-codedeploy: '1'
      package-erlang-release: '1'

      # Create GitHub release to store build artifacts, required for packaging
      create-release: '1'

      # Deploy assets to AWS S3 bucket
      deploy-assets-s3: '0'

      # Deploy using AWS CodeDeploy
      deploy-codedeploy: '0'
      # Deploy to AWS ECS
      deploy-ecs: '1'
      # Push prod image to AWS ECR
      deploy-ecr: '1'

      # Whether to tag released images
      tag-images: '1'

      # Tag for GHCR images. The repo is shared between all environments, so use tags to distinguish.
      image-tag-ghcr: ${{ (github.ref_name == 'main' && 'test') || (github.ref_name == 'qa' && 'qa') || (github.ref_name == 'prod' && 'latest') || github.ref_name }}

      # Tag for ECR images. Repos are separate between environments, so each gets "latest" tag
      image-tag-ecr: ${{ contains(fromJson('["main", "staging", "prod"]'), github.ref_name) && 'latest' || github.ref_name }}

      image-tag-deploy: ${{ steps.get-image-tag-deploy.outputs.result }}

      # Notify AppSignal of new app release
      notify-appsignal: '1'

      # Notify Sentry of new app release
      notify-sentry: '1'
    steps:
      - name: Configure common-matrix
        id: common-matrix
        uses: actions/github-script@v9
        # Specify versions of Erlang, Elixir, and base OS
        # in a combination supported by https://hub.docker.com/r/hexpm/elixir/tags
        with:
          script: |
            return {
              include: [
                {
                  os: "debian",
                  elixir: "1.19.5",
                  otp: "28.5",
                  build_os_ver: "trixie-20260518-slim",
                  prod_os_ver: "trixie-20260518-slim"
                },
              ]
            }

      # Combinations of Elixir/Erlang/OS to run internal tests on
      # Generally this will match prod
      - name: Configure test-matrix-parallel
        id: test-matrix-parallel
        uses: actions/github-script@v9
        with:
          script: |
            let platforms = [
              {
                os: "debian",
                elixir: "1.19.5",
                otp: "28.5",
                build_os_ver: "trixie-20260518-slim",
                prod_os_ver: "trixie-20260518-slim"
              },
            ]
            let ci_nodes = [1, 2]
            let ci_node_total = ci_nodes.length

            let matrix = []
            for (let i = 0; i < platforms.length; i++) {
              for (let j = 0; j < ci_nodes.length; j++) {
                matrix.push(Object.assign({}, platforms[i], {ci_node_total: ci_node_total, ci_node_index: ci_nodes[j]}) )
              }
            }

            return { include: matrix }

      # Combinations for prod images
      # Used for the final deploy image as well as external tests
      - name: Configure prod-matrix
        id: prod-matrix
        uses: actions/github-script@v9
        with:
          script: |
            return {
              include: [
                {
                  os: "debian",
                  elixir: "1.19.5",
                  otp: "28.5",
                  build_os_ver: "trixie-20260518-slim",
                  prod_os_ver: "trixie-20260518-slim"
                },
                {
                  os: "distroless",
                  elixir: "1.19.5",
                  otp: "28.5",
                  build_os_ver: "trixie-20260518-slim",
                  prod_os_ver: "cc-debian13"
                },
              ]
            }

      # Combinations to deploy (to prod or another environment)
      # Usually only a single variant
      - name: Configure deploy-matrix
        id: deploy-matrix
        uses: actions/github-script@v9
        with:
          script: |
            return {
              include: [
                {
                  os: "debian",
                  elixir: "1.19.5",
                  otp: "28.5",
                  build_os_ver: "trixie-20260518-slim",
                  prod_os_ver: "trixie-20260518-slim"
                },
              ]
            }

      # Select GitHub Actions environment based on branch or tag name
      # This also implicitly selects the AWS deploy environment.
      # * `main` branch or `test` tag deploys to test
      # * `staging` or `staging-*` tag/branch deploys to staging
      # * `qa` or `qa-*` tag/branch deploys to qa
      # * `prod` tag deploys to prod and dr
      - name: Configure environment
        id: get-environment
        uses: actions/github-script@v9
        env:
          GITHUB_REF_NAME: ${{ github.ref_name }}
        with:
          result-encoding: string
          script: |
            ref_name = process.env.GITHUB_REF_NAME

            var environments = {
              'test': 'test',
              'main': 'test',
              'staging': 'staging',
              'qa': 'qa',
              'prod': 'prod',
              'dr': 'prod',
              'media': 'prod',
            }

            if (ref_name in environments) {
              result = environments[ref_name]
            } else {
              if (ref_name.startsWith('staging-')) {
                result = 'staging'
              } else if (ref_name.startsWith('qa-')) {
                result = 'qa'
              } else {
                result = 'test'
              }
            }

            return result

      # Get image tag for deploy based on environment
      # In test, staging, and qa, deploy is based on GitHub SHA
      # In prod, deploy is based on staging ECR repo
      # In dr, deploy is based on prod ECR repo in DR region
      - name: Configure image-tag-deploy
        id: get-image-tag-deploy
        uses: actions/github-script@v9
        env:
          GITHUB_REF_NAME: ${{ github.ref_name }}
        with:
          result-encoding: string
          script: |
            ref_name = process.env.GITHUB_REF_NAME
            if (ref_name == 'prod' || ref_name == 'dr') {
              result = 'prod'
            } else {
              result = context.sha
            }
            return result

      # Determine whether to deploy to AWS based on branch or tag name
      - name: Configure deploy
        id: get-deploy
        uses: actions/github-script@v9
        env:
          GITHUB_REF_NAME: ${{ github.ref_name }}
        with:
          result-encoding: string
          script: |
            ref_name = process.env.GITHUB_REF_NAME
            if (ref_name == 'main' || ref_name == 'staging' || ref_name.startsWith('staging-')) {
              result = '1'
            } else if (ref_name == 'qa' || ref_name.startsWith('qa-')) {
              result = '1'
            } else if (ref_name == 'testing' || ref_name.startsWith('devops-')) {
              result = '1'
            } else {
              result = '0'
            }
            return result

      - name: Configure sync-images
        id: get-sync-images
        uses: actions/github-script@v9
        env:
          GITHUB_REF_NAME: ${{ github.ref_name }}
        with:
          result-encoding: string
          script: |
            ref_name = process.env.GITHUB_REF_NAME
            if (ref_name == 'main' || ref_name == 'staging' || ref_name.startsWith('staging-')) {
              result = '1'
            } else if (ref_name == 'testing' || ref_name.startsWith('devops-')) {
              result = '1'
            } else if (ref_name == 'qa' || ref_name.startsWith('qa-')) {
              result = '1'
            } else {
              result = '0'
            }
            return result

  create-release:
    name: Create GitHub release
    needs: [config]
    if: needs.config.outputs.create-release == '1'
    runs-on: ubuntu-latest
    outputs:
      release-tag: build-${{ github.run_number }}
      upload-url: ${{ steps.release-view.outputs.upload-url }}
    env:
      GH_TOKEN: ${{ github.token }}
      RELEASE_TAG: build-${{ github.run_number }}
    steps:
      - name: Check out source
        uses: actions/checkout@v6

      # - name: Tag repository
      #   run: |
      #     echo "$RELEASE_TAG"
      #     git tag "$RELEASE_TAG"
      #     git push origin "$RELEASE_TAG"

      - name: Create release
        if: ${{ github.run_attempt == '1' }}
        run: gh release create "$RELEASE_TAG" -n "Build $RELEASE_TAG"

      - name: Get release info
        id: release-view
        run: |
          UPLOAD_URL="$(gh release view "$RELEASE_TAG" --json uploadUrl --jq .uploadUrl | cut -d'{' -f'1')"
          echo "upload-url=$UPLOAD_URL" >> "$GITHUB_OUTPUT"

  build-test:
    name: Build test image
    needs: [sync-images-matrix, config]
    permissions:
      id-token: write
      contents: read
      packages: write
    runs-on: ubuntu-latest
    timeout-minutes: 30
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.test-matrix) }}
    env:
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
      TEST_DATA: '0'
    steps:
      - name: Check out source
        uses: actions/checkout@v6

      - name: Set variables
        shell: bash
        run: |
          echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> "$GITHUB_ENV"
          echo "run_id=${GITHUB_RUN_ID}" >> "$GITHUB_OUTPUT"
          echo "run_num=${GITHUB_RUN_NUMBER}" >> "$GITHUB_OUTPUT"

      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # - name: Debug environment
      #   # env.ACTIONS_STEP_DEBUG == 'true'
      #   run: |
      #     # echo "github.event_name: ${{ github.event_name }}"
      #     # echo "ref: ${{ github.ref }}"
      #     # echo "ref_name: ${{ github.ref_name }}"
      #     # echo "environment: ${{ needs.config.outputs.environment }}"
      #     # echo "components: ${{ needs.config.outputs.components }}"
      #     # echo -n "AWS_ACCOUNT_ID: "
      #     # echo ${{ secrets.AWS_ACCOUNT_ID }} | sed 's/./& /g'
      #     # echo "AWS_REGION var: ${{ vars.AWS_REGION }}"

      # Pull public images without rate limits
      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      # - name: Configure ssh keys
      #   uses: webfactory/ssh-agent@v0.7.0
      #   # Configure machine key or deploy keys to access private repos from build
      #   # https://github.com/marketplace/actions/webfactory-ssh-agent
      #   # https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys
      #   # ssh-keygen -t ed25519 -m pem -C "git@github.com:reachfh/api-utils.git" -f api-utils
      #   with:
      #     ssh-private-key: |
      #       ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Set env vars for builds
        run: |
          echo 'DATABASE_HOST=postgres' >> ./.env.test

      # - name: Set up QEMU for multi-platform builds
      #   id: qemu
      #   uses: docker/setup-qemu-action@v2
      #   with:
      #     platforms: linux/amd64,linux/arm64

      # - name: Display available platforms
      #   run: echo "${{ steps.qemu.outputs.platforms }}"

      - name: Configure AWS credentials
        if: env.TEST_DATA == '1'
        uses: aws-actions/configure-aws-credentials@v6
        # https://github.com/aws-actions/configure-aws-credentials
        # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Get test data files
        if: env.TEST_DATA == '1'
        run: |
          mkdir -p test/bert
          aws s3 cp "s3://${{ vars.S3_BUCKET_CI }}/test-data.zip" test-data.zip
          unzip -o test-data.zip -d test/data

      - name: Set up Docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v4
        with:
          driver-opts: network=host

      - name: Build test image and push to GHCR
        uses: docker/build-push-action@v7
        with:
          file: ${{ env.DOCKER_FILE }}
          target: test
          build-args: |
            ELIXIR_VER=${{ matrix.elixir }}
            OTP_VER=${{ matrix.otp }}
            BUILD_OS_VER=${{ matrix.build_os_ver }}
            PROD_OS_VER=${{ matrix.prod_os_ver }}
            PUBLIC_REGISTRY=${{ env.PUBLIC_REGISTRY }}
            REPO_ORG_PROD_OS=${{ env.REPO_ORG }}/
            REPO_ORG_ELIXIR=${{ env.REPO_ORG_ELIXIR }}
          context: .
          builder: ${{ steps.buildx.outputs.name }}
          push: true
          cache-from: |
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-test${{ env.VAR }}-${{ env.MAIN_BRANCH }}
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-test${{ env.VAR }}-${{ github.ref_name }}
          cache-to: |
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-test${{ env.VAR }}-${{ github.ref_name }},mode=max
          no-cache: ${{ github.run_attempt != '1' }}
          # platforms: linux/amd64,linux/arm64
          # ssh: default
          tags: |
            ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:test${{ env.VAR }}${{ env.IMAGE_VER }}
          # secrets: |
          #   "access_token=${{ secrets.DEVOPS_ACCESS_TOKEN }}"
          #   "oban_key_fingerprint=${{ secrets.OBAN_KEY_FINGERPRINT }}"
          #   "oban_license_key=${{ secrets.OBAN_LICENSE_KEY }}"

  test:
    name: Run tests
    if: needs.config.outputs.test == '1'
    needs: [build-test, config]
    environment: ${{ needs.config.outputs.environment }}
    permissions:
      packages: read
      contents: read
      issues: read
      checks: write
      pull-requests: write
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.test-matrix-parallel) }}
    env:
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # Pull public images without rate limits
      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Check out source
        uses: actions/checkout@v6

      - name: Pull repos
        run: |
          docker compose pull --quiet --include-deps test
          docker images --no-trunc

      - name: Start services
        run: docker compose up --detach test

      - name: Show docker logs
        if: failure()
        run: |
          docker compose logs --timestamps postgres
          # docker compose logs --timestamps kafka
          # docker compose logs --timestamps zookeeper
          docker compose logs --timestamps test
          docker compose ps --format json | jq .

      - name: Display health check results
        if: failure()
        run: |
          echo postgres
          docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q postgres)" | jq
          # echo redis
          # docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q redis)" | jq
          # echo kafka
          # docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q kafka)" | jq
          # echo zookeeper
          # docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q zookeeper)" | jq

      - name: Debug env
        if: failure()
        run: docker compose run test env

      - name: Initialize test database
        run: docker compose run test mix 'do' ecto.setup

      - name: Run tests
        # continue-on-error: true
        # run: docker compose run test mix test --cover
        run: docker compose run test env MIX_TEST_PARTITION=${{ matrix.ci_node_index }} mix test --partitions ${{ matrix.ci_node_total }}
        # run: docker compose run test mix test

      - name: Run quality checks
        # run: docker compose run test mix 'do' format --check-formatted + deps.unlock --check-unused + credo --mute-exit-status --all + hex.audit + deps.audit + sobelow --exit --quiet --skip -i DOS.StringToAtom,Config.HTTPS
        run: docker compose run test mix quality.ci

      - name: Publish unit test results to GitHub
        uses: EnricoMi/publish-unit-test-result-action@v2
        # Run even if tests fail
        if: (!cancelled())
        with:
          # Volume mounted from local filesystem into build by docker compose
          files: junit-reports/*.xml

      - name: Upload test results to Datadog
        if: (needs.config.outputs.test-datadog == '1' && !cancelled())
        continue-on-error: true
        env:
          DD_API_KEY: ${{ secrets.ACTIONS_DD_API_KEY }}
          DD_ENV: ci
          DD_TAGS: "environment:ci"
        run: |
          npm install -g @datadog/datadog-ci
          datadog-ci junit upload --service api-graphql junit-reports/

  test-dialyzer:
    name: Run dialyzer
    if: needs.config.outputs.test-dialyzer == '1'
    needs: [build-test, config]
    environment: ${{ needs.config.outputs.environment }}
    permissions:
      contents: read
      packages: read
      checks: write
      pull-requests: write
      issues: read
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.test-matrix) }}
    env:
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Check out source
        uses: actions/checkout@v6

      - name: Pull repos
        run: |
          docker compose pull --quiet test

      - name: Run dialyzer
        run: docker compose run test mix dialyzer --no-check --quiet-with-result --format github
        # run: docker compose run test mix dialyzer --no-check --ignore-exit-status --format github

  test-scan:
    name: Security scan code
    needs: [build-test, config]
    if: needs.config.outputs.scan-code == '1'
    environment: ${{ needs.config.outputs.environment }}
    permissions:
      contents: read
      packages: read
      checks: write
      pull-requests: write
      issues: read
      # Upload SARIF report files
      security-events: write
    runs-on: ubuntu-latest
    env:
      GITHUB_ADVANCED_SECURITY: ${{ needs.config.outputs.github-advanced-security }}
      GITLEAKS: ${{ needs.config.outputs.scan-gitleaks }}
      GRYPE: ${{ needs.config.outputs.scan-grype }}
      TRIVY: ${{ needs.config.outputs.scan-trivy }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Checkout source
        uses: actions/checkout@v6
        # with:
        #   # Fetch all history for all branches and tags for Gitleaks
        #   fetch-depth: 0

      - name: Manual Trivy Setup
        if: env.TRIVY == '1'
        uses: aquasecurity/setup-trivy@v0.3.1
        with:
          cache: true
          version: v0.68.2

      - name: Scan code with trivy
        if: env.TRIVY == '1'
        uses: aquasecurity/trivy-action@0.69.3
        # https://github.com/marketplace/actions/aqua-security-trivy
        with:
          scan-type: 'fs'
          scan-ref: '.'
          ignore-unfixed: true
          severity: 'CRITICAL'
          # trivy-config: trivy.yaml
          format: ${{ env.GITHUB_ADVANCED_SECURITY == '1' && 'sarif' || 'table' }}
          # output: ${{ env.GITHUB_ADVANCED_SECURITY == '1' && 'trivy-results.sarif' }}
          skip-setup-trivy: true

      - name: Display scan results
        if: ${{ env.TRIVY == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        run: cat trivy-results.sarif | jq .

      - name: Upload trivy scan results to GitHub Security tab
        if: ${{ env.TRIVY == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        uses: github/codeql-action/upload-sarif@v4
        with:
          sarif_file: 'trivy-results.sarif'
          category: trivy

      - name: Scan code with Grype
        if: env.GRYPE == '1'
        uses: anchore/scan-action@v7
        # https://github.com/marketplace/actions/anchore-container-scan
        id: scan-grype
        with:
          path: .
          output-format: table
          # output-format: 'sarif'
          # output-format: ${{ env.GITHUB_ADVANCED_SECURITY == '1' && 'sarif' || 'table' }}
          fail-build: false
          # severity-cutoff: critical

      - name: Display scan results
        if: ${{ env.GRYPE == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        run: cat ${{ steps.scan-grype.outputs.sarif }} | jq .

      - name: Upload grype scan results to GitHub Security tab
        if: ${{ env.GRYPE == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        uses: github/codeql-action/upload-sarif@v4
        with:
          sarif_file: ${{ steps.scan-grype.outputs.sarif }}
          category: grype

      - name: Scan with Gitleaks
        if: env.GITLEAKS == '1'
        uses: gitleaks/gitleaks-action@v3
        env:
          GITHUB_TOKEN: ${{ github.token }}
          GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} # Only required for Organizations, not personal accounts.
          # upload sarif artifact when secrets are detected
          GITLEAKS_ENABLE_UPLOAD_ARTIFACT: true

  build-prod:
    name: Build prod image
    needs: [sync-images-matrix, create-release, config]
    permissions:
      id-token: write
      contents: write
      packages: write
    runs-on: ubuntu-latest
    timeout-minutes: 30
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.prod-matrix) }}
    env:
      AWS_ENABLED: ${{ needs.config.outputs.aws-enabled }}
      DEPLOY_ASSETS_S3: ${{ needs.config.outputs.deploy-assets-s3 }}
      DEPLOY_ECR: ${{ needs.config.outputs.deploy-ecr }}
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      PACKAGE_ANSIBLE: ${{ needs.config.outputs.package-ansible }}
      PACKAGE_CODEDEPLOY: ${{ needs.config.outputs.package-codedeploy }}
      PACKAGE_RELEASE: ${{ needs.config.outputs.package-erlang-release }}
      RELEASE_TAG: ${{ needs.create-release.outputs.release-tag }}
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
    steps:
      - name: Debug environment
        run: |
          echo "github.event_name: ${{ github.event_name }}"
          echo "ref: ${{ github.ref }}"
          echo "ref_name: ${{ github.ref_name }}"
          echo "environment: ${{ needs.config.outputs.environment }}"

      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Configure AWS credentials
        if: env.AWS_ENABLED == '1'
        uses: aws-actions/configure-aws-credentials@v6
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Log in to Amazon ECR
        if: env.DEPLOY_ECR == '1'
        id: ecr-login
        uses: aws-actions/amazon-ecr-login@v2
        # with:
        #   registries: ${{ (needs.config.outputs.environment == 'qa') && env.AWS_ACCOUNT_STAGING || vars.AWS_ACCOUNT }}

      - name: Set vars
        if: env.DEPLOY_ECR == '1'
        run: echo "ECR_REGISTRY=${{ steps.ecr-login.outputs.registry }}" >> "$GITHUB_ENV"

      - name: Check out source
        uses: actions/checkout@v6

      - name: Set variables
        id: vars
        shell: bash
        run: |
          echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> "$GITHUB_ENV"
          echo "run_id=${GITHUB_RUN_ID}" >> "$GITHUB_OUTPUT"
          echo "run_num=${GITHUB_RUN_NUMBER}" >> "$GITHUB_OUTPUT"
          echo "NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_ENV"

      - name: Get app version
        run: |
          APP_VERSION=$(cat mix.exs | grep version: | awk -F'"' '{print $2}')
          echo "APP_VERSION=$APP_VERSION" >> "$GITHUB_ENV"

      # - name: Configure ssh keys
      #   uses: webfactory/ssh-agent@v0.7.0
      #   # https://github.com/marketplace/actions/webfactory-ssh-agent
      #   # https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys
      #   # ssh-keygen -t ed25519 -m pem -C "git@github.com:reachfh/api-utils.git" -f api-utils
      #   with:
      #     ssh-private-key: |
      #       ${{ secrets.SSH_PRIVATE_KEY }}

      # Build assets on host and copy into container
      # Doing it outside of the prod image speeds up the build, as we don't
      # have to copy big static files around.
      # We do it here instead of a separate task, as we need
      # priv/static/cache_manifest.json in the container.

      - uses: actions/setup-node@v6
        with:
          node-version: 24

      - name: Enable corepack
        run: corepack enable

      - name: Install Erlang and Elixir
        uses: erlef/setup-beam@v1
        with:
          otp-version: ${{ matrix.otp }}
          elixir-version: ${{ matrix.elixir }}

      - name: Install tools
        run: mix 'do' local.rebar --force + local.hex --force

      - name: Cache deps
        uses: actions/cache@v6
        env:
          cache-name: cache-deps-${{ github.ref_name }}
        with:
          path: deps
          key: |
            ${{env.cache-name}}-${{runner.os}}-${{matrix.otp}}-${{matrix.elixir}}-${{ hashFiles('**/mix.lock') }}
          restore-keys: |
            ${{env.cache-name}}-${{runner.os}}-${{matrix.otp}}-${{matrix.elixir}}-

      - name: Cache hex
        uses: actions/cache@v6
        env:
          cache-name: cache-deps-${{ github.ref_name }}
        with:
          path: ~/.hex
          key: |
            ${{env.cache-name}}-${{runner.os}}-${{matrix.otp}}-${{matrix.elixir}}-${{ hashFiles('**/mix.lock') }}
          restore-keys: |
            ${{env.cache-name}}-${{runner.os}}-${{matrix.otp}}-${{matrix.elixir}}-

      - name: Cache compiled build
        uses: actions/cache@v6
        env:
          cache-name: cache-build-${{ github.ref_name }}
        with:
          path: _build
            # ${{runner.os}}-mix-${{env.cache-name}}-${{matrix.otp}}-${{matrix.elixir}}-${{hashFiles('**/mix.lock')}}-${{hashFiles( '**/lib/**/*.{ex,eex}', '**/config/*.exs', '**/mix.exs')}}
          key: |
            ${{env.cache-name}}-${{runner.os}}-${{matrix.otp}}-${{matrix.elixir}}-${{hashFiles('**/mix.lock')}}
          restore-keys: |
            ${{env.cache-name}}-${{runner.os}}-${{matrix.otp}}-${{matrix.elixir}}-

      - name: Cache node-modules
        uses: actions/cache@v6
        env:
          cache-name: node-modules-${{ github.ref_name }}
        with:
          path: assets/node_modules
          key: |
            ${{env.cache-name}}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{env.cache-name}}-

      # Force full recompile on builds that are retried.
      - name: Clean deps to avoid flaky incremental builds
        if: github.run_attempt != '1'
        run: |
          mix deps.clean --all
          mix clean

      - name: Get Elixir deps
        run: mix deps.get

      - name: Compile deps
        run: |
          # https://github.com/elixir-lang/elixir/blob/v1.19/CHANGELOG.md#parallel-compilation-of-dependencies
          export MIX_OS_DEPS_COMPILE_PARTITION_COUNT=$((($(getconf _NPROCESSORS_ONLN) + 1) / 2)) ; \
          # `mix loadpaths` is better than `mix deps.compile` when dependencies are already fully compiled
          # https://elixirforum.com/t/github-action-cache-elixir-always-recompiles-dependencies-elixir-1-13-3/45994/12
          # mix deps.compile
          mix loadpaths

      # Asset building doesn't require compiling the app
      # - name: Compile
      #   # run: mix compile --warnings-as-errors
      #   run: mix compile

      - name: Setup
        run: |
          mkdir -p ./assets
          mix assets.setup

      # - name: npm install
      #   working-directory: assets
      #   run: npm install

      # - name: npm run deploy
      #   working-directory: assets
      #   run: npm run deploy

      # - name: yarn install
      #   working-directory: assets
      #   run: yarn install --prod

      # - name: Build digested assets
      #   run: mix phx.digest

      - name: Build and deploy assets
        run: mix assets.deploy

      - name: Copy assets to S3
        if: env.DEPLOY_ASSETS_S3 == '1'
        # Copying is faster than syncing. With digested files, multiple versions
        # can coexist on the server, and the app will choose the right ones.
        run: aws s3 cp --recursive priv/static "s3://${{ vars.S3_BUCKET_ASSETS }}/"

      # - name: Sync assets to S3
      #   if: env.DEPLOY_ASSETS_S3 == '1'
      #   working-directory: app
      #   run: aws s3 sync priv/static "s3://${{ vars.S3_BUCKET_ASSETS }}" --delete

      # We don't have to invalidate when using digested files.
      #   if: env.DEPLOY_ASSETS_CLOUDFRONT == '1'
      # - name: Create CloudFront invalidation
      #   run: aws cloudfront create-invalidation --distribution-id "${{ secrets.CLOUDFRONT_CDN_DISTRIBUTION_ID }}" --paths "/*"

      - name: Set up Docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v4
        with:
          driver-opts: network=host

      - name: Build image and push
        uses: docker/build-push-action@v7
        with:
          file: ${{ env.DOCKER_FILE }}
          target: prod
          build-args: |
            ELIXIR_VER=${{ matrix.elixir }}
            OTP_VER=${{ matrix.otp }}
            BUILD_OS_VER=${{ matrix.build_os_ver }}
            PROD_OS_VER=${{ matrix.prod_os_ver }}
            BUILD_NUM=${{ github.run_number }}
            PUBLIC_REGISTRY=${{ env.PUBLIC_REGISTRY }}
            REPO_ORG_ELIXIR=${{ env.REPO_ORG_ELIXIR }}
            REPO_ORG_PROD_OS=${{ env.REPO_ORG }}/
          context: .
          builder: ${{ steps.buildx.outputs.name }}
          push: true
          cache-from: |
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-prod${{ env.VAR }}-${{ env.MAIN_BRANCH }}
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-prod${{ env.VAR }}-${{ github.ref_name }}
          cache-to: |
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-prod${{ env.VAR }}-${{ github.ref_name }},mode=max
          no-cache: ${{ github.run_attempt != '1' }}
          # ssh: default
          # https://github.com/opencontainers/image-spec/blob/main/annotations.md
          # https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
          labels: |
            org.opencontainers.image.source=https://github.com/${{ github.repository }}
            org.opencontainers.image.created=${{ env.NOW }}
            org.opencontainers.image.revision=${{ github.sha }}
            org.opencontainers.image.version=${{ github.run_number }}
            app.kubernetes.io/name=${{ env.IMAGE_NAME }}
            app.kubernetes.io/part-of=${{ github.repository }}
            app.kubernetes.io/version=${{ github.run_number }}
          tags: |
            ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
            ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:${{ env.VAR }}${{ env.IMAGE_VER }}
            ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:build-${{ github.run_number }}
            ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:version-${{ env.APP_VERSION }}
            ${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:git-${{ github.sha }}
            ${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:${{ github.sha }}
            ${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:build-${{ github.run_number }}
            ${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:version-${{ env.APP_VERSION }}
            ${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:${{ env.VAR }}${{ env.IMAGE_VER }}

        # docker.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:${{ env.VAR }}${{ env.IMAGE_VER }}
        # docker.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }}-${{ env.GITHUB_SHA_SHORT }}
        # secrets: |
        #   "access_token=${{ secrets.DEVOPS_ACCESS_TOKEN }}"
        #   "oban_key_fingerprint=${{ secrets.OBAN_KEY_FINGERPRINT }}"
        #   "oban_license_key=${{ secrets.OBAN_LICENSE_KEY }}"

      - name: Build package image
        if: ${{ needs.config.outputs.package == '1' }}
        uses: docker/build-push-action@v7
        with:
          file: ${{ env.DOCKER_FILE }}
          target: package
          build-args: |
            ELIXIR_VER=${{ matrix.elixir }}
            OTP_VER=${{ matrix.otp }}
            BUILD_OS_VER=${{ matrix.build_os_ver }}
            PROD_OS_VER=${{ matrix.prod_os_ver }}
            BUILD_NUM=${{ github.run_number }}
            PUBLIC_REGISTRY=${{ env.PUBLIC_REGISTRY }}
            REPO_ORG_ELIXIR=${{ env.REPO_ORG_ELIXIR }}
            REPO_ORG_PROD_OS=${{ env.REPO_ORG }}/
          context: .
          builder: ${{ steps.buildx.outputs.name }}
          push: false
          outputs: type=local,dest=out
          cache-from: |
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-prod${{ env.VAR }}-${{ env.MAIN_BRANCH }}
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-prod${{ env.VAR }}-${{ github.ref_name }}
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-package${{ env.VAR }}-${{ env.MAIN_BRANCH }}
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-package${{ env.VAR }}-${{ github.ref_name }}
          cache-to: |
            type=registry,ref=ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:buildcache-package${{ env.VAR }}-${{ github.ref_name }},mode=max
          no-cache: ${{ github.run_attempt != '1' }}

      - name: List package output
        if: ${{ needs.config.outputs.package == '1' }}
        run: |
          ls -l out

      - name: Copy Erlang release
        if: env.PACKAGE_RELEASE == '1'
        env:
          GH_TOKEN: ${{ github.token }}
          SRC: erlang-release.tar.gz
          DST: erlang-release-${{ github.run_number }}-${{ env.VAR }}.tar.gz
        run: |
          mv -v "out/${SRC}" "out/${DST}"
          gh release upload "${RELEASE_TAG}" "out/${DST}"

      - name: Copy CodeDeploy release
        if: env.PACKAGE_CODEDEPLOY == '1'
        env:
          GH_TOKEN: ${{ github.token }}
          SRC: revision.zip
          DST: "codedeploy-revision-${{ github.run_number }}-${{ env.VAR }}.zip"
        run: |
          mv -v "out/${SRC}" "out/${DST}"
          gh release upload "${RELEASE_TAG}" "out/${DST}"

      - name: Copy Ansible release
        if: env.PACKAGE_ANSIBLE == '1'
        env:
          GH_TOKEN: ${{ github.token }}
          SRC: ansible.zip
          DST: "ansible-release-${{ github.run_number }}-${{ env.VAR }}.zip"
        run: |
          mv -v "out/${SRC}" "out/${DST}"
          gh release upload "${RELEASE_TAG}" "out/${DST}"

  test-prod:
    name: Run external API tests
    needs: [build-prod, config]
    permissions:
      id-token: write
      packages: read
      contents: read
      issues: read
      checks: write
      pull-requests: write
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.prod-matrix) }}
    env:
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      # GH_TOKEN: ${{ github.token }}
      IMAGE_VER: ${{ github.sha }}
      TEST_DATA: '0'
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Check out source
        uses: actions/checkout@v6

      - name: Configure AWS credentials
        if: env.TEST_DATA == '1'
        uses: aws-actions/configure-aws-credentials@v6
        # https://github.com/aws-actions/configure-aws-credentials
        # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Get test data files
        if: env.TEST_DATA == '1'
        run: |
          mkdir -p test-data
          aws s3 cp "s3://${{ vars.S3_BUCKET_CI }}/test-data.zip" test-data.zip
          unzip -o test-data.zip -d test-data

      # Create dirs mounted into runtime container by docker compose
      # - name: Create log dir
      #   run: |
      #     mkdir -p log
      #     chmod 775 log

      # Log into AWS to avoid rate limits on public ECR images
      # - name: Configure AWS credentials
      #   if: env.AWS_ENABLED == '1'
      #   uses: aws-actions/configure-aws-credentials@v6
      #   with:
      #     role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
      #     aws-region: ${{ vars.AWS_REGION }}

      # - name: Log in to Amazon ECR
      #   if: env.AWS_ENABLED == '1'
      #   id: ecr-login
      #   uses: aws-actions/amazon-ecr-login@v2

      - name: Pull repos
        # Pull images separately to make timing clearer
        # run: echo "newman postgres router prod" | xargs -n 1 -P 8 docker compose pull --quiet
        # docker compose pull --quiet newman
        # docker images --no-trunc
        run: |
          docker compose pull --quiet --include-deps prod

      - name: Start services
        env:
          LOG_CONSOLE: true
          LOG_LEVEL: debug
        run: |
          env | sort
          docker compose up --wait --wait-timeout 60 prod

      - name: Display logs
        if: failure()
        run: |
          echo "ps"
          docker compose ps --format json | jq .
          echo "postgres logs"
          docker compose logs --timestamps postgres
          # echo "kafka logs"
          # docker compose logs --timestamps kafka
          # echo "zookeeper logs"
          # docker compose logs --timestamps zookeeper
          echo "prod logs"
          docker compose logs --timestamps prod
          echo "postgres health"
          docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q postgres)" | jq
          # echo "kafka health"
          # docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q kafka)" | jq
          # echo "zookeeper health"
          # docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q zookeeper)" | jq
          echo "prod health"
          docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q prod)" | jq

      # - name: Debug env
      #   if: failure()
      #   run: |
      #     docker compose run --entrypoint /bin/bash prod env
      #     docker compose run --entrypoint /bin/bash prod env PGPASSWORD=postgres /usr/bin/psql -w -h postgres -U postgres -d postgres -c "SELECT 1"
      #     # docker compose run --entrypoint /bin/bash prod pg_isready -h postgres -p 5432 -d postgres -U postgres

      # - name: Initialize database
      #   run: |
      #     # Redundant with bin/start-docker
      #     # docker compose run prod eval '${{ env.ELIXIR_MODULE }}.Release.create_repos()'
      #     # docker compose run prod eval '${{ env.ELIXIR_MODULE }}.Release.migrate()'
      #     docker compose run --entrypoint bin/prod prod eval '${{ env.ELIXIR_MODULE }}.Release.run_seeds()'

      # Basic smoke test to make sure service came up
      - name: Run health check
        run: for _i in 1 2 3 4 5 6 7 8 9 10; do curl -v "http://localhost:${APP_PORT}/healthz" && break || sleep 1; done
        # run: for _i in 1 2 3 4 5 6 7 8 9 10; do curl -v "http://localhost:9111/metrics" && break || sleep 1; done

      - name: Display logs
        if: failure()
        run: |
          echo "postgres logs"
          docker compose logs --timestamps postgres
          echo "postgres health"
          docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q postgres)" | jq
          # echo "kafka logs"
          # docker compose logs --timestamps kafka
          # echo "kafka health"
          # docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q kafka)" | jq
          # echo "zookeeper logs"
          # docker compose logs --timestamps zookeeper
          # echo "zookeeper health"
          # docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q zookeeper)" | jq
          echo "prod logs"
          docker compose logs --timestamps prod
          echo "prod health"
          docker inspect --format "{{json .State.Health }}" "$(docker compose ps -q prod)" | jq
          echo "ps"
          docker compose ps --format json | jq .

      # - name: Run API tests
      #   env:
      #     NEWMAN_ARGS: "--verbose -r cli,junitfull --reporter-junitfull-export /junit-reports/newman.xml -e ci.json"
      #     # NEWMAN_ARGS: "--verbose -r cli,junit --reporter-junit-export /junit-reports/newman.xml -e ci.json"
      #   # https://hub.docker.com/r/postman/newman/
      #   run: |
      #     mkdir -p junit-reports
      #     docker compose run newman run ${{env.NEWMAN_ARGS}} Queries.postman_collection.json

      # - name: Display logs
      #   if: failure()
      #   run: |
      #     echo "postgres logs"
      #     docker compose logs --timestamps postgres
      #     echo "postgres health"
      #     docker inspect --format "{{json .State.Health }}" $(docker compose ps -q postgres) | jq
      #     echo "prod logs"
      #     docker compose logs --timestamps prod
      #     echo "prod health"
      #     docker inspect --format "{{json .State.Health }}" $(docker compose ps -q prod) | jq
      #     echo "ps"
      #     docker compose ps --format json | jq .
      #     docker compose logs --timestamps newman

      # - name: Publish unit test results to GitHub
      #   uses: EnricoMi/publish-unit-test-result-action@v2
      #   # Run even if tests fail
      #   if: (!cancelled())
      #   with:
      #     # Volume mounted from local filesystem into build
      #     junit_files: ./junit-reports/*.xml
      #     check_name: "External API Tests"

  scan:
    name: Security scan prod image
    if: needs.config.outputs.scan-prod-image == '1'
    needs: [build-prod, config]
    permissions:
      id-token: write
      contents: read
      packages: read
      checks: write
      pull-requests: write
      issues: read
      # Upload SARIF report files
      security-events: write
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.prod-matrix) }}
    env:
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
      GITHUB_ADVANCED_SECURITY: ${{ needs.config.outputs.github-advanced-security }}
      TRIVY: ${{ needs.config.outputs.scan-trivy }}
      GRYPE: ${{ needs.config.outputs.scan-grype }}
      SNYK: ${{ needs.config.outputs.scan-snyk }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Pull image
        run: docker pull "ghcr.io/${IMAGE_OWNER}/${IMAGE_NAME}:${VAR}${IMAGE_VER}"

      - name: Scan image with Trivy
        if: env.TRIVY == '1'
        uses: aquasecurity/trivy-action@0.69.3
        # https://github.com/aquasecurity/trivy-action
        # https://github.com/marketplace/actions/aqua-security-trivy#inputs
        with:
          image-ref: ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:${{ env.VAR }}${{ env.IMAGE_VER }}
          # exit-code: '1' # fail build
          # ignore-unfixed: true
          # vuln-type: 'os,library'
          # severity: 'CRITICAL,HIGH'
          # cache-dir: /var/cache
          format: ${{ env.GITHUB_ADVANCED_SECURITY == '1' && 'sarif' || 'table' }}
          # output: ${{ env.GITHUB_ADVANCED_SECURITY == '1' && 'trivy-results.sarif' }}

      - name: Display scan results
        if: ${{ env.TRIVY == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        run: cat trivy-results.sarif | jq .

      - name: Upload Trivy scan results to GitHub Security tab
        if: ${{ env.TRIVY == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        uses: github/codeql-action/upload-sarif@v4
        # Requires GitHub Advanced Security
        # https://docs.github.com/en/get-started/learning-about-github/about-github-advanced-security
        # https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning
        # https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github
        with:
          sarif_file: 'trivy-results.sarif'
          category: trivy

      - name: Scan image with Grype
        if: env.GRYPE == '1'
        uses: anchore/scan-action@v7
        # https://github.com/marketplace/actions/anchore-container-scan
        id: scan-grype
        with:
          image: ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:${{ env.VAR }}${{ env.IMAGE_VER }}
          # severity-cutoff: critical
          fail-build: false
          # output-format: 'sarif'
          output-format: table
          # output-format: ${{ env.GITHUB_ADVANCED_SECURITY == '1' && 'sarif' || 'table' }}

      - name: Display scan results
        if: ${{ env.GRYPE == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        run: cat ${{ steps.scan-grype.outputs.sarif }} | jq .

      - name: Upload Grype scan results to GitHub Security tab
        if: ${{ env.GRYPE == '1' && always() && env.GITHUB_ADVANCED_SECURITY == '1' }}
        uses: github/codeql-action/upload-sarif@v4
        with:
          sarif_file: ${{ steps.scan-grype.outputs.sarif }}
          category: grype

      - name: Scan image with Snyk
        if: env.SNYK == '1'
        uses: snyk/actions/docker@master
        continue-on-error: true
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: test
          image: ghcr.io/${{ env.IMAGE_OWNER }}/${{ env.IMAGE_NAME }}:${{ env.VAR }}${{ env.IMAGE_VER }}
          args: --file=${{ env.DOCKER_FILE }} --project-name=api

  deploy-ecs:
    name: Deploy to AWS ECS
    if: needs.config.outputs.deploy-ecs == '1'
    # needs: [test, test-prod, test-dialyzer, upload-assets, config]
    needs: [test, test-prod, config]
    permissions:
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.deploy-matrix) }}
    env:
      AWS_ENABLED: ${{ needs.config.outputs.aws-enabled }}
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      ECS_CLUSTER: foo
      # ECR_REGISTRY_ID: ${{ secrets.AWS_ACCOUNT_ID }}
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
      IMAGE_TAG: ${{ github.sha }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v6
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Check out source
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Set vars for runtime info
        shell: bash
        run: |
          echo "GIT_SHA_SHORT=$(echo $GITHUB_SHA | cut -c 1-7)" >> "$GITHUB_ENV"
          echo "GIT_SHA=${GITHUB_SHA}" >> "$GITHUB_ENV"
          echo "GIT_AUTHOR=$(git log -1 --pretty=format:'%ae')" >> "$GITHUB_ENV"

      - name: Log in to Amazon ECR
        id: ecr-login
        uses: aws-actions/amazon-ecr-login@v2

      - name: Set vars
        run: echo "ECR_REGISTRY=${{ steps.ecr-login.outputs.registry }}" >> "$GITHUB_ENV"
        # echo "ECR_REGISTRY=${{ env.ECR_REGISTRY_ID }}.dkr.ecr.${{ env.ECR_REGISTRY_REGION }}.amazonaws.com" >> "$GITHUB_ENV"

      - name: Generate ECS task-defintion.json
        env:
          APP_CPU: 256
          APP_RAM: 512
          ARCH: "X86_64"
          AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
          AWS_PS_PREFIX: ${{ vars.AWS_PS_PREFIX }}
          AWSLOGS_GROUP: /ecs/foo-app
          AWSLOGS_REGION: ${{ vars.AWS_REGION }}
          AWSLOGS_STREAM_PREFIX: foo-app
          BUILD_NUM: ${{ github.run_number }}
          COMP: "app"
          CONTAINER_NAME: foo-app
          ENV: ${{ needs.config.outputs.environment }}
          EXECUTION_ROLE_ARN: ${{ secrets.EXECUTION_ROLE_ARN}}
          FAMILY: "foo-app"
          GIT_HEAD_REF: ${{ github.head_ref }}
          HTTP_PORT: 4000
          HTTPS_PORT: 4001
          IMAGE_OTEL: "${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}aws-otel-collector:latest"
          IMAGE: "${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:${{ env.IMAGE_VER }}"
          ROLES: "app,web"
          SERVICE: foo-app
          TASK_ROLE_ARN: ${{ secrets.TASK_ROLE_ARN }}
        run: jq --null-input -f "ecs/task-definition.${COMP}.json.jq" | tee "/tmp/task-definition.${COMP}.json"

      - name: Deploy to Amazon ECS
        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
        with:
          task-definition: /tmp/task-definition.app.json
          cluster: foo
          service: foo-app
          # wait-for-service-stability: true

      - name: Generate ECS task-defintion.json (api)
        env:
          APP_CPU: 256
          APP_RAM: 512
          ARCH: "X86_64"
          AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
          AWS_PS_PREFIX: ${{ vars.AWS_PS_PREFIX }}
          AWSLOGS_GROUP: /ecs/foo-api
          AWSLOGS_REGION: ${{ vars.AWS_REGION }}
          AWSLOGS_STREAM_PREFIX: foo-api
          BUILD_NUM: ${{ github.run_number }}
          COMP: "api"
          CONTAINER_NAME: foo-app
          ENV: ${{ needs.config.outputs.environment }}
          EXECUTION_ROLE_ARN: ${{ secrets.EXECUTION_ROLE_ARN}}
          FAMILY: "foo-api"
          GIT_HEAD_REF: ${{ github.head_ref }}
          HTTP_PORT: 4000
          HTTPS_PORT: 4001
          IMAGE_OTEL: "${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}aws-otel-collector:latest"
          IMAGE: "${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:${{ env.IMAGE_VER }}"
          ROLES: "api"
          SERVICE: foo-api
          TASK_ROLE_ARN: ${{ secrets.TASK_ROLE_ARN }}
        run: jq --null-input -f "ecs/task-definition.${COMP}.json.jq" | tee "/tmp/task-definition.${COMP}.json"

      - name: Deploy to Amazon ECS
        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
        with:
          task-definition: /tmp/task-definition.app.json
          cluster: foo
          service: foo-app
          # wait-for-service-stability: true

      - name: Generate ECS task-defintion.json (worker)
        env:
          APP_CPU: 256
          APP_RAM: 512
          ARCH: "X86_64"
          AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
          AWS_PS_PREFIX: ${{ vars.AWS_PS_PREFIX }}
          AWSLOGS_GROUP: /ecs/foo-worker
          AWSLOGS_REGION: ${{ vars.AWS_REGION }}
          AWSLOGS_STREAM_PREFIX: foo-worker
          BUILD_NUM: ${{ github.run_number }}
          COMP: "worker"
          CONTAINER_NAME: foo-app
          ECS_SERVICE: foo-worker
          ENV: ${{ needs.config.outputs.environment }}
          EXECUTION_ROLE_ARN: ${{ secrets.EXECUTION_ROLE_ARN}}
          FAMILY: "foo-worker"
          GIT_HEAD_REF: ${{ github.head_ref }}
          HTTP_PORT: 4000
          HTTPS_PORT: 4001
          IMAGE_OTEL: "${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}aws-otel-collector:latest"
          IMAGE: "${{ env.ECR_REGISTRY }}/${{ env.ECR_IMAGE_OWNER }}${{ env.IMAGE_NAME }}:${{ env.IMAGE_VER }}"
          ROLES: "worker"
          SERVICE: foo-worker
          TASK_ROLE_ARN: ${{ secrets.TASK_ROLE_ARN }}
        run: jq --null-input -f "ecs/task-definition.${COMP}.json.jq" | tee "/tmp/task-definition.${COMP}.json"

      - name: Deploy to Amazon ECS
        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
        with:
          task-definition: /tmp/task-definition.worker.json
          cluster: foo
          service: foo-worker
          # wait-for-service-stability: true

      # - name: Run CodeBuild
      #   uses: aws-actions/aws-codebuild-run-build@v1
      #   with:
      #     project-name: "foo-app"
      #     # buildspec-override: ecs/buildspec-docker.yml
      #     env-vars-for-codebuild: |
      #       custom,
      #       requester,
      #       event-name
      #   env:
      #     custom: my environment variable
      #     requester: ${{ github.actor }}
      #     event-name: ${{ github.event_name }}

      - name: Create Sentry release
        uses: getsentry/action-release@v3
        if: needs.config.outputs.notify-sentry == '1'
        env:
          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
          SENTRY_ORG: ${{ vars.SENTRY_ORG }}
          SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}
          # SENTRY_URL: https://sentry.io/
        with:
          environment: ${{ (github.ref_name == 'main' && 'staging') || (github.ref_name == 'prod' && 'production') }}
          dist: ${{ github.run_number }}
          # ignore_missing: true

      - name: Create AppSignal revision
        if: needs.config.outputs.notify-appsignal == '1'
        env:
          APPSIGNAL_APP_ENV: ${{ (github.ref_name == 'main' && 'stage') || (github.ref_name == 'qa' && 'qa') || (github.ref_name == 'prod' && 'prod') }}
          # APPSIGNAL_APP_ENV: ${{ (github.ref == 'refs/tags/prod' && 'prod') || (github.ref_name == 'main' && 'stage') || 'stage' }}
          APPSIGNAL_APP_NAME: ${{ env.ECS_SERVICE }}
          APPSIGNAL_PUSH_API_KEY: ${{ secrets.APPSIGNAL_PUSH_API_KEY }}
          REVISION: ${{ env.GITHUB_SHA_SHORT }}
          # REVISION: ${{ env.GITHUB_SHA }}
        run: |
          printf '{"revision":"%s","user":"GitHub"}' "$REVISION" | tee revision.json
          APPSIGNAL_URL=$(printf 'https://push.appsignal.com/1/markers?api_key=%s&name=%s&environment=%s' "$APPSIGNAL_PUSH_API_KEY" "$APPSIGNAL_APP_NAME" "$APPSIGNAL_APP_ENV")
          export APPSIGNAL_URL
          # CodeBuild has an insane conflict with YAML syntax and the URL, so put it in a file
          echo "url = \"$APPSIGNAL_URL\"" | tee curl.txt
          curl -g --config curl.txt -d @revision.json

  deploy-codedeploy:
    name: Deploy using CodeDeploy
    if: needs.config.outputs.deploy-codedeploy == '1'
    # needs: [test, test-prod, test-dialyzer, upload-assets, config]
    needs: [test, test-prod, create-release, config]
    permissions:
      id-token: write
      contents: read
      packages: write
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.deploy-matrix) }}
    env:
      CODEDEPLOY_APPLICATION: foo-app
      CODEDEPLOY_BUCKET: foo-prod-bounce-public-deploy
      CODEDEPLOY_DEPLOYMENT_GROUP: foo-app
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      GH_TOKEN: ${{ github.token }}
      RELEASE_TAG: ${{ needs.create-release.outputs.release-tag }}
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v6
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Check out source
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Set vars for runtime info
        shell: bash
        run: |
          echo "GIT_SHA_SHORT=$(echo "$GITHUB_SHA" | cut -c 1-7)" >> "$GITHUB_ENV"
          echo "GIT_SHA=${GITHUB_SHA}" >> "$GITHUB_ENV"
          echo "GIT_AUTHOR=$(git log -1 --pretty=format:'%ae')" >> "$GITHUB_ENV"

      - name: Download release files
        run: |
          gh release download "${RELEASE_TAG}" -p "codedeploy-revision-${{ github.run_number }}-${VAR}.zip"
          mkdir -p revision
          unzip -o "codedeploy-revision-${{ github.run_number }}-${VAR}.zip" -d revision

      - name: Set vars
        # run: echo "REVISION_FILE=${{env.CODEDEPLOY_APPLICATION}}-$(date +'%Y%m%d%H%M%S')-${{ github.run_number }}-${{env.VAR}}.zip" >> $GITHUB_ENV
        # run: echo "REVISION_FILE=${{env.CODEDEPLOY_APPLICATION}}-${{github.event.head_commit.timestamp}}-${{env.GITHUB_SHA_SHORT}}.zip" >> $GITHUB_ENV
        # echo "REVISION_FILE=${CODEDEPLOY_APPLICATION}-$(date +'%Y%m%d%H%M%S')-${RELEASE_TAG}-${VAR}.zip" >> $GITHUB_ENV
        run: |
          echo "RELEASE_TIMESTAMP=$(date +'%Y%m%d%H%M%S')" >> "$GITHUB_ENV"
          echo "REVISION_FILE=${CODEDEPLOY_APPLICATION}-${RELEASE_TAG}-${VAR}.zip" >> "$GITHUB_ENV"
          echo "GIT_AUTHOR_EMAIL=$(git log -1 --pretty=format:'%ae')" >> "$GITHUB_ENV"

      - name: Copy Erlang release file to S3
        env:
          ERLANG_RELEASE_FILE: "erlang-release-${{ github.run_number }}-${{ env.VAR }}.tar.gz"
        run: |
          gh release download "${RELEASE_TAG}" -p "${ERLANG_RELEASE_FILE}"
          aws s3 cp "${ERLANG_RELEASE_FILE}" "s3://${CODEDEPLOY_BUCKET}/erlang-release/${{ github.run_number }}/${VAR}/public.tar.gz"

      - name: Copy Ansible release file to S3
        env:
          # Convert matrix os/version name to name in Ansible inventory
          ANSIBLE_OS: ${{(matrix.os == 'alma' && 'AlmaLinux') || (matrix.os == 'centos' && 'CentOS') || (matrix.os == 'rocky' && 'Rocky') || (matrix.os == 'ubuntu' && 'Ubuntu') }}
          ANSIBLE_OS_VER: ${{ (matrix.os == 'ubuntu' && matrix.prod_os_ver == 'jammy' && '22') || (matrix.os == 'ubuntu' && matrix.prod_os_ver == 'noble' && '24') || (matrix.os != 'ubuntu' && matrix.build_os_ver) || (matrix.os == 'centos7' && '7') }}
          ANSIBLE_RELEASE_FILE: "ansible-release-${{ github.run_number }}-${{ env.VAR }}.zip"
        run: |
          gh release download "${RELEASE_TAG}" -p "${ANSIBLE_RELEASE_FILE}"
          aws s3 cp "${ANSIBLE_RELEASE_FILE}" "s3://${CODEDEPLOY_BUCKET}/ansible-release/${{ github.run_number }}/ansible-release-${ANSIBLE_OS}-${ANSIBLE_OS_VER}.zip"

      - name: Create CodeDeploy revision
        run: |
          aws deploy push --application-name "${CODEDEPLOY_APPLICATION}" \
            --description "App ${RELEASE_TAG} ${RELEASE_TIMESTAMP} ${GITHUB_SHA_SHORT}" \
            --s3-location "s3://${CODEDEPLOY_BUCKET}/${REVISION_FILE}" \
            --source revision

      - name: Create CodeDeploy deployment
        run: |
          aws deploy create-deployment --application-name "${CODEDEPLOY_APPLICATION}" \
            --deployment-group-name "${CODEDEPLOY_DEPLOYMENT_GROUP}-${{ matrix.os }}" \
            --s3-location "bucket=${CODEDEPLOY_BUCKET},key=${REVISION_FILE},bundleType=zip" \
            --description "App ${RELEASE_TAG} ${RELEASE_TIMESTAMP} ${GITHUB_SHA_SHORT}"

      - name: Create Sentry release
        uses: getsentry/action-release@v3
        if: needs.config.outputs.notify-sentry == '1'
        env:
          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
          SENTRY_ORG: ${{ vars.SENTRY_ORG }}
          SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}
          # SENTRY_URL: https://sentry.io/
        with:
          environment: ${{ (github.ref_name == 'main' && 'staging') || (github.ref_name == 'prod' && 'production') }}
          dist: ${{ github.run_number }}
          # ignore_missing: true

      - name: Create AppSignal revision
        if: needs.config.outputs.notify-appsignal == '1'
        env:
          APPSIGNAL_APP_ENV: ${{ (github.ref_name == 'main' && 'stage') || (github.ref_name == 'qa' && 'qa') || (github.ref_name == 'prod' && 'prod') }}
          # APPSIGNAL_APP_ENV: ${{ (github.ref == 'refs/tags/prod' && 'prod') || (github.ref_name == 'main' && 'stage') || 'stage' }}
          APPSIGNAL_APP_NAME: ${{ env.ECS_SERVICE }}
          APPSIGNAL_PUSH_API_KEY: ${{ secrets.APPSIGNAL_PUSH_API_KEY }}
          REVISION: ${{ env.GITHUB_SHA_SHORT }}
          # REVISION: ${{ env.GITHUB_SHA }}
        run: |
          printf '{"revision":"%s","user":"GitHub"}' "$REVISION" | tee revision.json
          APPSIGNAL_URL=$(printf 'https://push.appsignal.com/1/markers?api_key=%s&name=%s&environment=%s' "$APPSIGNAL_PUSH_API_KEY" "$APPSIGNAL_APP_NAME" "$APPSIGNAL_APP_ENV")
          export APPSIGNAL_URL
          # CodeBuild has an insane conflict with YAML syntax and the URL, so put it in a file
          echo "url = \"$APPSIGNAL_URL\"" | tee curl.txt
          curl -g --config curl.txt -d @revision.json

  tag:
    name: Tag images as latest in GHCR
    if: ${{ needs.config.outputs.tag-images == '1' && needs.config.outputs.deploy }}
    needs: [deploy-ecs, config]
    permissions:
      id-token: write
      contents: read
      packages: write
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.deploy-matrix) }}
    env:
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      IMAGE_TAG_GHCR: ${{ needs.config.outputs.image-tag-ghcr }}
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Tag GHCR release as latest
        run: |
          # skopeo copy --multi-arch=index-only \
          skopeo copy \
            "docker://ghcr.io/${IMAGE_OWNER}/${IMAGE_NAME}:${VAR}${IMAGE_VER}" \
            "docker://ghcr.io/${IMAGE_OWNER}/${IMAGE_NAME}:${IMAGE_TAG_GHCR}"

  tag-ecr:
    name: Tag images as latest in ECR
    if: ${{ needs.config.outputs.tag-images == '1' && needs.config.outputs.deploy && needs.config.outputs.deploy-ecr == '1' }}
    needs: [deploy-ecs, config]
    permissions:
      id-token: write
      contents: read
      packages: write
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.deploy-matrix) }}
    env:
      DOCKER_FILE: deploy/${{ matrix.os }}.Dockerfile
      IMAGE_TAG_ECR: ${{ needs.config.outputs.image-tag-ecr }}
      IMAGE_VER: ${{ github.sha }}
      VAR: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v6
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Log in to Amazon ECR
        id: ecr-login
        uses: aws-actions/amazon-ecr-login@v2

      - name: Set vars
        run: echo "ECR_REGISTRY=${{ steps.ecr-login.outputs.registry }}" >> "$GITHUB_ENV"

      - name: Tag ECR release as latest
        run: |
          # aws ecr batch-get-image --repository-name "${ECR_IMAGE_OWNER}${IMAGE_NAME}" --image-ids imageTag="${VAR}${IMAGE_VER}" --output json
          MANIFEST=$(aws ecr batch-get-image --repository-name "${ECR_IMAGE_OWNER}${IMAGE_NAME}" \
            --image-ids imageTag="${VAR}${IMAGE_VER}" \
            --output json | jq --raw-output --join-output '.images[0].imageManifest')
          export MANIFEST
          echo "MANIFEST=$MANIFEST"
          aws ecr put-image --repository-name "${ECR_IMAGE_OWNER}${IMAGE_NAME}" \
            --image-tag "$IMAGE_TAG_ECR" --image-manifest "$MANIFEST"
          # Requires higher permissions
          # aws ecr describe-images --repository-name "${ECR_IMAGE_OWNER}${IMAGE_NAME}"

  # Copy Elixir and OS base images to GHCR
  # Avoids Docker Hub rate limits and speeds up pull performance
  sync-images-matrix:
    name: Sync Elixir and OS base images
    needs: [config]
    if: needs.config.outputs.sync-images == '1'
    permissions:
      id-token: write
      contents: read
      packages: write
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.sync-matrix) }}
    env:
      ELIXIR_BASE_TAG: ${{ matrix.elixir }}-erlang-${{ matrix.otp }}-${{ matrix.os }}-${{ matrix.build_os_ver }}
      MATRIX_OS: ${{ matrix.os }}
      MATRIX_BUILD_OS_VER: ${{ matrix.build_os_ver }}
      MATRIX_PROD_OS_VER: ${{ matrix.prod_os_ver }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Check out source
        uses: actions/checkout@v6

      - name: Install regsync
        uses: regclient/actions/regsync-installer@v0.1.0
        with:
          # optional version
          release: 'v0.11.1'

      - name: Sync matrix images to GHCR
        run: regsync once -c deploy/regsync-ghcr-matrix.yml

  # Copy 3rd-party images to GHCR
  # Avoids Docker Hub rate limits and improves performance
  sync-images-ghcr:
    name: Sync 3rd-party images GHCR
    needs: [config]
    if: needs.config.outputs.sync-images == '1'
    permissions:
      id-token: write
      contents: read
      packages: write
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    steps:
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Check out source
        uses: actions/checkout@v6

      # https://github.com/regclient/actions
      - name: Install regsync
        uses: regclient/actions/regsync-installer@v0.1.0
        with:
          # optional version
          release: 'v0.11.1'

      - name: Sync images to GHCR
        run: regsync once -c deploy/regsync-ghcr.yml

  # Copy 3rd-party images to ECR
  # Avoids Docker Hub rate limits and speeds up pull performance
  # Required if you have 3rd-party images used at runtime
  sync-images-ecr:
    name: Sync 3rd-party images ECR
    needs: [config]
    if: needs.config.outputs.sync-images-ecr == '1'
    permissions:
      id-token: write
      contents: read
      packages: write
    runs-on: ubuntu-latest
    environment: ${{ needs.config.outputs.environment }}
    steps:
      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v6
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Log in to Amazon ECR
        id: ecr-login
        uses: aws-actions/amazon-ecr-login@v2
        with:
          registries: ${{ (needs.config.outputs.environment == 'qa') && env.AWS_ACCOUNT_STAGING || secrets.AWS_ACCOUNT_ID }}

      - name: Set vars
        run: echo "ECR_REGISTRY=${{ steps.ecr-login.outputs.registry }}" >> "$GITHUB_ENV"

      - name: Check out source
        uses: actions/checkout@v6

      - name: Install regsync
        uses: regclient/actions/regsync-installer@v0.1.0
        with:
          # optional version
          release: 'v0.11.1'

      - name: Sync images to ECR
        run: regsync once -c deploy/regsync-ecr.yml

  # To see debug logs, create a secret named ACTIONS_STEP_DEBUG with value 'true'
  # https://github.com/marketplace/actions/slack-notify-build
  # https://github.com/marketplace/actions/post-slack-message