env-test.yml

# -----------------------------------------------------------------------------------------
# GitHub Actions Workflow: Environment Test
#
# Description:
# This workflow verifies that a specific version of the application has been correctly
# deployed to a given environment and then runs environment-specific tests against it.
# It can be triggered manually via `workflow_dispatch` or programmatically via
# `repository_dispatch`.
#
# Features:
# - Resolves parameters from either `workflow_dispatch` or `repository_dispatch`.
# - Verifies that the expected Git ref has been deployed to the target environment.
# - Executes a suite of automated environment tests, as defined in `environments.yml`.
# - Uses NSG rule automation to temporarily open SSH access to a jumphost for tunneling.
# - Tags test outcomes (PASS/FAIL/PENDING) using semver-like Git tags.
# - If all tests pass, optionally triggers promotion to the next environment.
#
# Tag format: <version>/CD/<env>/tests/<test-id>/<PASS|FAIL|PENDING>
#
# Inputs:
# - ref: The Git ref (branch, tag, or SHA) to test
# - version: The logical version (e.g. 1.2.3) under test
# - mode: One of 'release', 'ci-test', or 'dry-run' (affects behavior)
# - target_env: The target environment (e.g. dev, qa, prod)
# - private_app_fqdn: FQDN of the deployed internal app
#
# Dependencies:
# - `./devops/environments.yml` must define test specs for each environment.
# - Several reusable workflows and local actions (e.g., `tag-git.yml`, `nsg-port-access`)
# - Assumes presence of secrets and variables like `AZURE_CREDENTIALS`, `GH_PAT`, etc.
# -----------------------------------------------------------------------------------------

name: Environment Test

on:
  workflow_dispatch:
    inputs:
      ref:
        description: 'Git ref to test (e.g., commit SHA, branch, or tag like v1.2.3)'
        required: true
        type: string
      version:
        description: 'Version to test (e.g., 1.2.3)'
        required: true
        type: string
      mode:
        description: 'Mode ("release", "ci-test", or "dry-run")'
        required: true
        default: 'release'
        type: string
      target_env:
        description: 'Target environment (e.g., dev, qa, prod)'
        required: true
        type: string
      private_app_fqdn:
        description: 'Private app FQDN'
        required: true
        type: string
      internal_app_fqdn:
        description: 'Internal app FQDN (if different from private_app_fqdn)'
        required: true
        type: string
  repository_dispatch:
    types: [trigger-env-test]

jobs:

  # -----------------------------------------------
  # Resolve parameters from inputs or client payload
  # -----------------------------------------------
  resolve-params:
    runs-on: ubuntu-latest
    env:
      REF: ${{ github.event.inputs.ref || github.event.client_payload.ref }}
      VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }}
      MODE: ${{ github.event.inputs.mode || github.event.client_payload.mode }}
      TARGET_ENV: ${{ github.event.inputs.target_env || github.event.client_payload.target_env }}
      PRIVATE_APP_FQDN: ${{ github.event.inputs.private_app_fqdn || github.event.client_payload.private_app_fqdn }}
      INTERNAL_APP_FQDN: ${{ github.event.inputs.internal_app_fqdn || github.event.client_payload.internal_app_fqdn }}
    outputs:
      ref: ${{ steps.setvars.outputs.ref }}
      version: ${{ steps.setvars.outputs.version }}
      mode: ${{ steps.setvars.outputs.mode }}
      target_env: ${{ steps.setvars.outputs.target_env }}
      private_app_fqdn: ${{ steps.setvars.outputs.private_app_fqdn }}
      internal_app_fqdn: ${{ github.event.inputs.internal_app_fqdn || github.event.client_payload.internal_app_fqdn }}
    steps:
      - name: Print GitHub event
        env:
          EVENT: ${{ toJson(github.event) }}
        run: echo "$EVENT"

      - name: Print resolved parameters
        run: |
          echo "Resolved parameters:"
          echo "  ref: $REF"
          echo "  version: $VERSION"
          echo "  mode: $MODE"
          echo "  target_env: $TARGET_ENV"
          echo "  private_app_fqdn: $PRIVATE_APP_FQDN"
          echo "  internal_app_fqdn: $INTERNAL_APP_FQDN"

      - name: Validate required parameters
        run: |
          for var in REF VERSION MODE TARGET_ENV PRIVATE_APP_FQDN INTERNAL_APP_FQDN; do
            if [ -z "${!var}" ]; then
              echo "Missing required parameter: $var"
              exit 1
            fi
          done
          
      - name: Validate mode value
        run: |
          case "$MODE" in
            release|ci-test|dry-run)
              echo "Valid mode: $MODE"
              ;;
            *)
              echo "❌ Invalid mode: '$MODE'. Must be one of: release, ci-test, dry-run"
              exit 1
              ;;
          esac    

      - name: Set output variables
        id: setvars
        run: |
          echo "ref=$REF" >> $GITHUB_OUTPUT
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "mode=$MODE" >> $GITHUB_OUTPUT
          echo "target_env=$TARGET_ENV" >> $GITHUB_OUTPUT
          echo "private_app_fqdn=$PRIVATE_APP_FQDN" >> $GITHUB_OUTPUT
          echo "internal_app_fqdn=$INTERNAL_APP_FQDN" >> $GITHUB_OUTPUT

  # -----------------------------------------------
  # Verify that the expected ref is deployed to the target environment
  # -----------------------------------------------
  verify-deployed-version:
    needs: resolve-params
    runs-on: ubuntu-latest
    env:
      TARGET_ENV: ${{ needs.resolve-params.outputs.target_env }}
      EXPECTED_VERSION: ${{ needs.resolve-params.outputs.version }}
      EXPECTED_REF: ${{ needs.resolve-params.outputs.ref }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      REPO: ${{ github.repository }}
      DRY_RUN: ${{ needs.resolve-params.outputs.mode == 'dry-run' }}
    steps:
      - name: Verify deployed version via GitHub API
        if: ${{ env.DRY_RUN != 'true' }}
        run: |
          set -euo pipefail
          echo "Verifying that ref $EXPECTED_REF (version $EXPECTED_VERSION) is deployed to $TARGET_ENV"
          DEPLOYED_REF=$(gh api repos/$REPO/deployments \
            --jq 'map(select(.environment=="'"$TARGET_ENV"'")) | sort_by(.created_at) | last.ref')
          echo "Expected ref: $EXPECTED_REF"
          echo "Deployed ref: $DEPLOYED_REF"
          if [[ "$DEPLOYED_REF" != "$EXPECTED_REF" ]]; then
            echo "❌ ERROR: Deployed ref ($DEPLOYED_REF) does NOT match expected ref ($EXPECTED_REF) in $TARGET_ENV"
            exit 1
          fi
          echo "✅ SUCCESS: Deployed ref matches expected ref."

  # -----------------------------------------------
  # Run environment tests
  # -----------------------------------------------
  test:
    needs: [resolve-params, verify-deployed-version]
    outputs:
      pass_count: ${{ steps.test.outputs.pass_count }}
      fail_count: ${{ steps.test.outputs.fail_count }}
      pending_count: ${{ steps.test.outputs.pending_count }}
    runs-on: ubuntu-latest
    environment: ${{ needs.resolve-params.outputs.target_env }}
    env:
      VERSION: ${{ needs.resolve-params.outputs.version }}
      REF: ${{ needs.resolve-params.outputs.ref }}
      MODE: ${{ needs.resolve-params.outputs.mode }}
      TARGET_ENV: ${{ needs.resolve-params.outputs.target_env }}
      PRIVATE_APP_FQDN: ${{ needs.resolve-params.outputs.private_app_fqdn }}
      INTERNAL_APP_FQDN: ${{ needs.resolve-params.outputs.internal_app_fqdn }}
      JUMPHOST_FQDN: ${{ vars.JUMPHOST_FQDN }}
      AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
    steps:

      - name: Print params
        run: |
          echo "Preparing to run tests for version '$VERSION' in env '$TARGET_ENV'"
          echo "Mode: $MODE"          
          echo "Ref: $REF"
          echo "Private app FQDN: $PRIVATE_APP_FQDN"
          echo "Internal app FQDN: $INTERNAL_APP_FQDN"
          
      - name: Check for existing test result tags
        id: check-test-tags
        if: ${{ env.MODE != 'dry-run' }}
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          BASE_TAG="${VERSION}/CD/${TARGET_ENV}/tests/"
          echo "Checking for existing tags matching: $BASE_TAG*"

          EXISTING=$(gh api \
            repos/${{ github.repository }}/git/matching-refs/tags/${BASE_TAG} \
            --jq '.[] | .ref' || true)

          if [[ -n "$EXISTING" ]]; then
            echo "Found existing test tags:"
            echo "$EXISTING"
            echo "❌ ERROR: Tests already executed for version ${VERSION} in env ${TARGET_ENV}"
            exit 1
          else
            echo "No existing test tags found. Proceeding."
          fi      

      - name: Checkout repository
        if: ${{ env.MODE != 'dry-run' }}
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ env.REF }}

      # Update permissions for scripts
      - name: Update permissions for scripts
        run: find ./devops/scripts -type f -name '*.sh' -exec chmod +x {} +

      - name: Get public IP
        id: get-ip
        run: |
          IP=$(curl -s https://api.ipify.org)
          echo "public_ip=$IP" >> "$GITHUB_OUTPUT"
          echo "Public (runner) IP: $IP"  

      - name: Open port in NSG
        uses: ./.github/actions/nsg-port-access
        if: ${{ env.MODE != 'dry-run' }}
        with:
          azure_nsg_name: ${{ vars.JUMPHOST_NSG_NAME }}
          action: open
          port: 2022
          public_ip: ${{ steps.get-ip.outputs.public_ip }}
          azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
          azure_resource_group: ${{ vars.AZURE_RESOURCE_GROUP }}

      - name: Check port opened
        if: ${{ env.MODE != 'dry-run' }}
        run: |
          source ./devops/scripts/bootstrap.sh
          log INFO "Checking if port 2022 is open on the jumphost ($JUMPHOST_FQDN)..."
          if ! nc -z "$JUMPHOST_FQDN" 2022; then
            log ERROR "Port 2022 is not open on the jumphost!"
            exit 1
          fi
          log INFO "Port 2022 is open on the jumphost."    

      - name: Azure Login
        uses: azure/login@v2
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}

      # Run environment tests defined in ./devops/environments.yml and tag results
      - name: Run environment tests and tag results
        if: ${{ env.MODE != 'dry-run' }}
        id: test
        env:
          GIT_TAG_REF: ${{ github.sha }}
          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
        shell: bash
        run: |
          source ./devops/scripts/bootstrap.sh        
          
          # Start ACA polling of the ssh-relay container to ensure it's relaying the necessary tunnel
          ./devops/scripts/start-aca-polling.sh aca-ssh-relay "$AZURE_RESOURCE_GROUP"
          
          # Create SSH key files from secrets
          JUMPKEY_PATH=~/.ssh/jump-key
          RELAYKEY_PATH=~/.ssh/relay-key
          LOCAL_APP_PORT=8443
          
          mkdir -p ~/.ssh
          
          echo "${{ secrets.JUMPHOST_SSH_PRIVATE_KEY }}" > "$JUMPKEY_PATH"
          echo "${{ secrets.ACA_SSH_RELAY_PRIVATE_KEY }}" > "$RELAYKEY_PATH"
          chmod 600 "$JUMPKEY_PATH" "$RELAYKEY_PATH"
          
          # Start SSH tunnel to the aca-ssh-relay via the jumphost
          ./devops/scripts/start-ssh-tunnel.sh \
            --jump-host $JUMPHOST_FQDN \
            --jump-port 2022 \
            --jump-key "$JUMPKEY_PATH" \
            --relay-key "$RELAYKEY_PATH" \
            --target-hostname $INTERNAL_APP_FQDN \
            --target-port 443 \
            --forward-port $LOCAL_APP_PORT
          
          # Export environment variables for test scripts
          export APP_FQDN=localhost
          export APP_PORT=$LOCAL_APP_PORT
          export APP_INTERNAL_FQDN=$INTERNAL_APP_FQDN

          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"          
          
          PASS=0
          FAIL=0
          PENDING=0

          tag_and_push() {
            local tag="$1"
            local msg="$2"
            local ref="$3"
          
            # Remove local and remote tag if exists
            git tag | grep -qxF "$tag" && git tag -d "$tag" || true
            git push --delete origin "$tag" 2>/dev/null || true
            git tag -a "$tag" -m "$msg" "$ref" || return 0
            git push origin "$tag" || return 0
            return 0
          }        
        
          TESTS=$(yq -o=json '.environments[] | select(.name == env(TARGET_ENV)) | .tests[]' ./devops/environments.yml)
          NUM_TESTS=$(echo "$TESTS" | jq -s 'length')

          log INFO "Found $NUM_TESTS tests for environment '$TARGET_ENV' (version: $VERSION)"
          echo "$TESTS" | jq .

          for idx in $(seq 0 $((NUM_TESTS - 1))); do
            TEST=$(echo "$TESTS" | jq -s ".[$idx]")
            TEST_ID=$(echo "$TEST" | jq -r '.id')
            SCRIPTS=$(echo "$TEST" | jq -r '.scripts[]?')
            FAILED_SCRIPTS=()
            FAILED_CODES=()
          
            log INFO "Processing test $((idx + 1))/$NUM_TESTS: $TEST_ID"
            log INFO "Scripts: $SCRIPTS"
            log INFO "PASS=$PASS, FAIL=$FAIL, PENDING=$PENDING"
            
            if [[ -z "$TEST_ID" ]]; then
              log ERROR "Test ID is empty, skipping this test"
              continue
            fi

            if [[ -z "$SCRIPTS" ]]; then
              TAG="${VERSION}/CD/${TARGET_ENV}/tests/${TEST_ID}/PENDING"
              MSG="Manual test '${TEST_ID}' pending for $VERSION in $TARGET_ENV"
              log INFO "No scripts found for test '$TEST_ID' – tagging as PENDING"
              tag_and_push "$TAG" "$MSG" "$GIT_TAG_REF" || log WARN "tag_and_push failed for $TAG"    
              ((PENDING++)) || true         
              continue
            fi

            for SCRIPT in $SCRIPTS; do
              log INFO "Running script $SCRIPT for test '$TEST_ID'"
              RC=0
              bash "$SCRIPT" || RC=$?
              if [[ $RC -ne 0 ]]; then
                log ERROR "Script $SCRIPT failed for test '$TEST_ID' (rc=$RC)"
                FAILED_SCRIPTS+=("$SCRIPT")
                FAILED_CODES+=("$RC")
              else
                log INFO "Script $SCRIPT succeeded for test '$TEST_ID'"
              fi
            done

            if [[ ${#FAILED_SCRIPTS[@]} -eq 0 ]]; then
              TAG="${VERSION}/CD/${TARGET_ENV}/tests/${TEST_ID}/PASS"
              MSG="Test '${TEST_ID}' passed for $VERSION in $TARGET_ENV"
              log INFO "Tagging $TAG as PASS"
              tag_and_push "$TAG" "$MSG" "$GIT_TAG_REF" || log WARN "tag_and_push failed for $TAG"
              ((PASS++)) || true
            else
              TAG="${VERSION}/CD/${TARGET_ENV}/tests/${TEST_ID}/FAIL"
              FAIL_MSGS=""
              for i in "${!FAILED_SCRIPTS[@]}"; do
                FAIL_MSGS+="Script: ${FAILED_SCRIPTS[$i]} (rc=${FAILED_CODES[$i]}), "
              done

              MSG="Test '${TEST_ID}' failed for $VERSION in $TARGET_ENV: $FAIL_MSGS"
              log ERROR "Tagging $TAG as FAIL with: $FAIL_MSGS"
              tag_and_push "$TAG" "$MSG" "$GIT_TAG_REF" || log WARN "tag_and_push failed for $TAG"              
              ((FAIL++)) || true             
            fi
          done

          log INFO "Summary: PASS=$PASS FAIL=$FAIL PENDING=$PENDING"
          
          # Set output variables for the next steps
          echo "pass_count=$PASS" >> "$GITHUB_OUTPUT"
          echo "fail_count=$FAIL" >> "$GITHUB_OUTPUT"
          echo "pending_count=$PENDING" >> "$GITHUB_OUTPUT"

      - name: Close port in NSG
        uses: ./.github/actions/nsg-port-access
        if: always()
        with:
          azure_nsg_name: ${{ vars.JUMPHOST_NSG_NAME }}
          action: close
          port: 2022
          public_ip: ${{ steps.get-ip.outputs.public_ip }}
          azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
          azure_resource_group: ${{ vars.AZURE_RESOURCE_GROUP }}

  # -----------------------------------------------
  # Tag results based on test outcomes
  # -----------------------------------------------

  tag-test-success:
    needs: [resolve-params, test]
    if: ${{ needs.test.result == 'success' && needs.test.outputs.pending_count == '0' && needs.test.outputs.fail_count == '0' }}
    uses: ./.github/workflows/tag-git.yml
    with:
      tag_name: "${{ needs.resolve-params.outputs.version }}/CD/${{ needs.resolve-params.outputs.target_env }}/tests/PASS"
      ref: ${{ needs.resolve-params.outputs.ref }}
      message: > 
        All (automated) environment tests passed for ${{ needs.resolve-params.outputs.version }} in 
        ${{ needs.resolve-params.outputs.target_env }}
      dry_run: ${{ needs.resolve-params.outputs.mode == 'dry-run' }}

  tag-env-success:
    needs: [resolve-params, test]
    if: ${{ needs.test.result == 'success' && needs.test.outputs.pending_count == '0' && needs.test.outputs.fail_count == '0' }}
    uses: ./.github/workflows/tag-git.yml
    with:
      tag_name: "${{ needs.resolve-params.outputs.version }}/CD/${{ needs.resolve-params.outputs.target_env }}/PASS"
      ref: ${{ needs.resolve-params.outputs.ref }}
      message: >
        All automated environment tests passed for ${{ needs.resolve-params.outputs.version }} in 
        ${{ needs.resolve-params.outputs.target_env }}
      dry_run: ${{ needs.resolve-params.outputs.mode == 'dry-run' }}

  tag-env-test-pending:
    needs: [ resolve-params, test ]
    if: ${{ needs.test.outputs.fail_count == '0' && needs.test.outputs.pending_count != '0' }}
    uses: ./.github/workflows/tag-git.yml
    with:
      tag_name: "${{ needs.resolve-params.outputs.version }}/CD/${{ needs.resolve-params.outputs.target_env }}/tests/PENDING"
      ref: ${{ needs.resolve-params.outputs.ref }}
      message: > 
        Service deployed version ${{ needs.resolve-params.outputs.version }} to environment 
        ${{ needs.resolve-params.outputs.target_env }}, but some tests are still pending
      dry_run: ${{ needs.resolve-params.outputs.mode == 'dry-run' }}

  tag-env-test-fail:
    needs: [resolve-params, test]
    if: ${{ failure() || cancelled() || needs.test.outputs.fail_count != '0' }}
    uses: ./.github/workflows/tag-git.yml
    with:
      tag_name: "${{ needs.resolve-params.outputs.version }}/CD/${{ needs.resolve-params.outputs.target_env }}/tests/FAIL"
      ref: ${{ needs.resolve-params.outputs.ref }}
      message: > 
        Service failed deploy for version ${{ needs.resolve-params.outputs.version }} to environment 
        ${{ needs.resolve-params.outputs.target_env }}
      dry_run: ${{ needs.resolve-params.outputs.mode == 'dry-run' }}

  tag-cd-env-fail:
    needs: [resolve-params, test]
    if: ${{ failure() || cancelled() || needs.test.outputs.fail_count != '0' }}
    uses: ./.github/workflows/tag-git.yml
    with:
      tag_name: "${{ needs.resolve-params.outputs.version }}/CD/${{ needs.resolve-params.outputs.target_env }}/FAIL"
      ref: ${{ needs.resolve-params.outputs.ref }}
      dry_run: ${{ needs.resolve-params.outputs.mode == 'dry-run' }}

  tag-cd-fail:
    needs: [resolve-params, test]
    if: ${{ failure() || cancelled() || needs.test.outputs.fail_count != '0' }}
    uses: ./.github/workflows/tag-git.yml
    with:
      tag_name: "${{ needs.resolve-params.outputs.version }}/CD/FAIL"
      ref: ${{ needs.resolve-params.outputs.ref }}
      dry_run: ${{ needs.resolve-params.outputs.mode == 'dry-run' }}

  # -----------------------------------------------
  # Promote to next environment if all tests passed
  # -----------------------------------------------
  promote-to-next-env:
    needs: [resolve-params, test, tag-env-success, tag-test-success]
    if: ${{ needs.resolve-params.outputs.target_env != '' }}
    runs-on: ubuntu-latest
    env:
        VERSION: ${{ needs.resolve-params.outputs.version }}
        REF: ${{ needs.resolve-params.outputs.ref }}
        MODE: ${{ needs.resolve-params.outputs.mode }}
        SRC_ENV: ${{ needs.resolve-params.outputs.target_env }}
        GITHUB_TOKEN: ${{ secrets.GH_PAT }}
    steps:
      - name: Print promotion parameters
        run: |
          echo "All tests passed. Preparing to promote version '$VERSION' in environment '$SRC_ENV'"
          echo "  Mode: $MODE"
          echo "  Ref: $REF"

      - name: Trigger promotion to next environment
        if: ${{ env.ACT != 'true' }}
        uses: peter-evans/repository-dispatch@v3
        with:
          token: ${{ env.GITHUB_TOKEN }}
          event-type: trigger-promote
          client-payload: >-
            {              
              "version": "${{ env.VERSION }}",
              "ref": "${{ env.REF }}",
              "mode": "${{ env.MODE }}",              
              "src_env": "${{ env.SRC_ENV }}"
            }

      - name: Log dispatch if platform  is 'act'
        if: ${{ env.ACT == 'true' }}
        run: |
          echo "Running in 'act' mode, skipping actual dispatch."
          echo "Promotion parameters:"
          echo "  Version: $VERSION"
          echo "  Ref: $REF"
          echo "  Mode: $MODE"
          echo "  Source Environment: $SRC_ENV"