ci.yml

name: Validate Deployment

on:
  push:
    branches:
      - main
      - dev
      - demo
    paths:
      - 'infra/**'
      - 'scripts/**'
      - 'azure.yaml'
      - 'pyproject.toml'
      - 'Makefile'
      - '.github/workflows/ci.yml'
  schedule:
    - cron: '0 8,20 * * *'  # Runs at 8:00 AM and 8:00 PM GMT
  workflow_dispatch:

permissions:
  id-token: write
  contents: read
  packages: write

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}

jobs:
  check-changes:
    runs-on: ubuntu-latest
    outputs:
      should_deploy: ${{ steps.filter.outputs.deploy_related }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Check for relevant changes
        id: filter
        uses: dorny/paths-filter@v3
        with:
          filters: |
            deploy_related:
              - 'infra/**'
              - 'scripts/**'
              - 'azure.yaml'
              - 'pyproject.toml'
              - 'Makefile'
              - '.github/workflows/ci.yml'

      - name: Skip - No Relevant Changes
        if: steps.filter.outputs.deploy_related != 'true' && github.event_name != 'workflow_dispatch' && github.event_name != 'schedule'
        run: echo "No relevant changes detected. Skipping deployment validation."

  deploy:
    needs: check-changes
    if: needs.check-changes.outputs.should_deploy == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
    runs-on: ubuntu-latest
    environment: production
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      PRINCIPAL_ID: ${{ secrets.PRINCIPAL_ID }}
      PRINCIPAL_NAME: ${{ secrets.PRINCIPAL_NAME }}
      PRINCIPAL_TYPE: 'ServicePrincipal'

    outputs:
      imageTag: ${{ steps.set-image-tag.outputs.imageTag }}
      web_url: ${{ steps.extract-urls.outputs.web_url }}
      admin_url: ${{ steps.extract-urls.outputs.admin_url }}
      DEPLOYMENT_SUCCESS: ${{ steps.final-status.outputs.DEPLOYMENT_SUCCESS }}
      resource_group: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }}
      solution_suffix: ${{ steps.generate_solution_prefix.outputs.SOLUTION_SUFFIX }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Install azd
        uses: Azure/setup-azd@v2

      - name: Login to Azure
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Run Quota Check
        id: quota-check
        run: |
          export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
          export GPT_MIN_CAPACITY="150"
          export TEXT_EMBEDDING_MIN_CAPACITY="30"
          export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"

          chmod +x scripts/checkquota.sh
          if ! scripts/checkquota.sh; then
            if grep -q "No region with sufficient quota found" scripts/checkquota.sh; then
              echo "QUOTA_FAILED=true" >> $GITHUB_ENV
            fi
            exit 1
          fi

      - name: Send Notification on Quota Failure
        if: env.QUOTA_FAILED == 'true'
        run: |
          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          EMAIL_BODY=$(cat <<EOF
          {
            "subject": "CWYD Pipeline - Quota Check Failed",
            "body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
          }
          EOF
          )

          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
            -H "Content-Type: application/json" \
            -d "$EMAIL_BODY" || echo "Failed to send notification"

      - name: Fail Pipeline if Quota Check Fails
        if: env.QUOTA_FAILED == 'true'
        run: exit 1

      - name: Set Deployment Region
        run: |
            echo "Selected Region: $VALID_REGION"
            echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV

      - name: Generate Resource Group Name
        id: generate_rg_name
        run: |
          echo "Generating a unique resource group name..."
          ACCL_NAME="cwyd"  # Account name as specified
          SHORT_UUID=$(uuidgen | cut -d'-' -f1)
          UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
          echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
          echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}"

      - name: Check and Create Resource Group
        id: check_create_rg
        run: |
          echo "RESOURCE_GROUP: ${{ env.RESOURCE_GROUP_NAME }}"
          set -e
          OWNER_TAG_VALUE="${{ github.actor }}"
          echo "🔍 Checking if resource group '${{ env.RESOURCE_GROUP_NAME }}' exists..."
          rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
          if [ "$rg_exists" = "false" ]; then
            echo "📦 Resource group does not exist. Creating..."
            echo "🏷️  Adding Owner tag: Owner=${OWNER_TAG_VALUE}"
            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} --tags SecurityControl=Ignore CreatedBy="Pipeline" "Owner=${OWNER_TAG_VALUE}" || { echo "Error creating resource group"; exit 1; }
          else
            echo "✅ Resource group already exists."
            echo "🏷️  Adding Owner tag on existing resource group: Owner=${OWNER_TAG_VALUE}"
            az group update --name "${{ env.RESOURCE_GROUP_NAME }}" --set tags.Owner="${OWNER_TAG_VALUE}" --output none || echo "⚠️  Warning: failed to update Owner tag on existing resource group '${{ env.RESOURCE_GROUP_NAME }}'."
          fi
          # Set output for other jobs
          echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT


      - name: Generate Unique Solution Prefix
        id: generate_solution_prefix
        run: |
          set -e
          COMMON_PART="pslc"
          TIMESTAMP=$(date +%s)
          UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 3)
          UNIQUE_SOLUTION_SUFFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
          echo "SOLUTION_SUFFIX=${UNIQUE_SOLUTION_SUFFIX}" >> $GITHUB_ENV
          echo "SOLUTION_SUFFIX=${UNIQUE_SOLUTION_SUFFIX}" >> $GITHUB_OUTPUT
          echo "Generated SOLUTION_SUFFIX: ${UNIQUE_SOLUTION_SUFFIX}"

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set imageTag
        id: set-image-tag
        run: |
          if [[ "${{ github.event_name }}" == "schedule" ]]; then
            echo "imageTag=latest_waf" >> $GITHUB_ENV
            echo "::set-output name=imageTag::latest_waf"
          elif [[ "${{ github.ref_name }}" == "main" ]]; then
            echo "imageTag=latest_waf" >> $GITHUB_ENV
            echo "::set-output name=imageTag::latest_waf"
          else
            echo "imageTag=${{ github.ref_name }}" >> $GITHUB_ENV
            echo "::set-output name=imageTag::${{ github.ref_name }}"
          fi

      - name: Pre-build image and deploy
        uses: devcontainers/ci@v0.3
        env:
                      AZURE_ENV_NAME: ${{ env.SOLUTION_SUFFIX }}
                      AZURE_LOCATION: ${{ env.AZURE_LOCATION }}
                      AZURE_RESOURCE_GROUP: ${{ env.RESOURCE_GROUP_NAME }}
        with:
                            push: never
                            imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
                            cacheFrom: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
                            imageTag: ${{ env.imageTag }}
                            runCmd: |
                              export optional_args="./code/tests"

                              # Azure login via OIDC federated token
                              OIDC_TOKEN=$(curl -s -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | jq -r '.value')
                              az login --service-principal -u "$AZURE_CLIENT_ID" --tenant "$AZURE_TENANT_ID" --federated-token "$OIDC_TOKEN"
                              az account set --subscription $AZURE_SUBSCRIPTION_ID

                              # Capture deployment output to a log file
                              make ci 2>&1 | tee ci_output.log
                              make deploy 2>&1 | tee deploy_output.log

                              # Try to extract URLs from deployment logs as backup
                              echo "=== Extracting URLs from deployment logs ==="
                              grep -oE "https://[a-zA-Z0-9.-]*\.azurewebsites\.net/" deploy_output.log | sort | uniq | head -2 | tee extracted_urls.txt || echo "No URLs found in logs"

                              # Check if we have admin URL in logs
                              grep -E "Done: Deploying service adminweb" -A 2 deploy_output.log | grep -oE "https://[a-zA-Z0-9.-]*\.azurewebsites\.net/" | head -1 > log_admin_url.txt || echo "" > log_admin_url.txt

                              # Check if we have web URL in logs
                              grep -E "Done: Deploying service web" -A 2 deploy_output.log | grep -oE "https://[a-zA-Z0-9.-]*\.azurewebsites\.net/" | head -1 > log_web_url.txt || echo "" > log_web_url.txt

                              echo "URLs from logs:"
                              echo "Admin URL from logs: $(cat log_admin_url.txt)"
                              echo "Frontend URL from logs: $(cat log_web_url.txt)"
                            env: |
                              AZURE_CLIENT_ID
                              AZURE_TENANT_ID
                              AZURE_SUBSCRIPTION_ID
                              ACTIONS_ID_TOKEN_REQUEST_URL
                              ACTIONS_ID_TOKEN_REQUEST_TOKEN
                              AZURE_ENV_NAME
                              AZURE_LOCATION
                              AZURE_RESOURCE_GROUP
                              AUTH_ENABLED=false
                              AZURE_USE_AUTHENTICATION=false
                              AZURE_ENABLE_AUTH=false
                              FORCE_NO_AUTH=true
                              ENFORCE_AUTH=false
                              REQUIRE_AUTHENTICATION=false
                              AUTHENTICATION_ENABLED=false
                              WEBSITES_AUTH_ENABLED=false
                              WEBSITE_AUTH_ENABLED=false
                              AZURE_AUTH_ENABLED=false
                              ENABLE_AUTHENTICATION=false
                              DISABLE_AUTHENTICATION=true
                              NO_AUTH=true
                              SKIP_AUTH=true
                              PRINCIPAL_ID
                              PRINCIPAL_NAME
                              PRINCIPAL_TYPE

      - name: Extract URLs from deployment
        id: extract-urls
        run: |
                      # Method 1: Check files created by Makefile
                      echo "=== Method 1: Files from Makefile ==="
                      if [ -f "frontend_url.txt" ]; then
                        WEB_URL=$(cat frontend_url.txt | tr -d '\n\r' | xargs)
                        echo "Web URL from makefile: '$WEB_URL'"
                      fi

                      if [ -f "admin_url.txt" ]; then
                        ADMIN_URL=$(cat admin_url.txt | tr -d '\n\r' | xargs)
                        echo "Admin URL from makefile: '$ADMIN_URL'"
                      fi

                      # Method 2: Check URLs extracted from logs
                      echo "=== Method 2: URLs from deployment logs ==="
                      if [ -f "log_web_url.txt" ]; then
                        LOG_WEB_URL=$(cat log_web_url.txt | tr -d '\n\r' | xargs)
                        echo "Web URL from logs: '$LOG_WEB_URL'"
                        if [ -z "$WEB_URL" ] && [ -n "$LOG_WEB_URL" ]; then
                          WEB_URL="$LOG_WEB_URL"
                        fi
                      fi

                      if [ -f "log_admin_url.txt" ]; then
                        LOG_ADMIN_URL=$(cat log_admin_url.txt | tr -d '\n\r' | xargs)
                        echo "Admin URL from logs: '$LOG_ADMIN_URL'"
                        if [ -z "$ADMIN_URL" ] && [ -n "$LOG_ADMIN_URL" ]; then
                          ADMIN_URL="$LOG_ADMIN_URL"
                        fi
                      fi

                      # Set outputs
                      if [ -n "$WEB_URL" ] && [ "$WEB_URL" != "null" ] && [ "$WEB_URL" != "" ]; then
                        echo "web_url=$WEB_URL" >> $GITHUB_OUTPUT
                        echo "FRONTEND_WEBSITE_URL=$WEB_URL" >> $GITHUB_ENV
                        echo "✅ Web URL: $WEB_URL"
                      else
                        echo "web_url=" >> $GITHUB_OUTPUT
                        echo "FRONTEND_WEBSITE_URL=" >> $GITHUB_ENV
                        echo "❌ Frontend URL: Not found"
                      fi

                      if [ -n "$ADMIN_URL" ] && [ "$ADMIN_URL" != "null" ] && [ "$ADMIN_URL" != "" ]; then
                        echo "admin_url=$ADMIN_URL" >> $GITHUB_OUTPUT
                        echo "ADMIN_WEBSITE_URL=$ADMIN_URL" >> $GITHUB_ENV
                        echo "✅ Admin URL: $ADMIN_URL"
                      else
                        echo "admin_url=" >> $GITHUB_OUTPUT
                        echo "ADMIN_WEBSITE_URL=" >> $GITHUB_ENV
                        echo "❌ Admin URL: Not found"
                      fi

                      # Additional debugging
                      echo "=== All Available Files ==="
                      ls -la *.txt *.log *.json 2>/dev/null || echo "No relevant files found"

      - name: Display URLs
        run: |
                      echo "Web URL: ${{ env.FRONTEND_WEBSITE_URL }}"
                      echo "Admin URL: ${{ env.ADMIN_WEBSITE_URL }}"

      - name: Disable Authentication with Script
        run: |
                chmod +x scripts/disable_auth.sh
                ./scripts/disable_auth.sh
        env:
                FRONTEND_WEBSITE_URL: ${{ env.FRONTEND_WEBSITE_URL }}
                ADMIN_WEBSITE_URL: ${{ env.ADMIN_WEBSITE_URL }}

      - name: Export PostgreSQL Host Endpoint from Makefile
        run: |
                # Only extract the PostgreSQL host endpoint from Makefile output
                # Other values are hardcoded in the script
                echo "=== Extracting PostgreSQL Host Endpoint ==="

                if [ -f "pg_host.txt" ] && [ -s "pg_host.txt" ]; then
                  PG_HOST_ENDPOINT=$(cat pg_host.txt | tr -d '\n\r' | xargs)
                  echo "PG_HOST_DESTINATION=$PG_HOST_ENDPOINT" >> $GITHUB_ENV
                  echo "✅ PostgreSQL Host Endpoint: $PG_HOST_ENDPOINT"
                else
                  echo "❌ PostgreSQL host endpoint not found in pg_host.txt"
                  echo "PG_HOST_DESTINATION=localhost" >> $GITHUB_ENV
                  echo "Warning: Using localhost as fallback"
                fi

                echo "=== PostgreSQL Configuration Summary ==="
                echo "Host Endpoint: $PG_HOST_DESTINATION"
                echo "Database: postgres (hardcoded)"
                echo "Port: 5432 (hardcoded)"

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
                python-version: '3.11'

      - name: Install Python dependencies
        run: |
                pip install psycopg2-binary python-dotenv azure-identity

      - name: Populate PostgreSQL Database
        run: |
          python - <<EOF
          import os
          import psycopg2
          from azure.identity import DefaultAzureCredential

          pg_host = os.environ.get("PG_HOST_DESTINATION", "localhost")

          # Acquire Azure AD access token for PostgreSQL via OIDC (Azure CLI credential)
          credential = DefaultAzureCredential()
          token = credential.get_token("https://ossrdbms-aad.database.windows.net/.default").token

          # Get the service principal display name for PostgreSQL user
          import subprocess
          sp_display_name = subprocess.run(
              ["az", "ad", "sp", "show", "--id", os.environ["AZURE_CLIENT_ID"], "--query", "displayName", "-o", "tsv"],
              capture_output=True, text=True
          ).stdout.strip()

          db_params = {
              "user": sp_display_name,   # Use service principal display name
              "password": token,   # Use AAD token instead of password
              "host": pg_host,
              "port": "5432",
              "dbname": "postgres",
              "sslmode": "require"
          }

          csv_file = "exported_data_vector_score.csv"
          target_table = "vector_store"

          try:
              with psycopg2.connect(**db_params) as conn:
                  with conn.cursor() as cur:
                      with open(csv_file, "r", encoding="utf-8") as f:
                          next(f)  # Skip header
                          cur.copy_expert(f"COPY {target_table} FROM STDIN WITH CSV", f)
                  conn.commit()
                  print(f"✅ Imported data from '{csv_file}' into table '{target_table}'.")
          except Exception as e:
              print(f"❌ Error during import: {e}")
          EOF

      - name: Final Status Check
        id: final-status
        run: |
                      echo "=== Final Deployment Status ==="
                      echo "Frontend URL: ${{ env.FRONTEND_WEBSITE_URL }}"
                      echo "Admin URL: ${{ env.ADMIN_WEBSITE_URL }}"
                      echo ""
                      echo "🚀 Deployment completed!"
                      echo "⏰ If you still see authentication errors, wait an additional 10-15 minutes."
                      echo "🔄 Azure authentication changes can take up to 15 minutes to fully propagate."
                      echo ""
                      echo "🌐 Try accessing your applications:"
                      echo "   Frontend: ${{ env.FRONTEND_WEBSITE_URL }}"
                      echo "   Admin: ${{ env.ADMIN_WEBSITE_URL }}"
                      echo "DEPLOYMENT_SUCCESS=true" >> $GITHUB_OUTPUT


      - name: Wait for 25 minutes for propagation
        run: sleep 1500


  e2e-test:
    needs: [check-changes, deploy]
    if: always() && needs.deploy.result == 'success' && needs.deploy.outputs.DEPLOYMENT_SUCCESS == 'true'
    uses: ./.github/workflows/test-automation.yml
    with:
      web_url: ${{ needs.deploy.outputs.web_url }}
      admin_url: ${{ needs.deploy.outputs.admin_url }}


  cleanup:
    if: always() && needs.deploy.result != 'skipped'
    needs: [check-changes, deploy, e2e-test]
    runs-on: ubuntu-latest
    environment: production

    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      AZURE_ENV_NAME: ${{ needs.deploy.outputs.solution_suffix }}  # Get from deploy job
      AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
      imageTag: ${{ needs.deploy.outputs.imageTag }}
      AZURE_RESOURCE_GROUP: ${{ needs.deploy.outputs.resource_group }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Destroy resources
        uses: devcontainers/ci@v0.3
        with:
          push: never
          imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
          imageTag: ${{ env.imageTag }}
          runCmd: |
            OIDC_TOKEN=$(curl -s -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | jq -r '.value')
            az login --service-principal -u "$AZURE_CLIENT_ID" --tenant "$AZURE_TENANT_ID" --federated-token "$OIDC_TOKEN"
            az account set --subscription $AZURE_SUBSCRIPTION_ID
            make destroy
          env: |
            AZURE_CLIENT_ID
            AZURE_TENANT_ID
            AZURE_SUBSCRIPTION_ID
            ACTIONS_ID_TOKEN_REQUEST_URL
            ACTIONS_ID_TOKEN_REQUEST_TOKEN
            AZURE_ENV_NAME
            AZURE_LOCATION
            AZURE_RESOURCE_GROUP

      - name: Send Notification on Failure
        if: failure()
        run: |
          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          EMAIL_BODY=$(cat <<EOF
          {
            "subject": "CWYD Pipeline - Deployment Failed",
            "body": "<p>Dear Team,</p><p>The CWYD Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please investigate.</p><p>Best regards,<br>Your Automation Team</p>"
          }
          EOF
          )

          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
            -H "Content-Type: application/json" \
            -d "$EMAIL_BODY" || echo "Failed to send notification"