chore: improve CI/CD workflows, automate dependency updates, and enforce linting standards

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E501
+exclude = .venv, frontend
+ignore = E722,E203, W503, G004, G200, F, E711

.github/dependabot.yml

@@ -0,0 +1,60 @@
+# Dependabot configuration for Container Migration Solution Accelerator
+# This file configures automated dependency updates for all package managers used in the project
+
+version: 2
+updates:
+  # Frontend Node.js dependencies (grouped)
+  - package-ecosystem: "npm"
+    directory: "/src/frontend"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "build"
+    target-branch: "dependabotchanges"
+    open-pull-requests-limit: 50
+    groups:
+      all-npm:
+        patterns:
+          - "*"
+
+  # Backend API Python dependencies (grouped)
+  - package-ecosystem: "pip"
+    directory: "/src/backend-api"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "build"
+    target-branch: "dependabotchanges"
+    open-pull-requests-limit: 50
+    groups:
+      all-pip:
+        patterns:
+          - "*"
+
+  # Processor Python dependencies (grouped)
+  - package-ecosystem: "pip"
+    directory: "/src/processor"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "build"
+    target-branch: "dependabotchanges"
+    open-pull-requests-limit: 50
+    groups:
+      all-pip:
+        patterns:
+          - "*"
+
+  # GitHub Actions dependencies (grouped)
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "build"
+    target-branch: "dependabotchanges"
+    open-pull-requests-limit: 50
+    groups:
+      all-actions:
+        patterns:
+          - "*"

.github/workflows/ci.yml

@@ -0,0 +1,319 @@
+name: Validate Deployment
+on:
+  push:
+    branches:
+      - main
+      - dev
+      - demo
+  schedule:
+    - cron: '0 9,21 * * *'  # Runs at 9:00 AM and 9:00 PM GMT
+  workflow_dispatch:  # Allow manual triggering
+env:
+  GPT_MIN_CAPACITY: 150
+  BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }}
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    outputs:
+      RESOURCE_GROUP_NAME: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }}
+      DEPLOYMENT_SUCCESS: ${{ steps.deployment_status.outputs.SUCCESS }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Setup Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version
+      - name: Login to Azure
+        run: |
+          az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
+      - name: Run Quota Check
+        id: quota-check
+        run: |
+          export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
+          export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
+          export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
+          export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+          export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
+          chmod +x scripts/checkquota.sh
+          if ! scripts/checkquota.sh; then
+            # If quota check fails due to insufficient quota, set the flag
+            if grep -q "No region with sufficient quota found" scripts/quota_check.sh; then
+              echo "QUOTA_FAILED=true" >> $GITHUB_ENV
+            fi
+            exit 1  # Fail the pipeline if any other failure occurs
+          fi
+      - name: Send Notification on Quota Failure
+        if: env.QUOTA_FAILED == 'true'
+        run: |
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          EMAIL_BODY=$(cat <<EOF
+          {
+            "body": "<p>Dear Team,</p><p>The KMGeneric quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a></p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>",
+            "subject": "KMGeneric Deployment - Quota Check Failed"
+          }
+          EOF
+          )
+          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
+            -H "Content-Type: application/json" \
+            -d "$EMAIL_BODY" || echo "Failed to send notification"
+      - name: Fail Pipeline if Quota Check Fails
+        if: env.QUOTA_FAILED == 'true'
+        run: exit 1
+      - name: Install Bicep CLI
+        run: az bicep install
+
+      - name: Set Deployment Region
+        run: |
+          echo "Selected Region: $VALID_REGION"
+          echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
+      - name: Generate Resource Group Name
+        id: generate_rg_name
+        run: |
+          echo "Generating a unique resource group name..."
+
+          ACCL_NAME="CM"
+
+          SHORT_UUID=$(uuidgen | cut -d'-' -f1)
+          UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
+          echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
+          echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}"
+
+      - name: Check and Create Resource Group
+        id: check_create_rg
+        run: |
+          set -e  
+          echo "Checking if resource group exists..."
+          rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
+          if [ "$rg_exists" = "false" ]; then
+            echo "Resource group does not exist. Creating..."
+            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} || { echo "Error creating resource group"; exit 1; }
+          else
+            echo "Resource group already exists."
+          fi
+          echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT
+      - name: Generate Unique Solution Prefix
+        id: generate_solution_prefix
+        run: |
+          set -e
+          COMMON_PART="Cm"
+          TIMESTAMP=$(date +%s)  
+          UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 6) 
+          UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
+          echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV
+          echo "Generated SOLUTION_PREFIX: ${UNIQUE_SOLUTION_PREFIX}" 
+
+      - name: Deploy Bicep Template
+        id: deploy
+        run: |
+          set -e
+          az deployment group create \
+            --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
+            --template-file infra/main.bicep \
+            --parameters solutionName=${{env.SOLUTION_PREFIX}} 
+
+
+      - name: Extract AI Services and Key Vault Names
+        if: always()
+        run: |
+              echo "Fetching AI Services and Key Vault names before deletion..."
+
+              # Get Key Vault name
+              KEYVAULT_NAME=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.KeyVault/vaults" --query "[].name" -o tsv)
+              echo "Detected Key Vault: $KEYVAULT_NAME"
+              echo "KEYVAULT_NAME=$KEYVAULT_NAME" >> $GITHUB_ENV
+
+              # Get AI Services names and convert them into a space-separated string
+              AI_SERVICES=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.CognitiveServices/accounts" --query "[].name" -o tsv | tr '\n' ' ')
+
+              echo "Detected AI Services: $AI_SERVICES"
+              echo "AI_SERVICES=$AI_SERVICES" >> $GITHUB_ENV          
+      - name: Set Deployment Status
+        id: deployment_status
+        if: always()
+        run: |
+          if [ "${{ job.status }}" == "success" ]; then
+            echo "SUCCESS=true" >> $GITHUB_OUTPUT
+          else
+            echo "SUCCESS=false" >> $GITHUB_OUTPUT
+          fi
+      - name: Logout from Azure
+        if: always()
+        run: |
+          az logout
+          echo "Logged out from Azure."
+  cleanup-deployment:
+    if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != ''
+    needs: [deploy]
+    runs-on: ubuntu-latest
+    env:
+      RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
+    steps:
+      - name: Setup Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version
+      - name: Login to Azure
+        run: |
+          az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
+          az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+      - name: Extract AI Services and Key Vault Names
+        if: always()
+        run: |
+          echo "Fetching AI Services and Key Vault names before deletion..."
+
+          # Get Key Vault name
+          KEYVAULT_NAME=$(az resource list --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --resource-type "Microsoft.KeyVault/vaults" --query "[].name" -o tsv)
+          echo "Detected Key Vault: $KEYVAULT_NAME"
+          echo "KEYVAULT_NAME=$KEYVAULT_NAME" >> $GITHUB_ENV
+          # Extract AI Services names
+          echo "Fetching AI Services..."
+          AI_SERVICES=$(az resource list --resource-group '${{ env.RESOURCE_GROUP_NAME }}' --resource-type "Microsoft.CognitiveServices/accounts" --query "[].name" -o tsv)
+          # Flatten newline-separated values to space-separated
+          AI_SERVICES=$(echo "$AI_SERVICES" | paste -sd ' ' -)
+          echo "Detected AI Services: $AI_SERVICES"
+          echo "AI_SERVICES=$AI_SERVICES" >> $GITHUB_ENV
+
+      - name: Delete Bicep Deployment
+        if: always()
+        run: |
+          set -e  
+          echo "Checking if resource group exists..."
+          rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
+          if [ "$rg_exists" = "true" ]; then
+            echo "Resource group exists. Cleaning..."
+            az group delete \
+                --name ${{ env.RESOURCE_GROUP_NAME }} \
+                --yes \
+                --no-wait
+            echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}"
+          else
+            echo "Resource group does not exist."
+          fi
+      - name: Wait for Resource Deletion to Complete
+        if: always()
+        run: |
+          echo "Waiting for all deployed resources (including AI Services) to be deleted..."
+
+          # Convert AI_SERVICES space-separated string into an array
+          IFS=' ' read -r -a resources_to_check <<< "${{ env.AI_SERVICES }}"
+
+          echo "Resources to check for deletion:"
+          printf '%s\n' "${resources_to_check[@]}"
+
+          # Get the current resource list in YAML
+          resource_list=$(az resource list --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" --output yaml)
+
+          # Set up retry logic
+          max_retries=3
+          retry_intervals=(30 60 120)
+          retries=0
+
+          while true; do
+            resource_found=false
+            for resource in "${resources_to_check[@]}"; do
+              echo "Checking if resource '$resource' still exists..."
+              if echo "$resource_list" | grep -q "name: $resource"; then
+                echo "Resource '$resource' still exists."
+                resource_found=true
+              else
+                echo "Resource '$resource' has been deleted."
+              fi
+            done
+
+            if [ "$resource_found" = true ]; then
+              retries=$((retries + 1))
+              if [ "$retries" -ge "$max_retries" ]; then
+                echo "Reached max retry attempts. Exiting wait loop."
+                break
+              else
+                echo "Some resources still exist. Waiting for ${retry_intervals[$((retries-1))]} seconds..."
+                sleep "${retry_intervals[$((retries-1))]}"
+                resource_list=$(az resource list --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" --output yaml)
+              fi
+            else
+              echo "All resources have been deleted."
+              break
+            fi
+          done
+      - name: Wait for Soft Deletion of Key Vault and AI Services
+        if: always()
+        run: |
+          echo "Waiting for resources to be soft deleted..."
+
+          # Wait for Key Vault to be soft deleted
+          if [ -n "${{ env.KEYVAULT_NAME }}" ]; then
+            while true; do
+              DELETED_VAULT=$(az keyvault show-deleted --name ${{ env.KEYVAULT_NAME }} --query "id" -o tsv 2>/dev/null || echo "")
+              if [ -n "$DELETED_VAULT" ]; then
+                echo "Key Vault soft deleted!"
+                break
+              fi
+              echo "Key Vault not yet soft deleted. Retrying in 15s..."
+              sleep 15
+            done
+          fi
+          # Wait for AI Services to be soft deleted
+          for AI_SERVICE in ${{ env.AI_SERVICES }}; do
+            while true; do
+              DELETED_AI_SERVICE=$(az cognitiveservices account list-deleted --query "[?name=='$AI_SERVICE'].id" -o tsv 2>/dev/null || echo "")
+              if [ -n "$DELETED_AI_SERVICE" ]; then
+                echo "AI Service $AI_SERVICE is soft deleted!"
+                break
+              fi
+              echo "AI Service $AI_SERVICE not yet soft deleted. Retrying in 15s..."
+              sleep 15
+            done
+          done
+
+      - name: Purge Key Vault and AI Services
+        if: always()
+        run: |
+          echo "Purging soft deleted resources..."
+
+          # Ensure AI_SERVICES is properly split into individual services
+          IFS=' ' read -r -a SERVICES <<< "${{ env.AI_SERVICES }}"
+          for AI_SERVICE in "${SERVICES[@]}"; do
+            echo "Checking location for AI Service: $AI_SERVICE"
+            # Fetch AI Service location
+            SERVICE_LOCATION=$(az cognitiveservices account list-deleted --query "[?name=='$AI_SERVICE'].location" -o tsv 2>/dev/null || echo "")
+            if [ -n "$SERVICE_LOCATION" ]; then
+              echo "Purging AI Service $AI_SERVICE in $SERVICE_LOCATION"
+              az cognitiveservices account purge --location "$SERVICE_LOCATION" --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --name "$AI_SERVICE"
+            else
+              echo "Could not determine location for AI Service: $AI_SERVICE. Skipping purge."
+            fi
+          done
+          # Purge Key Vaults
+          echo "Starting purge for Key Vaults..."
+          IFS=' ' read -r -a VAULTS <<< "${{ env.KEYVAULT_NAME }}"
+          for VAULT in "${VAULTS[@]}"; do
+            echo "Checking location for Key Vault: $VAULT"
+            # Fetch Key Vault location
+            VAULT_LOCATION=$(az keyvault list-deleted --query "[?name=='$VAULT'].properties.location" -o tsv 2>/dev/null || echo "")
+            if [ -n "$VAULT_LOCATION" ]; then
+              echo "Purging Key Vault $VAULT in $VAULT_LOCATION"
+              az keyvault purge --name "$VAULT" --location "$VAULT_LOCATION"
+            else
+              echo "Could not determine location for Key Vault: $VAULT. Skipping purge."
+            fi
+          done
+      - name: Send Notification on Failure
+        if: failure() || needs.deploy.result == 'failure' || needs.e2e-test.result == 'failure'
+        run: |
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          EMAIL_BODY=$(cat <<EOF
+          {
+            "body": "<p>Dear Team,</p><p>We would like to inform you that the Container Migration Deployment Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a><br></p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>",
+            "subject": "Container Migration - Pipeline Failed"
+          }
+          EOF
+          )
+          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
+            -H "Content-Type: application/json" \
+            -d "$EMAIL_BODY" || echo "Failed to send notification"
+      - name: Logout from Azure
+        if: always()
+        run: |
+          az logout
+          echo "Logged out from Azure."