Skip to content

Commit f47f378

Browse files
Roopan-MicrosoftRoopan-Microsoft
authored
Merge pull request #30 from Harmanpreet-Microsoft/dev
chore: improve CI/CD workflows, automate dependency updates, and enforce linting standards
2 parents 192d0c5 + 856939e commit f47f378

30 files changed

Lines changed: 778 additions & 162 deletions

‎.flake8‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[flake8]
2+
max-line-length = 88
3+
extend-ignore = E501
4+
exclude = .venv, frontend
5+
ignore = E722,E203, W503, G004, G200, F,E711

‎.github/dependabot.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Dependabot configuration for Container Migration Solution Accelerator
2+
# This file configures automated dependency updates for all package managers used in the project
3+
4+
version: 2
5+
updates:
6+
# Frontend Node.js dependencies (grouped)
7+
- package-ecosystem: "npm"
8+
directory: "/src/frontend"
9+
schedule:
10+
interval: "monthly"
11+
commit-message:
12+
prefix: "build"
13+
target-branch: "dependabotchanges"
14+
open-pull-requests-limit: 50
15+
groups:
16+
all-npm:
17+
patterns:
18+
- "*"
19+
20+
# Backend API Python dependencies (grouped)
21+
- package-ecosystem: "pip"
22+
directory: "/src/backend-api"
23+
schedule:
24+
interval: "monthly"
25+
commit-message:
26+
prefix: "build"
27+
target-branch: "dependabotchanges"
28+
open-pull-requests-limit: 50
29+
groups:
30+
all-pip:
31+
patterns:
32+
- "*"
33+
34+
# Processor Python dependencies (grouped)
35+
- package-ecosystem: "pip"
36+
directory: "/src/processor"
37+
schedule:
38+
interval: "monthly"
39+
commit-message:
40+
prefix: "build"
41+
target-branch: "dependabotchanges"
42+
open-pull-requests-limit: 50
43+
groups:
44+
all-pip:
45+
patterns:
46+
- "*"
47+
48+
# GitHub Actions dependencies (grouped)
49+
- package-ecosystem: "github-actions"
50+
directory: "/"
51+
schedule:
52+
interval: "monthly"
53+
commit-message:
54+
prefix: "build"
55+
target-branch: "dependabotchanges"
56+
open-pull-requests-limit: 50
57+
groups:
58+
all-actions:
59+
patterns:
60+
- "*"

‎.github/workflows/ci.yml‎

Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
name: Validate Deployment
2+
on:
3+
push:
4+
branches:
5+
- main
6+
- dev
7+
- demo
8+
schedule:
9+
- cron: '0 9,21 * * *' # Runs at 9:00 AM and 9:00 PM GMT
10+
workflow_dispatch: # Allow manual triggering
11+
env:
12+
GPT_MIN_CAPACITY: 150
13+
BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }}
14+
jobs:
15+
deploy:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
RESOURCE_GROUP_NAME: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }}
19+
DEPLOYMENT_SUCCESS: ${{ steps.deployment_status.outputs.SUCCESS }}
20+
steps:
21+
- name: Checkout Code
22+
uses: actions/checkout@v4
23+
- name: Setup Azure CLI
24+
run: |
25+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
26+
az --version
27+
- name: Login to Azure
28+
run: |
29+
az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
30+
- name: Run Quota Check
31+
id: quota-check
32+
run: |
33+
export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
34+
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
35+
export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
36+
export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
37+
export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
38+
chmod +x scripts/checkquota.sh
39+
if ! scripts/checkquota.sh; then
40+
# If quota check fails due to insufficient quota, set the flag
41+
if grep -q "No region with sufficient quota found" scripts/checkquota.sh then
42+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
43+
fi
44+
exit 1 # Fail the pipeline if any other failure occurs
45+
fi
46+
- name: Send Notification on Quota Failure
47+
if: env.QUOTA_FAILED == 'true'
48+
run: |
49+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
50+
EMAIL_BODY=$(cat <<EOF
51+
{
52+
"body": "<p>Dear Team,</p><p>The Container Migration quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a></p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>",
53+
"subject": "Container Migration Deployment - Quota Check Failed"
54+
}
55+
EOF
56+
)
57+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
58+
-H "Content-Type: application/json" \
59+
-d "$EMAIL_BODY" || echo "Failed to send notification"
60+
- name: Fail Pipeline if Quota Check Fails
61+
if: env.QUOTA_FAILED == 'true'
62+
run: exit 1
63+
- name: Install Bicep CLI
64+
run: az bicep install
65+
66+
- name: Set Deployment Region
67+
run: |
68+
echo "Selected Region: $VALID_REGION"
69+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
70+
- name: Generate Resource Group Name
71+
id: generate_rg_name
72+
run: |
73+
echo "Generating a unique resource group name..."
74+
75+
ACCL_NAME="CM"
76+
77+
SHORT_UUID=$(uuidgen | cut -d'-' -f1)
78+
UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
79+
echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
80+
echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}"
81+
82+
- name: Check and Create Resource Group
83+
id: check_create_rg
84+
run: |
85+
set -e
86+
echo "Checking if resource group exists..."
87+
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
88+
if [ "$rg_exists" = "false" ]; then
89+
echo "Resource group does not exist. Creating..."
90+
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} || { echo "Error creating resource group"; exit 1; }
91+
else
92+
echo "Resource group already exists."
93+
fi
94+
echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT
95+
- name: Generate Unique Solution Prefix
96+
id: generate_solution_prefix
97+
run: |
98+
set -e
99+
COMMON_PART="Cm"
100+
TIMESTAMP=$(date +%s)
101+
UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 6)
102+
UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
103+
echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV
104+
echo "Generated SOLUTION_PREFIX: ${UNIQUE_SOLUTION_PREFIX}"
105+
106+
- name: Deploy Bicep Template
107+
id: deploy
108+
run: |
109+
set -e
110+
az deployment group create \
111+
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
112+
--template-file infra/main.bicep \
113+
--parameters solutionName=${{env.SOLUTION_PREFIX}}
114+
115+
116+
- name: Extract AI Services and Key Vault Names
117+
if: always()
118+
run: |
119+
echo "Fetching AI Services and Key Vault names before deletion..."
120+
121+
# Get Key Vault name
122+
KEYVAULT_NAME=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.KeyVault/vaults" --query "[].name" -o tsv)
123+
echo "Detected Key Vault: $KEYVAULT_NAME"
124+
echo "KEYVAULT_NAME=$KEYVAULT_NAME" >> $GITHUB_ENV
125+
126+
# Get AI Services names and convert them into a space-separated string
127+
AI_SERVICES=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.CognitiveServices/accounts" --query "[].name" -o tsv | tr '\n' ' ')
128+
129+
echo "Detected AI Services: $AI_SERVICES"
130+
echo "AI_SERVICES=$AI_SERVICES" >> $GITHUB_ENV
131+
- name: Set Deployment Status
132+
id: deployment_status
133+
if: always()
134+
run: |
135+
if [ "${{ job.status }}" == "success" ]; then
136+
echo "SUCCESS=true" >> $GITHUB_OUTPUT
137+
else
138+
echo "SUCCESS=false" >> $GITHUB_OUTPUT
139+
fi
140+
- name: Logout from Azure
141+
if: always()
142+
run: |
143+
az logout
144+
echo "Logged out from Azure."
145+
cleanup-deployment:
146+
if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != ''
147+
needs: [deploy]
148+
runs-on: ubuntu-latest
149+
env:
150+
RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
151+
steps:
152+
- name: Setup Azure CLI
153+
run: |
154+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
155+
az --version
156+
- name: Login to Azure
157+
run: |
158+
az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
159+
az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
160+
- name: Extract AI Services and Key Vault Names
161+
if: always()
162+
run: |
163+
echo "Fetching AI Services and Key Vault names before deletion..."
164+
165+
# Get Key Vault name
166+
KEYVAULT_NAME=$(az resource list --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --resource-type "Microsoft.KeyVault/vaults" --query "[].name" -o tsv)
167+
echo "Detected Key Vault: $KEYVAULT_NAME"
168+
echo "KEYVAULT_NAME=$KEYVAULT_NAME" >> $GITHUB_ENV
169+
# Extract AI Services names
170+
echo "Fetching AI Services..."
171+
AI_SERVICES=$(az resource list --resource-group '${{ env.RESOURCE_GROUP_NAME }}' --resource-type "Microsoft.CognitiveServices/accounts" --query "[].name" -o tsv)
172+
# Flatten newline-separated values to space-separated
173+
AI_SERVICES=$(echo "$AI_SERVICES" | paste -sd ' ' -)
174+
echo "Detected AI Services: $AI_SERVICES"
175+
echo "AI_SERVICES=$AI_SERVICES" >> $GITHUB_ENV
176+
177+
- name: Delete Bicep Deployment
178+
if: always()
179+
run: |
180+
set -e
181+
echo "Checking if resource group exists..."
182+
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
183+
if [ "$rg_exists" = "true" ]; then
184+
echo "Resource group exists. Cleaning..."
185+
az group delete \
186+
--name ${{ env.RESOURCE_GROUP_NAME }} \
187+
--yes \
188+
--no-wait
189+
echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}"
190+
else
191+
echo "Resource group does not exist."
192+
fi
193+
- name: Wait for Resource Deletion to Complete
194+
if: always()
195+
run: |
196+
echo "Waiting for all deployed resources (including AI Services) to be deleted..."
197+
198+
# Convert AI_SERVICES space-separated string into an array
199+
IFS=' ' read -r -a resources_to_check <<< "${{ env.AI_SERVICES }}"
200+
201+
echo "Resources to check for deletion:"
202+
printf '%s\n' "${resources_to_check[@]}"
203+
204+
# Get the current resource list in YAML
205+
resource_list=$(az resource list --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" --output yaml)
206+
207+
# Set up retry logic
208+
max_retries=3
209+
retry_intervals=(30 60 120)
210+
retries=0
211+
212+
while true; do
213+
resource_found=false
214+
for resource in "${resources_to_check[@]}"; do
215+
echo "Checking if resource '$resource' still exists..."
216+
if echo "$resource_list" | grep -q "name: $resource"; then
217+
echo "Resource '$resource' still exists."
218+
resource_found=true
219+
else
220+
echo "Resource '$resource' has been deleted."
221+
fi
222+
done
223+
224+
if [ "$resource_found" = true ]; then
225+
retries=$((retries + 1))
226+
if [ "$retries" -ge "$max_retries" ]; then
227+
echo "Reached max retry attempts. Exiting wait loop."
228+
break
229+
else
230+
echo "Some resources still exist. Waiting for ${retry_intervals[$((retries-1))]} seconds..."
231+
sleep "${retry_intervals[$((retries-1))]}"
232+
resource_list=$(az resource list --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" --output yaml)
233+
fi
234+
else
235+
echo "All resources have been deleted."
236+
break
237+
fi
238+
done
239+
- name: Wait for Soft Deletion of Key Vault and AI Services
240+
if: always()
241+
run: |
242+
echo "Waiting for resources to be soft deleted..."
243+
244+
# Wait for Key Vault to be soft deleted
245+
if [ -n "${{ env.KEYVAULT_NAME }}" ]; then
246+
while true; do
247+
DELETED_VAULT=$(az keyvault show-deleted --name ${{ env.KEYVAULT_NAME }} --query "id" -o tsv 2>/dev/null || echo "")
248+
if [ -n "$DELETED_VAULT" ]; then
249+
echo "Key Vault soft deleted!"
250+
break
251+
fi
252+
echo "Key Vault not yet soft deleted. Retrying in 15s..."
253+
sleep 15
254+
done
255+
fi
256+
# Wait for AI Services to be soft deleted
257+
for AI_SERVICE in ${{ env.AI_SERVICES }}; do
258+
while true; do
259+
DELETED_AI_SERVICE=$(az cognitiveservices account list-deleted --query "[?name=='$AI_SERVICE'].id" -o tsv 2>/dev/null || echo "")
260+
if [ -n "$DELETED_AI_SERVICE" ]; then
261+
echo "AI Service $AI_SERVICE is soft deleted!"
262+
break
263+
fi
264+
echo "AI Service $AI_SERVICE not yet soft deleted. Retrying in 15s..."
265+
sleep 15
266+
done
267+
done
268+
269+
- name: Purge Key Vault and AI Services
270+
if: always()
271+
run: |
272+
echo "Purging soft deleted resources..."
273+
274+
# Ensure AI_SERVICES is properly split into individual services
275+
IFS=' ' read -r -a SERVICES <<< "${{ env.AI_SERVICES }}"
276+
for AI_SERVICE in "${SERVICES[@]}"; do
277+
echo "Checking location for AI Service: $AI_SERVICE"
278+
# Fetch AI Service location
279+
SERVICE_LOCATION=$(az cognitiveservices account list-deleted --query "[?name=='$AI_SERVICE'].location" -o tsv 2>/dev/null || echo "")
280+
if [ -n "$SERVICE_LOCATION" ]; then
281+
echo "Purging AI Service $AI_SERVICE in $SERVICE_LOCATION"
282+
az cognitiveservices account purge --location "$SERVICE_LOCATION" --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --name "$AI_SERVICE"
283+
else
284+
echo "Could not determine location for AI Service: $AI_SERVICE. Skipping purge."
285+
fi
286+
done
287+
# Purge Key Vaults
288+
echo "Starting purge for Key Vaults..."
289+
IFS=' ' read -r -a VAULTS <<< "${{ env.KEYVAULT_NAME }}"
290+
for VAULT in "${VAULTS[@]}"; do
291+
echo "Checking location for Key Vault: $VAULT"
292+
# Fetch Key Vault location
293+
VAULT_LOCATION=$(az keyvault list-deleted --query "[?name=='$VAULT'].properties.location" -o tsv 2>/dev/null || echo "")
294+
if [ -n "$VAULT_LOCATION" ]; then
295+
echo "Purging Key Vault $VAULT in $VAULT_LOCATION"
296+
az keyvault purge --name "$VAULT" --location "$VAULT_LOCATION"
297+
else
298+
echo "Could not determine location for Key Vault: $VAULT. Skipping purge."
299+
fi
300+
done
301+
- name: Send Notification on Failure
302+
if: failure() || needs.deploy.result == 'failure'
303+
run: |
304+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
305+
EMAIL_BODY=$(cat <<EOF
306+
{
307+
"body": "<p>Dear Team,</p><p>We would like to inform you that the Container Migration Deployment Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a><br></p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>",
308+
"subject": "Container Migration - Pipeline Failed"
309+
}
310+
EOF
311+
)
312+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
313+
-H "Content-Type: application/json" \
314+
-d "$EMAIL_BODY" || echo "Failed to send notification"
315+
- name: Logout from Azure
316+
if: always()
317+
run: |
318+
az logout
319+
echo "Logged out from Azure."

0 commit comments

Comments
 (0)