Skip to content

Commit e8e00a7

Browse files
jaredfholgatejaredfholgate
authored
feat: accelerator permission flattening (#132)
* feat: accelerator permission flattening * fmt * add lib to test * fmt * typo * fix path * fix data type * fix roles * fix formatting * move subs * fmt * bug fixes * Move exists = true method * bin redundant role assignments * fix ternary * bit of debugging * more debugging * fix test * bug in role assignment * fix test * fix first run check * fix composite keys and bicep mg * Create 4 subs per test * fix write-verbose * re-instate variable needed for classic bicep * get the subscription id * Add bootstrap subs and clean up steps * fix missing variable * fix copilot typo * import module * sort cleanup script * register resource providers * fix message * fix local tests * add forced subscription placement * delete all management groups that match * remove redundant extra role assignments * add another az login * remove redundant variable * linting * fix storage container perms for local * bin off uksouth
1 parent 6d8bb28 commit e8e00a7

29 files changed

Lines changed: 839 additions & 635 deletions

‎.github/tests/scripts/create-subscriptions.ps1‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
[CmdletBinding()]
2+
param(
3+
[string]$billingScope,
4+
[string]$subscriptionNamePrefix = "accelerator-bootstrap-modules",
5+
[string[]]$subscriptionTypes = @("connectivity", "management", "identity", "security", "bootstrap"),
6+
[string[]]$resourceProviders = @("Microsoft.Security"),
7+
[int]$maxRetries = 5,
8+
[int]$throttleLimit = 2,
9+
[int]$resourceProviderThrottleLimit = 10,
10+
[switch]$planOnly
11+
)
12+
13+
# Get current Azure account information
14+
$accountInfo = az account show --output json | ConvertFrom-Json
15+
16+
# Look up tenant name from Graph API domains
17+
$domains = az rest --method get --url "https://graph.microsoft.com/v1.0/domains" --output json | ConvertFrom-Json
18+
$defaultDomain = $domains.value | Where-Object { $_.isDefault -eq $true }
19+
$tenantName = if ($defaultDomain.id) { $defaultDomain.id } else { "(unknown)" }
20+
21+
Write-Host ""
22+
Write-Host "=== Azure Connection Information ===" -ForegroundColor Cyan
23+
Write-Host "Tenant ID: $($accountInfo.tenantId)" -ForegroundColor Yellow
24+
Write-Host "Tenant Name: $tenantName" -ForegroundColor Yellow
25+
Write-Host "Account: $($accountInfo.user.name)" -ForegroundColor Yellow
26+
Write-Host "Subscription: $($accountInfo.name)" -ForegroundColor Yellow
27+
Write-Host "====================================" -ForegroundColor Cyan
28+
Write-Host ""
29+
30+
$confirmation = Read-Host "Do you want to continue with this account? (y/n)"
31+
if ($confirmation -ine 'y') {
32+
Write-Host "Operation cancelled by user." -ForegroundColor Red
33+
exit 0
34+
}
35+
36+
Write-Host ""
37+
38+
$tests = ./.github/tests/scripts/generate-matrix.ps1
39+
40+
# Get all existing aliases once using REST API with paging (more efficient than checking each one individually)
41+
Write-Host "Fetching existing subscription aliases..." -ForegroundColor Cyan
42+
$existingAliasNames = @()
43+
$aliasUrl = "https://management.azure.com/providers/Microsoft.Subscription/aliases?api-version=2021-10-01"
44+
45+
do {
46+
$response = az rest --method get --url "`"$aliasUrl`"" | ConvertFrom-Json
47+
if ($response.value) {
48+
$existingAliasNames += $response.value | ForEach-Object { $_.name }
49+
}
50+
$aliasUrl = $response.nextLink
51+
} while ($aliasUrl)
52+
53+
Write-Host "Fetched $($existingAliasNames.Count) existing aliases." -ForegroundColor Green
54+
55+
# Build list of subscriptions to create
56+
$subscriptionsToCreate = @()
57+
$existingSubscriptions = @()
58+
$skippedTests = @()
59+
60+
foreach ($test in $tests) {
61+
# Only create subscriptions for tests that deploy Azure resources
62+
if ($test.deployAzureResources -ne "true") {
63+
$skippedTests += $test.Name
64+
continue
65+
}
66+
67+
foreach ($subscriptionType in $subscriptionTypes) {
68+
$subscriptionName = "$subscriptionNamePrefix-$($test.ShortNamePrefix)-$subscriptionType"
69+
70+
if ($existingAliasNames -notcontains $subscriptionName) {
71+
$subscriptionsToCreate += $subscriptionName
72+
} else {
73+
$existingSubscriptions += $subscriptionName
74+
}
75+
}
76+
}
77+
78+
# Display skipped tests
79+
if ($skippedTests.Count -gt 0) {
80+
Write-Host ""
81+
Write-Host "=== Tests Skipped (deployAzureResources=false) ===" -ForegroundColor Cyan
82+
foreach ($test in $skippedTests) {
83+
Write-Host " - $test" -ForegroundColor Gray
84+
}
85+
}
86+
87+
# Display existing subscriptions
88+
if ($existingSubscriptions.Count -gt 0) {
89+
Write-Host ""
90+
Write-Host "=== Existing Subscription Aliases (will be skipped) ===" -ForegroundColor Cyan
91+
foreach ($sub in $existingSubscriptions) {
92+
Write-Host " - $sub" -ForegroundColor Gray
93+
}
94+
}
95+
96+
# Display subscriptions to create
97+
Write-Host ""
98+
if ($subscriptionsToCreate.Count -eq 0) {
99+
Write-Host "No new subscriptions to create. All aliases already exist." -ForegroundColor Green
100+
}
101+
102+
if ($subscriptionsToCreate.Count -gt 0) {
103+
Write-Host "=== Subscriptions to Create ===" -ForegroundColor Cyan
104+
foreach ($sub in $subscriptionsToCreate) {
105+
Write-Host " - $sub" -ForegroundColor Yellow
106+
}
107+
Write-Host ""
108+
Write-Host "Total: $($subscriptionsToCreate.Count) subscription(s) to create" -ForegroundColor Cyan
109+
Write-Host ""
110+
}
111+
112+
if ($planOnly) {
113+
Write-Host "Plan only mode - no subscriptions will be created." -ForegroundColor Magenta
114+
return
115+
}
116+
117+
if ($subscriptionsToCreate.Count -gt 0) {
118+
# Prompt for confirmation before creating
119+
$createConfirmation = Read-Host "Do you want to create these $($subscriptionsToCreate.Count) subscription(s)? (y/n)"
120+
if ($createConfirmation -ine 'y') {
121+
Write-Host "Operation cancelled by user." -ForegroundColor Red
122+
return
123+
}
124+
125+
Write-Host ""
126+
127+
# Create a thread-safe hashtable to track rate limiting across parallel tasks
128+
$rateLimitState = [hashtable]::Synchronized(@{
129+
WaitUntil = [DateTime]::MinValue
130+
})
131+
132+
# Create the subscriptions in parallel with retry logic
133+
Write-Host "Creating subscriptions (throttle: $throttleLimit)..." -ForegroundColor Cyan
134+
135+
$results = $subscriptionsToCreate | ForEach-Object -Parallel {
136+
$subscriptionName = $_
137+
$scope = $using:billingScope
138+
$retries = $using:maxRetries
139+
$state = $using:rateLimitState
140+
$VerbosePreference = $using:VerbosePreference
141+
$retryCount = 0
142+
$success = $false
143+
144+
while (-not $success -and $retryCount -lt $retries) {
145+
# Check if we're in a rate limit wait period
146+
$waitUntil = $state.WaitUntil
147+
if ($waitUntil -gt [DateTime]::Now) {
148+
$waitSeconds = [math]::Ceiling(($waitUntil - [DateTime]::Now).TotalSeconds)
149+
Write-Host "Rate limit active. $subscriptionName waiting $waitSeconds seconds..." -ForegroundColor Yellow
150+
Start-Sleep -Seconds $waitSeconds
151+
}
152+
153+
Write-Host "Creating subscription: $subscriptionName (Attempt $($retryCount + 1) of $retries)" -ForegroundColor Yellow
154+
$result = az account alias create --name "$subscriptionName" --billing-scope "$scope" --display-name "$subscriptionName" --workload "Production" 2>&1
155+
156+
if ($LASTEXITCODE -eq 0) {
157+
$success = $true
158+
Write-Host "Successfully created: $subscriptionName" -ForegroundColor Green
159+
} else {
160+
$errorMessage = $result | Out-String
161+
if ($errorMessage -match "TooManyRequests.*Retry in (\d{2}):(\d{2}):(\d{2})") {
162+
$hours = [int]$Matches[1]
163+
$minutes = [int]$Matches[2]
164+
$seconds = [int]$Matches[3]
165+
$waitSeconds = ($hours * 3600) + ($minutes * 60) + $seconds + (1 * 60) # Add 60 second buffer
166+
Write-Verbose $errorMessage
167+
168+
# Set the shared rate limit wait time
169+
$newWaitUntil = [DateTime]::Now.AddSeconds($waitSeconds)
170+
if ($newWaitUntil -gt $state.WaitUntil) {
171+
$state.WaitUntil = $newWaitUntil
172+
Write-Host "Rate limit hit! All tasks will wait until $($newWaitUntil.ToString('HH:mm:ss'))" -ForegroundColor Red
173+
}
174+
175+
Write-Host "Rate limited for $subscriptionName. Waiting $waitSeconds seconds before retry..." -ForegroundColor Yellow
176+
Start-Sleep -Seconds $waitSeconds
177+
$retryCount++
178+
} else {
179+
Write-Host "Failed to create $subscriptionName : $errorMessage" -ForegroundColor Red
180+
break
181+
}
182+
}
183+
}
184+
185+
[PSCustomObject]@{
186+
Name = $subscriptionName
187+
Success = $success
188+
}
189+
} -ThrottleLimit $throttleLimit
190+
191+
$successCount = ($results | Where-Object { $_.Success }).Count
192+
$failCount = ($results | Where-Object { -not $_.Success }).Count
193+
194+
Write-Host ""
195+
Write-Host "Subscription creation complete." -ForegroundColor Green
196+
Write-Host " Successful: $successCount" -ForegroundColor Green
197+
if ($failCount -gt 0) {
198+
Write-Host " Failed: $failCount" -ForegroundColor Red
199+
}
200+
}
201+
202+
# Register resource providers for all subscriptions
203+
if ($resourceProviders.Count -gt 0 -and -not $planOnly) {
204+
Write-Host ""
205+
Write-Host "=== Registering Resource Providers ===" -ForegroundColor Cyan
206+
Write-Host "Providers: $($resourceProviders -join ', ')" -ForegroundColor Yellow
207+
208+
$allSubscriptionNames = $subscriptionsToCreate + $existingSubscriptions
209+
210+
# Get subscription IDs for all aliases and register providers
211+
$allSubscriptionNames | ForEach-Object -Parallel {
212+
$subscriptionName = $_
213+
$providers = $using:resourceProviders
214+
$VerbosePreference = $using:VerbosePreference
215+
216+
# Get the subscription ID from the alias
217+
$aliasInfo = az account alias show --name "$subscriptionName" --output json 2>$null | ConvertFrom-Json
218+
219+
if ($aliasInfo -and $aliasInfo.properties.subscriptionId) {
220+
$subscriptionId = $aliasInfo.properties.subscriptionId
221+
222+
foreach ($provider in $providers) {
223+
# Check if provider is already registered
224+
$providerState = az provider show --namespace $provider --subscription $subscriptionId --query "registrationState" --output tsv 2>$null
225+
226+
if ($providerState -ine "Registered") {
227+
Write-Host "Registering $provider for $subscriptionName ($subscriptionId)..." -ForegroundColor Yellow
228+
az provider register --namespace $provider --subscription $subscriptionId --output none --wait
229+
if ($LASTEXITCODE -eq 0) {
230+
Write-Host "Registration succeeded: $provider for $subscriptionName" -ForegroundColor Green
231+
} else {
232+
Write-Host "Failed to register: $provider for $subscriptionName" -ForegroundColor Red
233+
}
234+
} else {
235+
Write-Host "Already registered: $provider for $subscriptionName" -ForegroundColor Gray
236+
}
237+
}
238+
} else {
239+
Write-Host "Could not get subscription ID for alias: $subscriptionName" -ForegroundColor Red
240+
}
241+
} -ThrottleLimit $resourceProviderThrottleLimit
242+
243+
Write-Host ""
244+
Write-Host "Resource provider registration complete." -ForegroundColor Green
245+
}

‎.github/tests/scripts/generate-matrix.ps1‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ $combinations = [ordered]@{
4848
infrastructureAsCode = @("terraform")
4949
agentType = @("public", "private", "none")
5050
operatingSystem = @("ubuntu")
51-
starterModule = @("test_nested")
51+
starterModule = @("test")
5252
regions = @("multi")
5353
terraformVersion = @("latest")
5454
deployAzureResources = @("true")
@@ -58,7 +58,7 @@ $combinations = [ordered]@{
5858
infrastructureAsCode = @("terraform")
5959
agentType = @("public", "private", "none")
6060
operatingSystem = @("ubuntu")
61-
starterModule = @("test_nested")
61+
starterModule = @("test")
6262
regions = @("multi")
6363
terraformVersion = @("latest")
6464
deployAzureResources = @("true")
@@ -80,7 +80,7 @@ $combinations = [ordered]@{
8080
operatingSystem = @("ubuntu", "windows", "macos")
8181
starterModule = @("test")
8282
regions = @("multi")
83-
terraformVersion = @("1.5.0")
83+
terraformVersion = @("1.6.0")
8484
deployAzureResources = @("false")
8585
}
8686
local_single_region_tests = [ordered]@{
@@ -164,7 +164,8 @@ function Get-MatrixRecursively {
164164

165165
$combination.Name = $name.Trim("-")
166166
$combination.Hash = Get-Hash $name
167-
$combination.ShortName = "r" + $combination.Hash.Substring(0, 5).ToLower() + "r" + $runNumber
167+
$combination.ShortNamePrefix = "r" + $combination.Hash.Substring(0, 5).ToLower()
168+
$combination.ShortName = $combination.ShortNamePrefix + "r" + $runNumber
168169

169170
$calculatedCombinations += $combination
170171

0 commit comments

Comments
 (0)