Skip to content

Commit 28982f2

Browse files
troettingertroettinger
authored
Merge branch 'master' into master
2 parents 0316bc8 + c1df511 commit 28982f2

20 files changed

Lines changed: 2542 additions & 1526 deletions

‎CanaryValidator/Canary.Tests.ps1‎

Lines changed: 488 additions & 424 deletions
Large diffs are not rendered by default.

‎CanaryValidator/Canary.Utilities.psm1‎

Lines changed: 85 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
$Global:ContinueOnFailure = $false
22
$Global:JSONLogFile = "Run-Canary.JSON"
33
$Global:TxtLogFile = "AzureStackCanaryLog.Log"
4+
$Global:wttLogFileName= ""
5+
if (Test-Path -Path "$PSScriptRoot\..\WTTLog.ps1")
6+
{
7+
Import-Module -Name "$PSScriptRoot\..\WTTLog.ps1" -Force
8+
$Global:wttLogFileName = (Join-Path $PSScriptRoot "AzureStack_CanaryValidation_Test.wtl")
9+
}
10+
411
$UseCase = @{}
512
[System.Collections.Stack] $AllUseCases = New-Object System.Collections.Stack
6-
filter timestamp {"$(Get-Date -Format G): $_"}
13+
filter timestamp {"$(Get-Date -Format HH:mm:ss.ffff): $_"}
14+
715

816
function Log-Info
917
{
@@ -96,6 +104,41 @@ function Log-JSONReport
96104
}
97105
}
98106

107+
function Get-CanaryResult
108+
{
109+
$logContent = Get-Content -Raw -Path $Global:JSONLogFile | ConvertFrom-Json
110+
Log-Info ($logContent.UseCases | Format-Table -AutoSize @{Expression = {$_.Name}; Label = "Name"; Align = "Left"},
111+
@{Expression = {$_.Result}; Label="Result"; Align = "Left"},
112+
@{Expression = {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalSeconds}; Label = "Duration`n[Seconds]"; Align = "Left"},
113+
@{Expression = {$_.Description}; Label = "Description"; Align = "Left"})
114+
}
115+
116+
function Get-CanaryLonghaulResult
117+
{
118+
[CmdletBinding()]
119+
param(
120+
[parameter(Mandatory=$true, Position = 0)]
121+
[ValidateNotNullOrEmpty()]
122+
[string]$LogPath
123+
)
124+
125+
$logFiles = (Get-ChildItem -Path $LogPath -Filter *.JSON -File).FullName
126+
$logContent = @()
127+
foreach($file in $logFiles)
128+
{
129+
$logContent += (Get-Content -Raw -Path $file | ConvertFrom-Json).UseCases
130+
}
131+
$usecasesGroup = $logContent | Group-Object -Property Name
132+
$usecasesGroup | Format-Table -AutoSize @{Expression = {$_.Name}; Label = "Name"; Align = "Left"},
133+
@{Expression={$_.Count}; Label="Count"; Align = "Left"},
134+
@{Expression={$passPct = [math]::Round(((($_.Group | Where-Object Result -eq "PASS" | Measure-Object).Count/$_.Count)*100), 0); $passPct.ToString()+"%"};Label="Pass`n[Goal: >99%]"; Align = "Left"},
135+
@{Expression={[math]::Round(($_.Group | Where-Object Result -eq "PASS" | ForEach-Object {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalMilliseconds} | Measure-Object -Minimum).Minimum, 0)};Label="MinTime`n[msecs]"; Align = "Left"},
136+
@{Expression={[math]::Round(($_.Group | Where-Object Result -eq "PASS" | ForEach-Object {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalMilliseconds} | Measure-Object -Maximum).Maximum, 0)};Label="MaxTime`n[msecs]"; Align = "Left"},
137+
@{Expression={[math]::Round(($_.Group | Where-Object Result -eq "PASS" | ForEach-Object {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalMilliseconds} | Measure-Object -Average).Average, 0)};Label="AvgTime`n[MilliSeconds]"; Align = "Left"},
138+
@{Expression={$pCount = ($_.Group | Where-Object Result -eq "PASS").Count; $times = ($_.Group | Where-Object Result -eq "PASS" | ForEach-Object {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalMilliseconds}); $avgTime = ($times | Measure-Object -Average).Average; $sd = 0; foreach ($time in $times){$sd += [math]::Pow(($time - $avgTime), 2)}; [math]::Round([math]::Sqrt($sd/$pCount), 0)};Label="StdDev"; Align = "Left"},
139+
@{Expression={$pCount = ($_.Group | Where-Object Result -eq "PASS").Count; $times = ($_.Group | Where-Object Result -eq "PASS" | ForEach-Object {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalMilliseconds}); $avgTime = ($times | Measure-Object -Average).Average; $sd = 0; foreach ($time in $times){$sd += [math]::Pow(($time - $avgTime), 2)}; [math]::Round(([math]::Round([math]::Sqrt($sd/$pCount), 2)/$avgTime), 0) * 100};Label="RelativeStdDev`n[Goal: <50%]"; Align = "Left"}
140+
}
141+
99142
function Start-Scenario
100143
{
101144
[CmdletBinding()]
@@ -126,6 +169,11 @@ function Start-Scenario
126169
$Global:TxtLogFile = $LogFileName + ".Log"
127170
}
128171
}
172+
if ($Global:wttLogFileName)
173+
{
174+
OpenWTTLogger $Global:wttLogFileName
175+
}
176+
129177
New-Item -Path $Global:JSONLogFile -Type File -Force
130178
New-Item -Path $Global:TxtLogFile -Type File -Force
131179
$jsonReport = @{
@@ -137,7 +185,12 @@ function Start-Scenario
137185
}
138186

139187
function End-Scenario
140-
{}
188+
{
189+
if ($Global:wttLogFileName)
190+
{
191+
CloseWTTLogger
192+
}
193+
}
141194

142195
function Invoke-Usecase
143196
{
@@ -154,6 +207,10 @@ function Invoke-Usecase
154207
[ScriptBlock]$UsecaseBlock
155208
)
156209
Log-Info ("###### [START] Usecase: $Name ######`n")
210+
if ($Global:wttLogFileName)
211+
{
212+
StartTest "CanaryGate:$Name"
213+
}
157214

158215
if ($Description)
159216
{
@@ -167,14 +224,21 @@ function Invoke-Usecase
167224
{
168225
Log-Info ($result)
169226
}
170-
227+
if ($Global:wttLogFileName)
228+
{
229+
EndTest "CanaryGate:$Name" $true
230+
}
171231
Log-Info ("###### [END] Usecase: $Name ###### [RESULT = PASS] ######`n")
172232
return $result | Out-Null
173233
}
174234
catch [System.Exception]
175235
{
176236
Log-Exception ($_.Exception)
177237
Log-Error ("###### [END] Usecase: $Name ###### [RESULT = FAIL] ######`n")
238+
if ($Global:wttLogFileName)
239+
{
240+
EndTest "CanaryGate:$Name" $false
241+
}
178242
if (-not $Global:ContinueOnFailure)
179243
{
180244
throw $_.Exception
@@ -188,13 +252,16 @@ function GetAzureStackEndpoints
188252
param(
189253
[parameter(Mandatory=$true)]
190254
[ValidateNotNullOrEmpty()]
191-
[string]$EnvironmentDomainFQDN
255+
[string]$EnvironmentDomainFQDN,
256+
[parameter(Mandatory=$true)]
257+
[ValidateNotNullOrEmpty()]
258+
[string]$ArmEndpoint
259+
192260
)
193261

194262
$aadTenantId = $AADTenantId
195-
$armEndpoint = "https://api." + $EnvironmentDomainFQDN
196263
$endptres = Invoke-RestMethod "${armEndpoint}/metadata/endpoints?api-version=1.0" -ErrorAction Stop
197-
$ActiveDirectoryEndpoint = $($endptres.authentication.loginEndpoint)
264+
$ActiveDirectoryEndpoint = $($endptres.authentication.loginEndpoint).TrimEnd("/") + "/"
198265
$ActiveDirectoryServiceEndpointResourceId = $($endptres.authentication.audiences[0])
199266
$ResourceManagerEndpoint = $armEndpoint
200267
$GalleryEndpoint = $endptres.galleryEndpoint
@@ -399,7 +466,7 @@ function NewKeyVaultQuota
399466
[string] $ArmLocation
400467
)
401468

402-
$uri = "{0}/subscriptions/{1}/providers/Microsoft.Keyvault.Admin/locations/{2}/quotas?api-version=2014-04-01-preview" -f $AdminUri, $SubscriptionId, $ArmLocation
469+
$uri = "{0}/subscriptions/{1}/providers/Microsoft.Keyvault.Admin/locations/{2}/quotas?api-version=2017-02-01-preview" -f $AdminUri, $SubscriptionId, $ArmLocation
403470
$headers = @{ "Authorization" = "Bearer "+ $AzureStackToken }
404471
$kvQuota = Invoke-RestMethod -Method Get -Uri $uri -Headers $headers -ContentType 'application/json' -ErrorAction Stop
405472

@@ -418,11 +485,14 @@ function NewAzureStackToken
418485
[string]$EnvironmentDomainFQDN,
419486
[parameter(Mandatory=$true)]
420487
[ValidateNotNullOrEmpty()]
421-
[System.Management.Automation.PSCredential]$Credentials
488+
[System.Management.Automation.PSCredential]$Credentials,
489+
[parameter(Mandatory=$true)]
490+
[ValidateNotNullOrEmpty()]
491+
[string]$ArmEndpoint
492+
422493
)
423494

424-
$endpoints = GetAzureStackEndpoints -EnvironmentDomainFQDN $EnvironmentDomainFQDN
425-
#$asToken = Get-AzureStackToken -Authority ($endpoints.ActiveDirectoryEndpoint + $aadTenantId + "/oauth2") -Resource $endpoints.ActiveDirectoryServiceEndpointResourceId -AadTenantId $AADTenantID -Credential $Credentials -ErrorAction Stop
495+
$endpoints = GetAzureStackEndpoints -EnvironmentDomainFQDN $EnvironmentDomainFQDN -ArmEndPoint $ArmEndpoint
426496
$asToken = Get-AzureStackToken -Authority $endpoints.ActiveDirectoryEndpoint -Resource $endpoints.ActiveDirectoryServiceEndpointResourceId -AadTenantId $aadTenantId -Credential $Credentials -ErrorAction Stop
427497
return $asToken
428498
}
@@ -445,13 +515,15 @@ function NewAzureStackDefaultQuotas
445515
[string]$EnvironmentDomainFQDN,
446516
[parameter(Mandatory=$true)]
447517
[ValidateNotNullOrEmpty()]
448-
[System.Management.Automation.PSCredential]$Credentials
518+
[System.Management.Automation.PSCredential]$Credentials,
519+
[parameter(Mandatory=$true)]
520+
[ValidateNotNullOrEmpty()]
521+
[string]$ArmEndpoint
449522
)
450523

451524
$aadTenantId = $AADTenantId
452525
$serviceQuotas = @()
453-
$armEndpoint = "https://api." + $EnvironmentDomainFQDN
454-
$asToken = NewAzureStackToken -AADTenantId $AADTenantId -EnvironmentDomainFQDN $EnvironmentDomainFQDN -Credentials $Credentials
526+
$asToken = NewAzureStackToken -AADTenantId $AADTenantId -EnvironmentDomainFQDN $EnvironmentDomainFQDN -Credentials $Credentials -ArmEndpoint $ArmEndpoint
455527
#$serviceQuotas += NewSubscriptionsQuota -AdminUri $armEndpoint -SubscriptionId $SubscriptionId -AzureStackToken $asToken -ArmLocation $ResourceLocation
456528
$serviceQuotas += NewStorageQuota -AdminUri $armEndPoint -SubscriptionId $SubscriptionId -AzureStackToken $asToken -ArmLocation $ResourceLocation
457529
$serviceQuotas += NewComputeQuota -AdminUri $armEndPoint -SubscriptionId $SubscriptionId -AzureStackToken $asToken -ArmLocation $ResourceLocation

‎CanaryValidator/README.md‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,43 @@
1+
12
# AzureStack Canary validator
2-
Canary validator provides a breadth customer experience with the Azure Stack deployment. It tries to exercise the various customer usecases on the deployment.
3+
Canary validator provides a breadth customer experience with the Azure Stack deployment. It tries to exercise the various customer scenarios/usecases on the deployment.
4+
35
Instructions are relative to the .\CanaryValidator directory.
46
Canary can be invoked either as Service Administrator or Tenant Administrator.
57

6-
# Execute Canary as Service Administrator
8+
# Download Canary
9+
```powershell
10+
invoke-webrequest https://github.com/Azure/AzureStack-Tools/archive/master.zip -OutFile master.zip
11+
expand-archive master.zip -DestinationPath . -Force
12+
cd AzureStack-Tools-master\CanaryValidator
13+
```
14+
15+
# To execute Canary as Tenant Administrator
716
```powershell
17+
# Install-Module AzureRM -RequiredVersion 1.2.8 -Force
18+
# Install-Module AzureStack
19+
$TenantAdminCreds = New-Object System.Management.Automation.PSCredential "<Tenant Admin username>", (ConvertTo-SecureString "<Tenant Admin password>" -AsPlainText -Force)
820
$ServiceAdminCreds = New-Object System.Management.Automation.PSCredential "<Service Admin username>", (ConvertTo-SecureString "<Service Admin password>" -AsPlainText -Force)
9-
.\Canary.Tests.ps1 -ServiceAdminCredentials $ServiceAdminCreds -AADTenantID "<TenantID from Azure Active Directory>" -EnvironmentDomainFQDN "<Azure Stack deployment domain FQDN>"
21+
.\Canary.Tests.ps1 -AADTenantID "<TenantID from Azure Active Directory>" -AdminArmEndpoint "<Administrative ARM endpoint>" -ServiceAdminCredentials $ServiceAdminCreds -TenantArmEndpoint "<Tenant ARM endpoint>" -TenantAdminCredentials $TenantAdminCreds
1022
```
1123

12-
# Execute Canary as Tenant Administrator
24+
# To execute Canary as Service Administrator
1325
```powershell
14-
$TenantAdminCreds = New-Object System.Management.Automation.PSCredential "<Service Admin username>", (ConvertTo-SecureString "<Service Admin password>" -AsPlainText -Force)
15-
.\Canary.Tests.ps1 -ServiceAdminCredentials $ServiceAdminCreds -AADTenantID "<TenantID from Azure Active Directory>" -EnvironmentDomainFQDN "<Azure Stack deployment domain FQDN>" -TenantAdminCredentials $TenantAdminCreds
16-
```
26+
# Install-Module AzureRM -RequiredVersion 1.2.8 -Force
27+
# Install-Module AzureStack
28+
$ServiceAdminCreds = New-Object System.Management.Automation.PSCredential "<Service Admin username>", (ConvertTo-SecureString "<Service Admin password>" -AsPlainText -Force)
29+
.\Canary.Tests.ps1 -AADTenantID "<TenantID from Azure Active Directory>" -AdminArmEndpoint "<Administrative ARM endpoint>" -ServiceAdminCredentials $ServiceAdminCreds -TenantArmEndpoint "<Tenant ARM endpoint>"
30+
```
31+
# Reading the results & logs
32+
Canary generates log files in the TMP directory ($env:TMP). The logs can be found under the directory "CanaryLogs[DATETIME]". There are two types of logs generated, a text log and a JSON log. JSON log provides a quick and easy view of all the usecases and their corresponding results. Text log provides a more detailed output of each usecase execution, its output and results.
33+
34+
Each usecase entry in the JSON log consists of the following fields.
35+
- Name
36+
- Description
37+
- StartTime
38+
- EndTime
39+
- Result
40+
- Exception (in case a scenario fails)
41+
42+
The exception field is helpful to debug failed usecases.
43+

0 commit comments

Comments
 (0)