Skip to content

Commit 3187b27

Browse files
MatthewMcGlynnMatthewMcGlynn
authored
Merge pull request #255 from Azure/vnext
Merging vnext into master
2 parents ad17106 + eb33673 commit 3187b27

54 files changed

Lines changed: 8990 additions & 4966 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CanaryValidator/Canary.Tests.ps1‎

Lines changed: 461 additions & 126 deletions
Large diffs are not rendered by default.

‎CanaryValidator/Canary.Utilities.psm1‎

Lines changed: 127 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
$Global:ContinueOnFailure = $false
2-
$Global:JSONLogFile = "Run-Canary.JSON"
3-
$Global:TxtLogFile = "AzureStackCanaryLog.Log"
4-
$Global:wttLogFileName= ""
1+
$Global:ContinueOnFailure = $false
2+
$Global:JSONLogFile = "Run-Canary.JSON"
3+
$Global:TxtLogFile = "AzureStackCanaryLog.Log"
4+
$Global:wttLogFileName = ""
5+
$Global:listAvailableUsecases = $false
6+
$Global:exclusionList = @()
57
if (Test-Path -Path "$PSScriptRoot\..\WTTLog.ps1")
68
{
79
Import-Module -Name "$PSScriptRoot\..\WTTLog.ps1" -Force
@@ -10,7 +12,7 @@ if (Test-Path -Path "$PSScriptRoot\..\WTTLog.ps1")
1012

1113
$CurrentUseCase = @{}
1214
[System.Collections.Stack] $UseCaseStack = New-Object System.Collections.Stack
13-
filter timestamp {"$(Get-Date -Format HH:mm:ss.ffff): $_"}
15+
filter timestamp {"$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.ffff") $_"}
1416

1517

1618
function Log-Info
@@ -49,14 +51,11 @@ function Log-JSONReport
4951
[string] $Message
5052
)
5153
if ($Message)
52-
{
53-
if ($Message.Contains(": ["))
54-
{
55-
$time = $Message.Substring(0, $Message.IndexOf(": ["))
56-
}
54+
{
5755
if ($Message.Contains("[START]"))
5856
{
59-
$name = $Message.Substring($Message.LastIndexOf(":") + 1).Trim().Replace("######", "").Trim()
57+
$time = $Message.Substring(0, $Message.IndexOf("[")).Trim()
58+
$name = $Message.Substring($Message.LastIndexOf(":") + 1).Trim()
6059
if ($UseCaseStack.Count)
6160
{
6261
$nestedUseCase = @{
@@ -79,14 +78,19 @@ function Log-JSONReport
7978
}
8079
elseif ($Message.Contains("[END]"))
8180
{
81+
$time = $Message.Substring(0, $Message.IndexOf("[")).Trim()
8282
$result = ""
8383
if ($UseCaseStack.Peek().UseCase -and ($UseCaseStack.Peek().UseCase | Where-Object {$_.Result -eq "FAIL"}))
8484
{
8585
$result = "FAIL"
8686
}
87-
else
87+
elseif ($Message.Contains("[RESULT = PASS]"))
8888
{
89-
$result = $Message.Substring($Message.LastIndexOf("=") + 1).Trim().Replace("] ######", "").Trim()
89+
$result = "PASS"
90+
}
91+
elseif ($Message.Contains("[RESULT = FAIL]"))
92+
{
93+
$result = "FAIL"
9094
}
9195
$UseCaseStack.Peek().Add("Result", $result)
9296
$UseCaseStack.Peek().Add("EndTime", $time)
@@ -114,11 +118,53 @@ function Log-JSONReport
114118

115119
function Get-CanaryResult
116120
{
117-
$logContent = Get-Content -Raw -Path $Global:JSONLogFile | ConvertFrom-Json
118-
Log-Info ($logContent.UseCases | Format-Table -AutoSize @{Expression = {$_.Name}; Label = "Name"; Align = "Left"},
119-
@{Expression = {$_.Result}; Label="Result"; Align = "Left"},
120-
@{Expression = {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalSeconds}; Label = "Duration`n[Seconds]"; Align = "Left"},
121-
@{Expression = {$_.Description}; Label = "Description"; Align = "Left"})
121+
[CmdletBinding()]
122+
param(
123+
[parameter(Mandatory=$false)]
124+
[ValidateNotNullOrEmpty()]
125+
[string]$LogFilename
126+
)
127+
128+
if ($LogFilename)
129+
{
130+
$logContent = Get-Content -Raw -Path $LogFilename | ConvertFrom-Json
131+
}
132+
else
133+
{
134+
$logContent = Get-Content -Raw -Path $Global:JSONLogFile | ConvertFrom-Json
135+
}
136+
$results = @()
137+
foreach ($usecase in $logContent.UseCases)
138+
{
139+
$ucObj = New-Object -TypeName PSObject
140+
if ([bool]($usecase.PSobject.Properties.name -match "UseCase"))
141+
{
142+
$ucObj | Add-Member -Type NoteProperty -Name Name -Value $usecase.Name
143+
$ucObj | Add-Member -Type NoteProperty -Name Result -Value $usecase.Result
144+
$ucObj | Add-Member -Type NoteProperty -Name "Duration`n[Seconds]" -Value ((Get-Date $usecase.EndTime) - (Get-Date $usecase.StartTime)).TotalSeconds
145+
$ucObj | Add-Member -Type NoteProperty -Name Description -Value $usecase.Description
146+
$results += $ucObj
147+
148+
foreach ($subusecase in $usecase.UseCase)
149+
{
150+
$ucObj = New-Object -TypeName PSObject
151+
$ucObj | Add-Member -Type NoteProperty -Name Name -Value ("|-- $($subusecase.Name)")
152+
$ucObj | Add-Member -Type NoteProperty -Name Result -Value $subusecase.Result
153+
$ucObj | Add-Member -Type NoteProperty -Name "Duration`n[Seconds]" -Value ((Get-Date $subusecase.EndTime) - (Get-Date $subusecase.StartTime)).TotalSeconds
154+
$ucObj | Add-Member -Type NoteProperty -Name Description -Value ("|-- $($subusecase.Description)")
155+
$results += $ucObj
156+
}
157+
}
158+
else
159+
{
160+
$ucObj | Add-Member -Type NoteProperty -Name Name -Value $usecase.Name
161+
$ucObj | Add-Member -Type NoteProperty -Name Result -Value $usecase.Result
162+
$ucObj | Add-Member -Type NoteProperty -Name "Duration`n[Seconds]" -Value ((Get-Date $usecase.EndTime) - (Get-Date $usecase.StartTime)).TotalSeconds
163+
$ucObj | Add-Member -Type NoteProperty -Name Description -Value $usecase.Description
164+
$results += $ucObj
165+
}
166+
}
167+
Log-Info($results | Format-Table -AutoSize)
122168
}
123169

124170
function Get-CanaryLonghaulResult
@@ -147,6 +193,16 @@ function Get-CanaryLonghaulResult
147193
@{Expression={$pCount = ($_.Group | Where-Object Result -eq "PASS").Count; $times = ($_.Group | Where-Object Result -eq "PASS" | ForEach-Object {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalMilliseconds}); $avgTime = ($times | Measure-Object -Average).Average; $sd = 0; foreach ($time in $times){$sd += [math]::Pow(($time - $avgTime), 2)}; [math]::Round(([math]::Round([math]::Sqrt($sd/$pCount), 0)/$avgTime), 0) * 100};Label="RelativeStdDev`n[Goal: <50%]"; Align = "Left"}
148194
}
149195

196+
function Get-CanaryFailureStatus
197+
{
198+
$logContent = Get-Content -Raw -Path $Global:JSONLogFile | ConvertFrom-Json
199+
if ($logContent.Usecases.Result -contains "FAIL")
200+
{
201+
return $true
202+
}
203+
return $false
204+
}
205+
150206
function Start-Scenario
151207
{
152208
[CmdletBinding()]
@@ -161,7 +217,11 @@ function Start-Scenario
161217
[ValidateNotNullOrEmpty()]
162218
[string]$LogFilename,
163219
[parameter(Mandatory=$false)]
164-
[bool] $ContinueOnFailure = $false
220+
[bool]$ContinueOnFailure = $false,
221+
[parameter(Mandatory=$false)]
222+
[bool]$ListAvailable = $false,
223+
[parameter(Mandatory=$false)]
224+
[string[]]$ExclusionList
165225
)
166226

167227
if ($LogFileName)
@@ -181,14 +241,25 @@ function Start-Scenario
181241
{
182242
OpenWTTLogger $Global:wttLogFileName
183243
}
184-
185-
New-Item -Path $Global:JSONLogFile -Type File -Force
186-
New-Item -Path $Global:TxtLogFile -Type File -Force
187-
$jsonReport = @{
188-
"Scenario" = ($Name + "-" + $Type)
189-
"UseCases" = @()
190-
}
191-
$jsonReport | ConvertTo-Json -Depth 10 | Out-File -FilePath $Global:JSONLogFile
244+
if ($ListAvailable)
245+
{
246+
$Global:listAvailableUsecases = $true
247+
}
248+
if ($ExclusionList)
249+
{
250+
$Global:exclusionList = $ExclusionList
251+
}
252+
if (-not $ListAvailable)
253+
{
254+
New-Item -Path $Global:JSONLogFile -Type File -Force
255+
New-Item -Path $Global:TxtLogFile -Type File -Force
256+
$jsonReport = @{
257+
"Scenario" = ($Name + "-" + $Type)
258+
"UseCases" = @()
259+
}
260+
$jsonReport | ConvertTo-Json -Depth 10 | Out-File -FilePath $Global:JSONLogFile
261+
}
262+
192263
$Global:ContinueOnFailure = $ContinueOnFailure
193264
}
194265

@@ -214,7 +285,33 @@ function Invoke-Usecase
214285
[ValidateNotNullOrEmpty()]
215286
[ScriptBlock]$UsecaseBlock
216287
)
217-
Log-Info ("###### [START] Usecase: $Name ######`n")
288+
289+
if ($Global:listAvailableUsecases)
290+
{
291+
$parentUsecase = $Name
292+
if ((Get-PSCallStack)[1].Arguments.Contains($parentUsecase))
293+
{
294+
" `t"*([math]::Floor((Get-PSCallStack).Count/3)) + "|-- " + $Name
295+
}
296+
else
297+
{
298+
299+
"`t" + $Name
300+
}
301+
if ($UsecaseBlock.ToString().Contains("Invoke-Usecase"))
302+
{
303+
try {Invoke-Command -ScriptBlock $UsecaseBlock -ErrorAction SilentlyContinue}
304+
catch {}
305+
}
306+
return
307+
}
308+
309+
if (($Global:exclusionList).Contains($Name))
310+
{
311+
Log-Info ("Skipping Usecase: $Name")
312+
return
313+
}
314+
Log-Info ("[START] Usecase: $Name`n")
218315
if ($Global:wttLogFileName)
219316
{
220317
StartTest "CanaryGate:$Name"
@@ -236,7 +333,7 @@ function Invoke-Usecase
236333
{
237334
EndTest "CanaryGate:$Name" $true
238335
}
239-
Log-Info ("###### [END] Usecase: $Name ###### [RESULT = PASS] ######`n")
336+
Log-Info ("[END] [RESULT = PASS] Usecase: $Name`n")
240337
return $result | Out-Null
241338
}
242339
catch [System.Exception]
@@ -245,7 +342,7 @@ function Invoke-Usecase
245342
Log-Info ("###### <FAULTING SCRIPTBLOCK> ######")
246343
Log-Info ("$UsecaseBlock")
247344
Log-Info ("###### </FAULTING SCRIPTBLOCK> ######")
248-
Log-Error ("###### [END] Usecase: $Name ###### [RESULT = FAIL] ######`n")
345+
Log-Error ("[END] [RESULT = FAIL] Usecase: $Name`n")
249346
if ($Global:wttLogFileName)
250347
{
251348
EndTest "CanaryGate:$Name" $false

‎CanaryValidator/README.md‎

Lines changed: 110 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,16 @@ Canary validator provides a breadth customer experience with the Azure Stack dep
44
Instructions are relative to the .\CanaryValidator directory.
55
Canary can be invoked either as Service Administrator or Tenant Administrator.
66

7-
# Download Canary
7+
## Download Canary
8+
89
```powershell
910
Invoke-WebRequest https://github.com/Azure/AzureStack-Tools/archive/master.zip -OutFile master.zip
1011
Expand-Archive master.zip -DestinationPath . -Force
1112
Set-Location -Path ".\AzureStack-Tools-master\CanaryValidator" -PassThru
1213
```
1314

14-
# To execute Canary as Tenant Administrator (if Windows Server 2016 or Windows Server 2012-R2 images are already present in the PIR)
15+
## To execute Canary as Tenant Administrator (if Windows Server 2016 or Windows Server 2012-R2 images are already present in the PIR)
16+
1517
```powershell
1618
# Install-Module -Name 'AzureRm.Bootstrapper' -Scope CurrentUser
1719
# Install-AzureRmProfile -profile '2017-03-09-profile' -Force -Scope CurrentUser
@@ -21,7 +23,8 @@ $ServiceAdminCreds = New-Object System.Management.Automation.PSCredential "<Ser
2123
.\Canary.Tests.ps1 -TenantID "<TenantID from Azure Active Directory>" -AdminArmEndpoint "<Administrative ARM endpoint>" -ServiceAdminCredentials $ServiceAdminCreds -TenantArmEndpoint "<Tenant ARM endpoint>" -TenantAdminCredentials $TenantAdminCreds
2224
```
2325

24-
# To execute Canary as Tenant Administrator (if Windows Server 2016 or Windows Server 2012-R2 images are not present in PIR)
26+
## To execute Canary as Tenant Administrator (if Windows Server 2016 or Windows Server 2012-R2 images are not present in PIR)
27+
2528
```powershell
2629
# Download the WS2016 ISO image from: https://www.microsoft.com/en-us/evalcenter/evaluate-windows-server-2016, and place it on your local machine
2730
# Install-Module -Name 'AzureRm.Bootstrapper' -Scope CurrentUser
@@ -32,24 +35,124 @@ $ServiceAdminCreds = New-Object System.Management.Automation.PSCredential "<Ser
3235
.\Canary.Tests.ps1 -TenantID "<TenantID from Azure Active Directory>" -AdminArmEndpoint "<Administrative ARM endpoint>" -ServiceAdminCredentials $ServiceAdminCreds -TenantArmEndpoint "<Tenant ARM endpoint>" -TenantAdminCredentials $TenantAdminCreds -WindowsISOPath "<path where the WS2016 ISO is present>"
3336
```
3437

35-
# To execute Canary as Service Administrator
38+
## To execute Canary as Service Administrator
39+
3640
```powershell
3741
# Install-Module -Name 'AzureRm.Bootstrapper' -Scope CurrentUser
3842
# Install-AzureRmProfile -profile '2017-03-09-profile' -Force -Scope CurrentUser
3943
# Install-Module -Name AzureStack -RequiredVersion 1.2.9 -Scope CurrentUser
4044
$ServiceAdminCreds = New-Object System.Management.Automation.PSCredential "<Service Admin username>", (ConvertTo-SecureString "<Service Admin password>" -AsPlainText -Force)
4145
.\Canary.Tests.ps1 -TenantID "<TenantID from Azure Active Directory>" -AdminArmEndpoint "<Administrative ARM endpoint>" -ServiceAdminCredentials $ServiceAdminCreds
4246
```
43-
# Reading the results & logs
47+
48+
## To list the usecases in Canary
49+
50+
```powershell
51+
# Install-Module -Name 'AzureRm.Bootstrapper' -Scope CurrentUser
52+
# Install-AzureRmProfile -profile '2017-03-09-profile' -Force -Scope CurrentUser
53+
# Install-Module -Name AzureStack -RequiredVersion 1.2.9 -Scope CurrentUser
54+
.\Canary.Tests.ps1 -ListAvailable
55+
56+
Sample output:
57+
PS C:\AzureStack-Tools-vnext\CanaryValidator> .\Canary.Tests.ps1 -ListAvailable
58+
List of scenarios in Canary:
59+
CreateAdminAzureStackEnv
60+
LoginToAzureStackEnvAsSvcAdmin
61+
SelectDefaultProviderSubscription
62+
ListFabricResourceProviderInfo
63+
|--GetAzureStackInfraRole
64+
|--GetAzureStackInfraRoleInstance
65+
|--GetAzureStackLogicalNetwork
66+
|--GetAzureStackStorageCapacity
67+
|--GetAzureStackStorageShare
68+
|--GetAzureStackScaleUnit
69+
|--GetAzureStackScaleUnitNode
70+
|--GetAzureStackIPPool
71+
|--GetAzureStackMacPool
72+
|--GetAzureStackGatewayPool
73+
|--GetAzureStackSLBMux
74+
|--GetAzureStackGateway
75+
ListHealthResourceProviderAlerts
76+
|--GetAzureStackAlert
77+
ListUpdatesResourceProviderInfo
78+
|--GetAzureStackUpdateSummary
79+
|--GetAzureStackUpdateToApply
80+
CreateTenantAzureStackEnv
81+
CreateResourceGroupForTenantSubscription
82+
CreateTenantPlan
83+
CreateTenantOffer
84+
CreateTenantDefaultManagedSubscription
85+
LoginToAzureStackEnvAsTenantAdmin
86+
CreateTenantSubscription
87+
RoleAssignmentAndCustomRoleDefinition
88+
|--ListAssignedRoles
89+
|--ListExistingRoleDefinitions
90+
|--GetProviderOperations
91+
|--AssignReaderRole
92+
|--VerifyReaderRoleAssignment
93+
|--RemoveReaderRoleAssignment
94+
|--CustomRoleDefinition
95+
|--ListRoleDefinitionsAfterCustomRoleCreation
96+
|--RemoveCustomRoleDefinition
97+
RegisterResourceProviders
98+
CreateResourceGroupForUtilities
99+
CreateStorageAccountForUtilities
100+
CreateStorageContainerForUtilities
101+
CreateDSCScriptResourceUtility
102+
CreateCustomScriptResourceUtility
103+
CreateDataDiskForVM
104+
UploadUtilitiesToBlobStorage
105+
CreateKeyVaultStoreForCertSecret
106+
CreateResourceGroupForVMs
107+
DeployARMTemplate
108+
RetrieveResourceDeploymentTimes
109+
QueryTheVMsDeployed
110+
CheckVMCommunicationPreVMReboot
111+
AddDatadiskToVMWithPrivateIP
112+
|--StopDeallocateVMWithPrivateIPBeforeAddingDatadisk
113+
|--AddTheDataDiskToVMWithPrivateIP
114+
|--StartVMWithPrivateIPAfterAddingDatadisk
115+
ApplyDataDiskCheckCustomScriptExtensionToVMWithPrivateIP
116+
|--CheckForExistingCustomScriptExtensionOnVMWithPrivateIP
117+
|--ApplyCustomScriptExtensionToVMWithPrivateIP
118+
RestartVMWithPublicIP
119+
StopDeallocateVMWithPrivateIP
120+
StartVMWithPrivateIP
121+
CheckVMCommunicationPostVMReboot
122+
CheckExistenceOfScreenShotForVMWithPrivateIP
123+
EnumerateAllResources
124+
DeleteVMWithPrivateIP
125+
DeleteVMResourceGroup
126+
DeleteUtilitiesResourceGroup
127+
TenantRelatedcleanup
128+
|--DeleteTenantSubscriptions
129+
|--LoginToAzureStackEnvAsSvcAdminForCleanup
130+
|--RemoveLinuxImageFromPIR
131+
|--DeleteSubscriptionResourceGroup
132+
```
133+
134+
## To exclude certain usecases from getting executed
135+
136+
```powershell
137+
# Install-Module -Name 'AzureRm.Bootstrapper' -Scope CurrentUser
138+
# Install-AzureRmProfile -profile '2017-03-09-profile' -Force -Scope CurrentUser
139+
# Install-Module -Name AzureStack -RequiredVersion 1.2.9 -Scope CurrentUser
140+
# A new paramter called ExclusionList has been added which is a string array. Pass in the list of usecases you don't want to execute to this parameter.
141+
$ServiceAdminCreds = New-Object System.Management.Automation.PSCredential "<Service Admin username>", (ConvertTo-SecureString "<Service Admin password>" -AsPlainText -Force)
142+
.\Canary.Tests.ps1 -TenantID "<TenantID from Azure Active Directory>" -AdminArmEndpoint "<Administrative ARM endpoint>" -ServiceAdminCredentials $ServiceAdminCreds -ExclusionList "ListFabricResourceProviderInfo","ListUpdateResourceProviderInfo"
143+
```
144+
145+
## Reading the results & logs
146+
44147
Canary generates log files in the TMP directory ($env:TMP). The logs can be found under the directory "CanaryLogs[DATETIME]". There are two types of logs generated, a text log and a JSON log. JSON log provides a quick and easy view of all the usecases and their corresponding results. Text log provides a more detailed output of each usecase execution, its output and results.
45148

46149
Each usecase entry in the JSON log consists of the following fields.
150+
47151
- Name
48152
- Description
49153
- StartTime
50154
- EndTime
51155
- Result
52156
- Exception (in case a scenario fails)
53157

54-
The exception field is helpful to debug failed usecases.
55-
158+
The exception field is helpful to debug failed use cases.

0 commit comments

Comments
 (0)