Merge pull request #2556 from microsoft/dpaul-PesterUpdate

Replace E19/E16 Pester tests with Exchange SE coverage + priority scheduling

Author: dpaulson45

.build/Pester.ps1

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseUsingScopeModifierInNewRunspaces', '', Justification = 'Variable passed via -ArgumentList param()')]
 [CmdletBinding()]
 param(
     [switch]
@@ -18,11 +19,10 @@ begin {
         throw "Pester module could not be loaded"
     }
 
-    $jobsQueued = New-Object 'System.Collections.Generic.Queue[object]'
-    $childIds = 1
     $jobsCompleted = @{}
     $jobsProgress = @{}
-    $jobsRunning = @()
+    $jobsRunning = New-Object 'System.Collections.Generic.List[PSCustomObject]'
+    $childIds = 1
     # on Azure pipeline we have noticed 2 or 4 cores available. to get the most of out jobs, need at least a min of 2 threads running.
     $jobQueueMaxConcurrency = [System.Math]::Max(([System.Math]::Min(([System.Environment]::ProcessorCount - 1), 5)), 2)
     Write-Host "Max Job Threads: $jobQueueMaxConcurrency"
@@ -44,69 +44,139 @@ begin {
     $scripts = @(Get-ChildItem -Recurse $root |
             Where-Object { $_.Name -like "*.Tests.ps1" -and $_.FullName -notmatch "\.github" })
 
+    # Categorize test files by priority using comment tag: # PesterPriority: High
+    $highPriorityQueue = New-Object 'System.Collections.Generic.Queue[object]'
+    $lowPriorityQueue = New-Object 'System.Collections.Generic.Queue[object]'
+
+    foreach ($script in $scripts) {
+        $firstLines = Get-Content $script.FullName -TotalCount 15
+        if ($firstLines -match "# PesterPriority: High") {
+            $highPriorityQueue.Enqueue($script)
+        } else {
+            $lowPriorityQueue.Enqueue($script)
+        }
+    }
+
+    Write-Host "High Priority Tests: $($highPriorityQueue.Count) | Low Priority Tests: $($lowPriorityQueue.Count)"
+
+    # Thread allocation: high priority gets up to (threads - 1) slots, always leaving at least 1 for low.
+    # This ensures heavy tests start immediately while small tests still make progress.
+    $highThreadMax = [System.Math]::Min($highPriorityQueue.Count, $jobQueueMaxConcurrency - 1)
+    $lowThreadMax = $jobQueueMaxConcurrency - $highThreadMax
+
+    # If no high priority tests exist, all threads go to low priority
+    if ($highPriorityQueue.Count -eq 0) {
+        $lowThreadMax = $jobQueueMaxConcurrency
+        $highThreadMax = 0
+    }
+
+    Write-Host "Thread Allocation: High=$highThreadMax Low=$lowThreadMax"
+
     $parentProgress = @{
         Id              = 0
         Activity        = "Running Pester Tests"
-        Status          = [string]::Empty
+        Status          = "Initializing"
         PercentComplete = 0
     }
 
-    $scripts | ForEach-Object {
-        $jobsQueued.Enqueue(@{
-                ScriptBlock  = {
-                    param(
-                        [string]$FileName
-                    )
-                    return Invoke-Pester -Path $FileName -PassThru
-                }
-                ArgumentList = $_.FullName
-                Name         = $_.Name
-            })
-    }
-
-    $parentProgress.PercentComplete = ($jobsCompleted.Count / $scripts.Count * 100)
-    $parentProgress.Status = "Number of Jobs Running $($jobsRunning.Count)"
-
     if (-not $NoProgress) {
         Write-Progress @parentProgress
     }
 
-    while ($jobsQueued.Count -gt 0 -or $jobsRunning.Count -gt 0) {
+    while ($highPriorityQueue.Count -gt 0 -or $lowPriorityQueue.Count -gt 0 -or $jobsRunning.Count -gt 0) {
+
+        # Determine if low priority work is fully drained (queue empty AND no low jobs running)
+        $lowDrained = ($lowPriorityQueue.Count -eq 0 -and
+            (@($jobsRunning | Where-Object { $_.Priority -eq "Low" }).Count -eq 0))
+
+        $runningHigh = @($jobsRunning | Where-Object { $_.Priority -eq "High" }).Count
+        $runningLow = @($jobsRunning | Where-Object { $_.Priority -eq "Low" }).Count
+
+        # Once low priority is drained, all threads become available for high priority
+        $effectiveHighMax = if ($lowDrained) { $jobQueueMaxConcurrency } else { $highThreadMax }
+        $effectiveLowMax = if ($lowDrained) { 0 } else { $lowThreadMax }
+
+        # Start new jobs up to concurrency limit
+        while ($jobsRunning.Count -lt $jobQueueMaxConcurrency) {
+            $nextScript = $null
+            $nextPriority = $null
+
+            # Try high priority first if under high thread limit
+            if ($runningHigh -lt $effectiveHighMax -and $highPriorityQueue.Count -gt 0) {
+                $nextScript = $highPriorityQueue.Dequeue()
+                $nextPriority = "High"
+            } elseif ($runningLow -lt $effectiveLowMax -and $lowPriorityQueue.Count -gt 0) {
+                $nextScript = $lowPriorityQueue.Dequeue()
+                $nextPriority = "Low"
+            } elseif ($highPriorityQueue.Count -gt 0 -and $runningHigh -lt $jobQueueMaxConcurrency) {
+                # Fill any remaining slots with high priority
+                $nextScript = $highPriorityQueue.Dequeue()
+                $nextPriority = "High"
+            } elseif ($lowPriorityQueue.Count -gt 0 -and $runningLow -lt $jobQueueMaxConcurrency) {
+                # Fill any remaining slots with low priority
+                $nextScript = $lowPriorityQueue.Dequeue()
+                $nextPriority = "Low"
+            }
+
+            if ($null -eq $nextScript) { break }
+
+            if ($VerbosePreference -eq "Continue") {
+                $elapsed = [math]::Round($stopWatch.Elapsed.TotalSeconds, 1)
+                Write-Verbose "[DEBUG T+${elapsed}s] Starting [$nextPriority] $($nextScript.Name) (H:$runningHigh/$effectiveHighMax L:$runningLow/$effectiveLowMax)"
+            }
+
+            $newJob = Start-Job -ScriptBlock {
+                param([string]$FileName)
+                return Invoke-Pester -Path $FileName -PassThru
+            } -ArgumentList $nextScript.FullName -Name $nextScript.Name
+
+            $jobsRunning.Add([PSCustomObject]@{
+                    Job      = $newJob
+                    Priority = $nextPriority
+                    Name     = $nextScript.Name
+                })
 
-        if ($jobsRunning.Count -lt $jobQueueMaxConcurrency -and $jobsQueued.Count -gt 0) {
-            $jobArgs = $jobsQueued.Dequeue()
-            # Using Start-Job instead of Start-ThreadJob as this is faster for this script block
-            # If Start-ThreadJob is used, need to have $justFinished also use NotStarted State Filter
-            $newJob = Start-Job @jobArgs
-            $jobsRunning += $newJob
             $progress = @{
                 Id       = $childIds++
                 ParentId = 0
-                Activity = "Running: $($newJob.Name)"
+                Activity = "Running [$nextPriority]: $($nextScript.Name)"
             }
             $jobsProgress.Add($newJob.Name, $progress)
 
             if (-not $NoProgress) {
                 Write-Progress @progress
             }
+
+            $runningHigh = @($jobsRunning | Where-Object { $_.Priority -eq "High" }).Count
+            $runningLow = @($jobsRunning | Where-Object { $_.Priority -eq "Low" }).Count
         }
 
-        $justFinished = @($jobsRunning | Where-Object { $_.State -ne "Running" })
+        # Check for completed jobs
+        $justFinished = @($jobsRunning | Where-Object { $_.Job.State -ne "Running" })
 
         if ($justFinished.Count -gt 0) {
-            foreach ($job in $justFinished) {
-                $result = Receive-Job $job
-                $jobsCompleted.Add($job.Name, [PSCustomObject]@{
-                        Job    = $job
-                        Result = $result
+            foreach ($item in $justFinished) {
+                $result = Receive-Job $item.Job
+                $jobsCompleted.Add($item.Name, [PSCustomObject]@{
+                        Job      = $item.Job
+                        Priority = $item.Priority
+                        Result   = $result
                     })
-                $progress = $jobsProgress[$job.Name]
+                $progress = $jobsProgress[$item.Name]
 
                 if (-not $NoProgress) {
                     Write-Progress @progress -Completed
                 }
-                Write-Host $job.Name "job finished."
-                Remove-Job $job -Force
+
+                $jobDuration = [math]::Round(($item.Job.PSEndTime - $item.Job.PSBeginTime).TotalSeconds, 1)
+                Write-Host "[$($item.Priority)] $($item.Name) job finished. (${jobDuration}s)"
+
+                if ($VerbosePreference -eq "Continue") {
+                    $elapsed = [math]::Round($stopWatch.Elapsed.TotalSeconds, 1)
+                    Write-Verbose "[DEBUG T+${elapsed}s] Completed [$($item.Priority)] $($item.Name) - Pester: $([math]::Round($result.Duration.TotalSeconds, 1))s Job: ${jobDuration}s"
+                }
+
+                Remove-Job $item.Job -Force
                 $result
 
                 if ($result.Result -eq "Failed" -or
@@ -115,18 +185,28 @@ begin {
                 }
             }
 
-            $jobsRunning = @($jobsRunning | Where-Object { -not $justFinished.Contains($_) })
+            foreach ($item in $justFinished) {
+                $jobsRunning.Remove($item) | Out-Null
+            }
         }
 
+        $highRunning = @($jobsRunning | Where-Object { $_.Priority -eq "High" }).Count
+        $lowRunning = @($jobsRunning | Where-Object { $_.Priority -eq "Low" }).Count
+        $highCompleted = @($jobsCompleted.Values | Where-Object { $_.Priority -eq "High" }).Count
+        $lowCompleted = @($jobsCompleted.Values | Where-Object { $_.Priority -eq "Low" }).Count
+        $highTotal = $highCompleted + $highRunning + $highPriorityQueue.Count
+        $lowTotal = $lowCompleted + $lowRunning + $lowPriorityQueue.Count
+
         $parentProgress.PercentComplete = ($jobsCompleted.Count / $scripts.Count * 100)
-        $parentProgress.Status = "Number of Jobs Running $($jobsRunning.Count)"
+        $parentProgress.Status = "Running: $($jobsRunning.Count) | High: $highRunning running, $highCompleted/$highTotal done | Low: $lowRunning running, $lowCompleted/$lowTotal done"
 
         if (-not $NoProgress) {
             Write-Progress @parentProgress
         }
 
-        if ($jobsRunning.Count -eq $jobQueueMaxConcurrency -or $jobsQueued.Count -eq 0) {
-            Start-Sleep 1
+        if ($jobsRunning.Count -ge $jobQueueMaxConcurrency -or
+            ($highPriorityQueue.Count -eq 0 -and $lowPriorityQueue.Count -eq 0)) {
+            Start-Sleep -Milliseconds 500
         }
     }
 } end {
@@ -144,7 +224,7 @@ begin {
         $value = $jobsCompleted[$job]
         $totalSeconds = ($value.Job.PSEndTime - $value.Job.PSBeginTime).TotalSeconds
         $sumTotalPesterSeconds += $value.Result.Duration.TotalSeconds
-        Write-Host "$job took $totalSeconds seconds to complete"
+        Write-Host "[$($value.Priority)] $job took $totalSeconds seconds to complete"
         $sumTotalSeconds += $totalSeconds
 
         if ($value.Result.Result -eq "Failed") {
@@ -157,9 +237,29 @@ begin {
     Write-Host
     Write-Host "Total seconds for jobs: $sumTotalSeconds"
     Write-Host "Total seconds for pester results: $sumTotalPesterSeconds"
-    Write-Host "Average seconds per threads allowed: $($sumTotalSeconds/ $jobQueueMaxConcurrency)"
+    Write-Host "Average seconds per threads allowed: $($sumTotalSeconds / $jobQueueMaxConcurrency)"
     Write-Host "Total Seconds script took: $($stopWatch.Elapsed.TotalSeconds)"
 
+    if ($VerbosePreference -eq "Continue") {
+        $highJobs = $jobsCompleted.Values | Where-Object { $_.Priority -eq "High" }
+        $lowJobs = $jobsCompleted.Values | Where-Object { $_.Priority -eq "Low" }
+        $highSum = ($highJobs | ForEach-Object { ($_.Job.PSEndTime - $_.Job.PSBeginTime).TotalSeconds } | Measure-Object -Sum).Sum
+        $lowSum = ($lowJobs | ForEach-Object { ($_.Job.PSEndTime - $_.Job.PSBeginTime).TotalSeconds } | Measure-Object -Sum).Sum
+        $longestJob = $jobsCompleted.Values | Sort-Object { ($_.Job.PSEndTime - $_.Job.PSBeginTime).TotalSeconds } | Select-Object -Last 1
+        $longestDuration = [math]::Round(($longestJob.Job.PSEndTime - $longestJob.Job.PSBeginTime).TotalSeconds, 1)
+        $efficiency = [math]::Round(($sumTotalSeconds / ($stopWatch.Elapsed.TotalSeconds * $jobQueueMaxConcurrency)) * 100, 1)
+
+        Write-Verbose ""
+        Write-Verbose "=== Scheduling Debug ==="
+        Write-Verbose "High priority: $($highJobs.Count) jobs, $([math]::Round($highSum, 1))s total"
+        Write-Verbose "Low priority: $($lowJobs.Count) jobs, $([math]::Round($lowSum, 1))s total"
+        Write-Verbose "Longest job: [$($longestJob.Priority)] $($longestJob.Job.Name) (${longestDuration}s)"
+        Write-Verbose "Thread utilization: $efficiency% (ideal=100%)"
+        Write-Verbose "Theoretical minimum: $([math]::Round($sumTotalSeconds / $jobQueueMaxConcurrency, 1))s"
+        Write-Verbose "Actual wall clock: $([math]::Round($stopWatch.Elapsed.TotalSeconds, 1))s"
+        Write-Verbose "Overhead: $([math]::Round($stopWatch.Elapsed.TotalSeconds - ($sumTotalSeconds / $jobQueueMaxConcurrency), 1))s"
+    }
+
     if ($failPipeline) {
         throw "Failed Pester Testing Results"
     }

Diagnostics/HealthChecker/Analyzer/Tests/Get-URLRewriteRule.Tests.ps1

@@ -11,7 +11,7 @@ BeforeAll {
     Invoke-Expression $scriptContent
     function Invoke-CatchActions { throw "Called Invoke-CatchActions" }
 
-    $Script:mockDataRoot = "$PSScriptRoot\..\..\Tests\DataCollection\E19\Exchange\IIS"
+    $Script:mockDataRoot = "$PSScriptRoot\..\..\Tests\DataCollection\ExchangeSE\Exchange\IIS"
     [xml]$Script:appHost = Get-Content "$Script:mockDataRoot\applicationHost.config" -Raw -Encoding UTF8
 
     $Script:webConfigContent = @{