parallelize CI tests (#835)

tgauth · web-flow · commit fbfe284c9019 · 2026-04-17T15:03:17.000-04:00
* parallelize CI tests

* address copilot feedback
diff --git a/.azdo/ci.yml b/.azdo/ci.yml
@@ -74,10 +74,10 @@ stages:
   displayName: Test Win32-OpenSSH
   dependsOn: Build
   jobs:
-  - job: TestPkgWin32OpenSSH
+  - job: TestPkgWin32OpenSSHCore
     pool:
       vmImage: windows-latest
-    displayName: Win32-OpenSSH On Windows
+    displayName: Win32-OpenSSH Core Tests On Windows
     variables:
       testFilesDrivePath: '**'
     steps:
@@ -97,14 +97,6 @@ stages:
         artifactName: UnitTests-x64
         downloadPath: '$(System.ArtifactsDirectory)'
 
-    - task: DownloadBuildArtifacts@0
-      displayName: 'Download bash test config file artifact'
-      inputs:
-        buildType: current
-        downloadType: single
-        artifactName: ConfigFile-x64
-        downloadPath: '$(System.ArtifactsDirectory)'
-
     - pwsh: |
         $artifactDir = "$(System.ArtifactsDirectory)"
         Write-Verbose -Verbose -Message "Artifacts directory: $artifactDir"
@@ -121,13 +113,6 @@ stages:
         Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose
       displayName: Install Unit Tests
 
-    - pwsh: |
-        $configFileSrc = "$(System.ArtifactsDirectory)/ConfigFile-x64/config.h"
-        $configFileDest = "$(Build.SourcesDirectory)"
-        Write-Verbose -Verbose -Message "Copying config file from: ${configFileSrc} to: ${configFileDest}"
-        Copy-Item -Path $configFileSrc -Dest $configFileDest -Force
-      displayName: Copy config file artifact for bash tests
-
     - pwsh: |
         $sourceDir = "$(Build.SourcesDirectory)"
         Write-Verbose -Verbose -Message "Source repo directory: $sourceDir"
@@ -144,7 +129,7 @@ stages:
         # Run OpenSSH tests
         Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
         Invoke-OpenSSHTests -OpenSSHBinPath "$env:SystemDrive/OpenSSH"
-      displayName: Run tests
+      displayName: Run core tests
 
     - pwsh: |
         Write-Host "##vso[task.setvariable variable=testFilesDrivePath;]$env:SystemDrive"
@@ -167,7 +152,86 @@ stages:
         # Upload test results artifact
         if (Test-Path -Path $ResultsDirectory)
         {
-          $artifactName = 'Win32-OpenSSH-TestResults'
+          $artifactName = 'Win32-OpenSSH-TestResults-Core'
+          Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$ResultsDirectory"
+        }
+      displayName: Upload test results
+      condition: always()
+
+    - pwsh: |
+        Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
+        Clear-TestEnvironmentSetup
+      displayName: Clean up OpenSSH test environment
+      condition: always()
+
+  - job: TestPkgWin32OpenSSHBash
+    pool:
+      vmImage: windows-latest
+    displayName: Win32-OpenSSH Bash Tests On Windows
+    steps:
+    - task: DownloadBuildArtifacts@0
+      displayName: 'Download build artifacts'
+      inputs:
+        buildType: current
+        downloadType: single
+        artifactName: Win32-OpenSSH-x64
+        downloadPath: '$(System.ArtifactsDirectory)'
+
+    - task: DownloadBuildArtifacts@0
+      displayName: 'Download bash test config file artifact'
+      inputs:
+        buildType: current
+        downloadType: single
+        artifactName: ConfigFile-x64
+        downloadPath: '$(System.ArtifactsDirectory)'
+
+    - pwsh: |
+        $artifactDir = "$(System.ArtifactsDirectory)"
+        Write-Verbose -Verbose -Message "Artifacts directory: $artifactDir"
+        Get-ChildItem -Path $artifactDir -Recurse
+      displayName: Capture downloaded artifact directory
+
+    - pwsh: |
+        Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
+        Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose
+      displayName: Install Win32-OpenSSH
+
+    - pwsh: |
+        $configFileSrc = "$(System.ArtifactsDirectory)/ConfigFile-x64/config.h"
+        $configFileDest = "$(Build.SourcesDirectory)"
+        Write-Verbose -Verbose -Message "Copying config file from: ${configFileSrc} to: ${configFileDest}"
+        Copy-Item -Path $configFileSrc -Dest $configFileDest -Force
+      displayName: Copy config file artifact for bash tests
+
+    - pwsh: |
+        $sourceDir = "$(Build.SourcesDirectory)"
+        Write-Verbose -Verbose -Message "Source repo directory: $sourceDir"
+        Get-ChildItem -Path $sourceDir
+      displayName: Capture source repo directory for test
+
+    - pwsh: |
+        $installedOpenSSHDir = "$env:SystemDrive/OpenSSH"
+        Write-Verbose -Verbose -Message "Installed OpenSSH directory: $installedOpenSSHDir"
+        Get-ChildItem -Path $installedOpenSSHDir -Recurse
+      displayName: Capture installed OpenSSH directory
+
+    - pwsh: |
+        # Run OpenSSH bash tests
+        Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
+        Invoke-OpenSSHBashTestsOnly -OpenSSHBinPath "$env:SystemDrive/OpenSSH"
+      displayName: Run bash tests
+
+    - pwsh: |
+        Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
+        #
+        # Copy test results to results directory
+        $ResultsDirectory = "$(Build.SourcesDirectory)/Win32OpenSSHTestResults"
+        Copy-OpenSSHTestResults -ResultsPath $ResultsDirectory
+        #
+        # Upload test results artifact
+        if (Test-Path -Path $ResultsDirectory)
+        {
+          $artifactName = 'Win32-OpenSSH-TestResults-Bash'
           Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$ResultsDirectory"
         }
       displayName: Upload test results
diff --git a/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psd1 b/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psd1
@@ -31,6 +31,7 @@ FunctionsToExport = @(
     'Invoke-AzDOBuild',
     'Install-OpenSSH',
     'Invoke-OpenSSHTests',
+    'Invoke-OpenSSHBashTestsOnly',
     'Copy-OpenSSHTestResults',
     'Copy-BuildResults',
     'Copy-UnitTests',
diff --git a/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psm1 b/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psm1
@@ -162,6 +162,42 @@ function Install-CygWin
     choco install cygwin -y --force --params "/InstallDir:${InstallLocation} /NoStartMenu"
 }
 
+function Ensure-CygWinInstalled
+{
+    [CmdletBinding()]
+    param (
+        [string] $InstallLocation = "$env:SystemDrive/cygwin",
+        [Parameter(Mandatory=$true)]
+        [string] $FailureMessage
+    )
+
+    if (Test-Path -Path "$InstallLocation/bin/sh.exe")
+    {
+        return $true
+    }
+
+    Write-Verbose -Verbose -Message "CygWin not found"
+    Install-CygWin -InstallLocation $InstallLocation
+
+    $expectedCygWinPath = "$InstallLocation/bin/sh.exe"
+    if (Test-Path -Path $expectedCygWinPath)
+    {
+        return $true
+    }
+
+    Write-Verbose -Verbose -Message "CygWin did not install correctly, missing expected path: ${expectedCygWinPath}"
+
+    $cygWinDirs = Get-Item -Path "$env:SystemDrive/cygwin*" -ErrorAction SilentlyContinue
+    if ($cygWinDirs.Count -gt 1)
+    {
+        Write-Verbose -Verbose -Message "CygWin install failed with mangled folder locations: ${cygWinDirs}"
+        Write-Verbose -Verbose -Message 'TODO: Add hack to fix up CygWin folder.'
+    }
+
+    Write-BuildMessage -Message $FailureMessage -Category Error
+    return $false
+}
+
 <#
       .Synopsis
       Runs the tests for this repo
@@ -229,34 +265,7 @@ function Invoke-OpenSSHTests
     Write-Verbose -Verbose -Message "Running E2E Tests..."
     Set-OpenSSHTestEnvironment -Confirm:$false
 
-    # Ensure CygWin is installed, and install from Chocolatey if needed.
-    # used for bash tests and default shell pester tests
-    $cygwinInstalled = $true
-    $cygwinInstallLocation = "$env:SystemDrive/cygwin"
-    if (! (Test-Path -Path "$cygwinInstallLocation/bin/sh.exe"))
-    {
-        Write-Verbose -Verbose -Message "CygWin not found"
-        Install-CygWin -InstallLocation $cygwinInstallLocation
-
-        # Hack to fix up mangled CygWin directory, if needed.
-        $expectedCygWinPath = "$env:SystemDrive/cygwin/bin/sh.exe"
-        if (! (Test-Path -Path $expectedCygWinPath))
-        {
-            Write-Verbose -Verbose -Message "CygWin did not install correctly, missing expected path: ${expectedCygWinPath}"
-
-            $cygWinDirs = Get-Item -Path "$env:SystemDrive/cygwin*"
-            if ($cygWinDirs.Count -gt 1)
-            {
-                Write-Verbose -Verbose -Message "CygWin install failed with mangled folder locations: ${cygWinDirs}"
-                Write-Verbose -Verbose -Message 'TODO: Add hack to fix up CygWin folder.'
-            }
-
-            Write-BuildMessage -Message "All bash tests failed because CygWin install failed" -Category Error
-            $AllTestsPassed = $false
-            $cygwinInstalled = $false
-        }
-    }
-
+    $cygwinInstalled = Ensure-CygWinInstalled -FailureMessage "All E2E tests failed because CygWin install failed"
     if ($cygwinInstalled)
     {
         Invoke-OpenSSHE2ETest
@@ -279,33 +288,10 @@ function Invoke-OpenSSHTests
                 Write-BuildMessage -Message "All E2E tests passed!" -Category Information
             }
         }
-
-        # Bash tests.
-        Write-Verbose -Verbose -Message "Running Bash Tests..."
-
-        # Run UNIX bash tests.
-        Write-Verbose -Verbose -Message "Starting Bash Tests..."
-        Invoke-OpenSSHBashTests
-        if (-not $Global:bash_tests_summary)
-        {
-            $errorMessage = "Failed to start OpenSSH bash tests"
-            Write-BuildMessage -Message $errorMessage -Category Error
-            $AllTestsPassed = $false
-        }
-        else
-        {
-            if ($Global:bash_tests_summary["TotalBashTestsFailed"] -ne 0)
-            {
-                $total_bash_failed_tests = $Global:bash_tests_summary["TotalBashTestsFailed"]
-                $total_bash_tests = $Global:bash_tests_summary["TotalBashTests"]
-                $errorMessage = "At least one of the bash tests failed. [$total_bash_failed_tests of $total_bash_tests]"
-                Write-BuildMessage -Message $errorMessage -Category Error
-                $AllTestsPassed = $false
-            }
-
-            $OpenSSHTestInfo["BashTestSummaryFile"] = $Global:bash_tests_summary["BashTestSummaryFile"]
-            $OpenSSHTestInfo["BashTestLogFile"] = $Global:bash_tests_summary["BashTestLogFile"]
-        }
+    }
+    else
+    {
+        $AllTestsPassed = $false
     }
 
     # OpenSSH Uninstall Tests
@@ -346,6 +332,70 @@ function Invoke-OpenSSHTests
     }
 }
 
+<#
+    .Synopsis
+    Runs OpenSSH bash tests only.
+#>
+function Invoke-OpenSSHBashTestsOnly
+{
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory=$true)]
+        [string] $OpenSSHBinPath
+    )
+
+    Set-BasicTestInfo -OpenSSHBinPath $OpenSSHBinPath -Confirm:$false
+
+    $AllTestsPassed = $true
+
+    Write-Verbose -Verbose -Message "Running Bash-only OpenSSH test flow..."
+    Set-OpenSSHTestEnvironment -Confirm:$false
+
+    $cygwinInstalled = Ensure-CygWinInstalled -FailureMessage "All bash tests failed because CygWin install failed"
+    if (-not $cygwinInstalled)
+    {
+        $AllTestsPassed = $false
+    }
+
+    if ($AllTestsPassed)
+    {
+        Write-Verbose -Verbose -Message "Starting Bash Tests..."
+        Invoke-OpenSSHBashTests
+        if (-not $Global:bash_tests_summary)
+        {
+            $errorMessage = "Failed to start OpenSSH bash tests"
+            Write-BuildMessage -Message $errorMessage -Category Error
+            $AllTestsPassed = $false
+        }
+        else
+        {
+            if ($Global:bash_tests_summary["TotalBashTestsFailed"] -ne 0)
+            {
+                $total_bash_failed_tests = $Global:bash_tests_summary["TotalBashTestsFailed"]
+                $total_bash_tests = $Global:bash_tests_summary["TotalBashTests"]
+                $errorMessage = "At least one of the bash tests failed. [$total_bash_failed_tests of $total_bash_tests]"
+                Write-BuildMessage -Message $errorMessage -Category Error
+                $AllTestsPassed = $false
+            }
+
+            $OpenSSHTestInfo["BashTestSummaryFile"] = $Global:bash_tests_summary["BashTestSummaryFile"]
+            $OpenSSHTestInfo["BashTestLogFile"] = $Global:bash_tests_summary["BashTestLogFile"]
+        }
+    }
+
+    $OpenSSHTestInfo | Export-Clixml -Path "$repoRoot/OpenSSHTestInfo.xml" -Depth 10
+
+    if ($AllTestsPassed)
+    {
+        Write-BuildMessage -Message "All OpenSSH bash validation tests have passed!" -Category Information
+    }
+    else
+    {
+        Write-BuildMessage -Message "Some OpenSSH bash validation tests have failed." -Category Error
+        throw "OpenSSH bash validation tests failed!"
+    }
+}
+
 <#
       .Synopsis
       Collect OpenSSH pester test results into one directory
