Gate test steps on setup success in CI pipeline (#4141)

Author: paulmedynski

eng/pipelines/common/templates/jobs/ci-run-tests-job.yml

@@ -326,6 +326,33 @@ jobs:
           ${{ if parameters.configProperties.FileStreamDirectory }}:
             fileStreamDirectory: ${{ parameters.configProperties.FileStreamDirectory }}
 
+  # Set up for x86 tests by manually installing dotnet for x86 to an alternative location. This
+  # is only used to execute the test runtime (framework to test is specified in build params), so
+  # it should be acceptable to just install a specific version in all cases.
+  # @TODO: This setup is very confusing. Ideally we should just be utilizing the dotnet installation
+  #    earlier in the job. There has to be a cleaner way of doing this.
+  - ${{ if and(eq(parameters.enableX86Test, true), eq(parameters.operatingSystem, 'Windows')) }}:
+    - ${{ if ne(variables['dotnetx86RootPath'], '') }}:
+      # Install the .NET SDK and Runtimes for x86.
+      - template: /eng/pipelines/steps/install-dotnet.yml@self
+        parameters:
+          architecture: x86
+          debug: ${{ parameters.debug }}
+          installDir: $(dotnetx86RootPath)
+          runtimes: [8.x, 9.x]
+
+  # Ensure TestResults directory exists so that publish steps don't fail when
+  # tests are skipped due to a setup failure.
+  - pwsh: New-Item -ItemType Directory -Path TestResults -Force | Out-Null
+    displayName: 'Create TestResults Directory'
+
+  # Gate: record that all setup steps (SDK install, build, SQL config, x86 SDK
+  # install, etc.) completed successfully.  Test steps condition on this variable
+  # so they are skipped when setup fails, yet remain independent of each
+  # other's results.
+  - pwsh: Write-Host '##vso[task.setvariable variable=setupSucceeded]true'
+    displayName: 'Gate: Mark Setup Succeeded'
+
   - ${{ if eq(parameters.enableX64Test, true) }}: # run native tests
     - template: /eng/pipelines/common/templates/steps/run-all-tests-step.yml@self # run tests
       parameters:
@@ -340,20 +367,6 @@ jobs:
         mdsPackageVersion: ${{ parameters.mdsPackageVersion }}
 
   - ${{ if and(eq(parameters.enableX86Test, true), eq(parameters.operatingSystem, 'Windows')) }}:
-    # Set up for x86 tests by manually installing dotnet for x86 to an alternative location. This
-    # is only used to execute the test runtime (framework to test is specified in build params), so
-    # it should be acceptable to just install a specific version in all cases.
-    # @TODO: This setup is very confusing. Ideally we should just be utilizing the dotnet installation
-    #    earlier in the job. There has to be a cleaner way of doing this.
-    - ${{ if ne(variables['dotnetx86RootPath'], '') }}:
-      # Install the .NET SDK and Runtimes for x86.
-      - template: /eng/pipelines/steps/install-dotnet.yml@self
-        parameters:
-          architecture: x86
-          debug: ${{ parameters.debug }}
-          installDir: $(dotnetx86RootPath)
-          runtimes: [8.x, 9.x]
-
     - template: /eng/pipelines/common/templates/steps/run-all-tests-step.yml@self
       parameters:
         debug: ${{ parameters.debug }}

eng/pipelines/common/templates/steps/run-all-tests-step.yml

@@ -79,7 +79,7 @@ steps:
   - ${{if eq(parameters.referenceType, 'Project')}}:
     - task: MSBuild@1
       displayName: 'Run Unit Tests ${{parameters.msbuildArchitecture }}'
-      condition: succeededOrFailed()
+      condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
       inputs:
         solution: build2.proj
         msbuildArchitecture: ${{parameters.msbuildArchitecture }}
@@ -101,7 +101,7 @@ steps:
 
     - task: MSBuild@1
       displayName: 'Run Flaky Unit Tests ${{parameters.msbuildArchitecture }}'
-      condition: succeededOrFailed()
+      condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
       inputs:
         solution: build2.proj
         msbuildArchitecture: ${{parameters.msbuildArchitecture }}
@@ -128,7 +128,7 @@ steps:
 
   - task: MSBuild@1
     displayName: 'Run Functional Tests ${{parameters.msbuildArchitecture }}'
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     inputs:
       solution: build2.proj
       msbuildArchitecture: ${{parameters.msbuildArchitecture }}
@@ -155,7 +155,7 @@ steps:
           -p:TestResultsFolderPath=TestResults
 
   - task: MSBuild@1
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     displayName: 'Run Flaky Functional Tests ${{parameters.msbuildArchitecture }}'
     inputs:
       solution: build2.proj
@@ -188,7 +188,7 @@ steps:
     continueOnError: true
 
   - task: MSBuild@1
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     displayName: 'Run Manual Tests ${{parameters.msbuildArchitecture }}'
     inputs:
       solution: build2.proj
@@ -219,7 +219,7 @@ steps:
     retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
 
   - task: MSBuild@1
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     displayName: 'Run Flaky Manual Tests ${{parameters.msbuildArchitecture }}'
     inputs:
       solution: build2.proj
@@ -257,7 +257,7 @@ steps:
   - ${{if eq(parameters.referenceType, 'Project')}}:
     - task: DotNetCoreCLI@2
       displayName: 'Run Unit Tests'
-      condition: succeededOrFailed()
+      condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
       inputs:
         command: custom
         projects: build2.proj
@@ -276,7 +276,7 @@ steps:
 
     - task: DotNetCoreCLI@2
       displayName: 'Run Flaky Unit Tests'
-      condition: succeededOrFailed()
+      condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
       inputs:
         command: custom
         projects: build2.proj
@@ -298,7 +298,7 @@ steps:
 
   - task: DotNetCoreCLI@2
     displayName: 'Run Functional Tests'
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     inputs:
       command: custom
       projects: build2.proj
@@ -316,7 +316,7 @@ steps:
       verbosityPack: Detailed
 
   - task: DotNetCoreCLI@2
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     displayName: 'Run Flaky Functional Tests'
     inputs:
       command: custom
@@ -339,7 +339,7 @@ steps:
 
   - task: DotNetCoreCLI@2
     displayName: 'Run Manual Tests'
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     inputs:
       command: custom
       projects: build2.proj
@@ -360,7 +360,7 @@ steps:
 
   - task: DotNetCoreCLI@2
     displayName: 'Run Flaky Manual Tests'
-    condition: succeededOrFailed()
+    condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed())
     inputs:
       command: custom
       projects: build2.proj