Skip to content

Commit a900926

Browse files
paulmedynskipaulmedynski
committed
Task 42088: Suppress flaky tests errors in AzDO UI
- Testing suppression of errors/warnings in the UI due to flaky tests.
1 parent a0357b2 commit a900926

5 files changed

Lines changed: 78 additions & 109 deletions

File tree

‎build.proj‎

Lines changed: 55 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -279,10 +279,10 @@
279279
<!-- Run all tests applicable to the host OS. -->
280280
<Target Name="RunTests" DependsOnTargets="RunUnitTests;RunFunctionalTests;RunManualTests"/>
281281

282-
<!-- Run all unit tests applicable to the host OS. -->
282+
<!-- Run all (non-flaky) unit tests applicable to the host OS. -->
283283
<Target Name="RunUnitTests" DependsOnTargets="RunUnitTestsWindows;RunUnitTestsUnix" Condition="$(ReferenceType.Contains('Project'))"/>
284284

285-
<!-- Run all unit tests applicable to Windows. -->
285+
<!-- Run all (non-flaky) unit tests applicable to Windows. -->
286286
<Target Name="RunUnitTestsWindows" Condition="'$(IsEnabledWindows)' == 'true' AND $(ReferenceType.Contains('Project'))">
287287
<PropertyGroup>
288288
<TestCommand>
@@ -304,7 +304,7 @@
304304
<Exec ConsoleToMsBuild="true" Command="$(TestCommand)"/>
305305
</Target>
306306

307-
<!-- Run all unit tests applicable to Unix. -->
307+
<!-- Run all (non-flaky) unit tests applicable to Unix. -->
308308
<Target Name="RunUnitTestsUnix" Condition="'$(IsEnabledWindows)' != 'true' AND $(ReferenceType.Contains('Project'))">
309309
<PropertyGroup>
310310
<TestCommand>
@@ -326,6 +326,58 @@
326326
<Exec ConsoleToMsBuild="true" Command="$(TestCommand)"/>
327327
</Target>
328328

329+
<!-- Run all flaky unit tests applicable to the host OS. -->
330+
<Target Name="RunFlakyUnitTests" DependsOnTargets="RunFlakyUnitTestsWindows;RunFlakyUnitTestsUnix" Condition="$(ReferenceType.Contains('Project'))"/>
331+
332+
<!-- Run all flaky unit tests applicable to Windows. -->
333+
<Target Name="RunFlakyUnitTestsWindows" Condition="'$(IsEnabledWindows)' == 'true' AND $(ReferenceType.Contains('Project'))">
334+
<PropertyGroup>
335+
<TestCommand>
336+
$(DotnetPath)dotnet test "@(UnitTestsProj)"
337+
-f $(TF)
338+
-p:Configuration=$(Configuration)
339+
$(CollectStatement)
340+
--results-directory $(ResultsDirectory)
341+
--filter "category = flaky"
342+
--logger:"trx;LogFilePrefix=Unit-Windows$(TargetGroup)-$(TestSet)"
343+
--blame-hang
344+
--blame-hang-dump-type full
345+
--blame-hang-timeout 10m
346+
</TestCommand>
347+
<!-- Convert more than one whitespace character into one space -->
348+
<TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
349+
</PropertyGroup>
350+
<Message Text=">>> Running unit tests for Windows via command: $(TestCommand)"/>
351+
<!--
352+
We instruct MSBuild to ignore output from the tests that matches error/warning expressions.
353+
This avoids cluttering the Azure DevOps pipeline runs UI with spurious errors, while still
354+
considering the exit status of the test process to determine overall pass/fail.
355+
-->
356+
<Exec ConsoleToMsBuild="true" IgnoreStandardErrorWarningFormat="true" Command="$(TestCommand)"/>
357+
</Target>
358+
359+
<!-- Run all flaky unit tests applicable to Unix. -->
360+
<Target Name="RunFlakyUnitTestsUnix" Condition="'$(IsEnabledWindows)' != 'true' AND $(ReferenceType.Contains('Project'))">
361+
<PropertyGroup>
362+
<TestCommand>
363+
$(DotnetPath)dotnet test "@(UnitTestsProj)"
364+
-f $(TF)
365+
-p:Configuration=$(Configuration)
366+
$(CollectStatement)
367+
--results-directory $(ResultsDirectory)
368+
--filter "category = flaky"
369+
--logger:"trx;LogFilePrefix=Unit-Unixnetcoreapp-$(TestSet)"
370+
--blame-hang
371+
--blame-hang-dump-type full
372+
--blame-hang-timeout 10m
373+
</TestCommand>
374+
<!-- Convert more than one whitespace character into one space -->
375+
<TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
376+
</PropertyGroup>
377+
<Message Text=">>> Running unit tests for Unix via command: $(TestCommand)"/>
378+
<Exec ConsoleToMsBuild="true" IgnoreStandardErrorWarningFormat="true" Command="$(TestCommand)"/>
379+
</Target>
380+
329381
<!-- Run all Functional tests applicable to the host OS. -->
330382
<Target Name="RunFunctionalTests" DependsOnTargets="RunFunctionalTestsWindows;RunFunctionalTestsUnix" />
331383

‎eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
# The .NET Foundation licenses this file to you under the MIT license. #
44
# See the LICENSE file in the project root for more information. #
55
#################################################################################
6+
7+
# This template is part of the MDS Official pipeline.
8+
69
parameters:
710
- name: downloadPackageStep
811
type: step
@@ -35,20 +38,20 @@ jobs:
3538
isCustom: true
3639
name: ADO-1ES-Pool
3740
vmImage: 'ADO-MMS22-SQL19'
38-
41+
3942
variables: # More settings at https://aka.ms/obpipelines/yaml/jobs
4043
- template: /eng/pipelines/libraries/mds-validation-variables.yml@self
4144

4245
steps:
4346
- template: /eng/pipelines/common/templates/steps/pre-build-step.yml
44-
47+
4548
- ${{parameters.downloadPackageStep }}
4649

4750
- pwsh: Copy-Item -Path "$(Build.SourcesDirectory)/NuGet.config.local" -Destination "$(Build.SourcesDirectory)/NuGet.config" -Force
4851
displayName: Use local NuGet packages
4952

5053
- template: /eng/pipelines/common/templates/steps/update-config-file-step.yml
51-
parameters:
54+
parameters:
5255
TCPConnectionString: $(SQL_TCP_CONN_STRING)
5356
NPConnectionString: $(SQL_NP_CONN_STRING)
5457
SupportsIntegratedSecurity: false

‎eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml‎

Lines changed: 3 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
# The .NET Foundation licenses this file to you under the MIT license. #
44
# See the LICENSE file in the project root for more information. #
55
#################################################################################
6+
7+
# This template is part of the MDS Official pipeline.
8+
69
parameters:
710
- name: TargetNetCoreVersion
811
type: string
@@ -86,27 +89,6 @@ steps:
8689
--blame-hang-dump-type full
8790
--blame-hang-timeout 10m
8891
89-
- task: DotNetCoreCLI@2
90-
displayName: 'Run Flaky Functional Tests for ${{parameters.TargetNetCoreVersion }}'
91-
condition: succeededOrFailed()
92-
inputs:
93-
command: test
94-
projects: 'src\Microsoft.Data.SqlClient\tests\FunctionalTests\Microsoft.Data.SqlClient.FunctionalTests.csproj'
95-
arguments: >-
96-
-p:Platform=${{ parameters.platform }}
97-
-p:TestTargetOS=${{ parameters.TestTargetOS }}
98-
-p:TargetNetCoreVersion=${{ parameters.TargetNetCoreVersion }}
99-
-p:ReferenceType=${{ parameters.referenceType }}
100-
-p:Configuration=${{ parameters.buildConfiguration }}
101-
-p:MdsPackageVersion=${{ parameters.mdsPackageVersion }}
102-
--no-build
103-
-v n
104-
--filter "category=flaky"
105-
--blame-hang
106-
--blame-hang-dump-type full
107-
--blame-hang-timeout 10m
108-
continueOnError: true
109-
11092
- task: DotNetCoreCLI@2
11193
displayName: 'Run Manual Tests for ${{parameters.TargetNetCoreVersion }}'
11294
condition: succeededOrFailed()
@@ -128,26 +110,3 @@ steps:
128110
--blame-hang-dump-type full
129111
--blame-hang-timeout 10m
130112
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
131-
132-
- task: DotNetCoreCLI@2
133-
displayName: 'Run Flaky Manual Tests for ${{parameters.TargetNetCoreVersion }}'
134-
condition: succeededOrFailed()
135-
inputs:
136-
command: test
137-
projects: 'src\Microsoft.Data.SqlClient\tests\ManualTests\Microsoft.Data.SqlClient.ManualTesting.Tests.csproj'
138-
arguments: >-
139-
-p:Platform=${{ parameters.platform }}
140-
-p:TestTargetOS=${{ parameters.TestTargetOS }}
141-
-p:TargetNetCoreVersion=${{ parameters.TargetNetCoreVersion }}
142-
-p:ReferenceType=${{ parameters.referenceType }}
143-
-p:Configuration=${{ parameters.buildConfiguration }}
144-
-p:MdsPackageVersion=${{ parameters.mdsPackageVersion }}
145-
--no-build
146-
-v n
147-
--filter "category=flaky"
148-
--collect "Code Coverage"
149-
--blame-hang
150-
--blame-hang-dump-type full
151-
--blame-hang-timeout 10m
152-
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
153-
continueOnError: true

‎eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml‎

Lines changed: 3 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
# The .NET Foundation licenses this file to you under the MIT license. #
44
# See the LICENSE file in the project root for more information. #
55
#################################################################################
6+
7+
# This template is part of the MDS Official pipeline.
8+
69
parameters:
710
- name: TargetNetFxVersion
811
type: string
@@ -87,28 +90,6 @@ steps:
8790
--blame-hang-dump-type full
8891
--blame-hang-timeout 10m
8992
90-
- task: DotNetCoreCLI@2
91-
displayName: 'Run Flaky Functional Tests for ${{parameters.TargetNetFxVersion }}'
92-
condition: succeededOrFailed()
93-
inputs:
94-
command: test
95-
projects: 'src\Microsoft.Data.SqlClient\tests\FunctionalTests\Microsoft.Data.SqlClient.FunctionalTests.csproj'
96-
arguments: >-
97-
-p:Platform=${{ parameters.platform }}
98-
-p:TestTargetOS=${{ parameters.TestTargetOS }}
99-
-p:TargetNetFxVersion=${{ parameters.TargetNetFxVersion }}
100-
-p:ReferenceType=${{ parameters.referenceType }}
101-
-p:Configuration=${{ parameters.buildConfiguration }}
102-
-p:MdsPackageVersion=${{ parameters.mdsPackageVersion }}
103-
--no-build
104-
-v n
105-
--filter "category=flaky"
106-
--collect "Code Coverage"
107-
--blame-hang
108-
--blame-hang-dump-type full
109-
--blame-hang-timeout 10m
110-
continueOnError: true
111-
11293
- task: DotNetCoreCLI@2
11394
displayName: 'Run Manual Tests for ${{parameters.TargetNetFxVersion }}'
11495
condition: succeededOrFailed()
@@ -130,26 +111,3 @@ steps:
130111
--blame-hang-dump-type full
131112
--blame-hang-timeout 10m
132113
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
133-
134-
- task: DotNetCoreCLI@2
135-
displayName: 'Run Flaky Manual Tests for ${{parameters.TargetNetFxVersion }}'
136-
condition: succeededOrFailed()
137-
inputs:
138-
command: test
139-
projects: 'src\Microsoft.Data.SqlClient\tests\ManualTests\Microsoft.Data.SqlClient.ManualTesting.Tests.csproj'
140-
arguments: >-
141-
-p:Platform=${{ parameters.platform }}
142-
-p:TestTargetOS=${{ parameters.TestTargetOS }}
143-
-p:TargetNetFxVersion=${{ parameters.TargetNetFxVersion }}
144-
-p:ReferenceType=${{ parameters.referenceType }}
145-
-p:Configuration=${{ parameters.buildConfiguration }}
146-
-p:MdsPackageVersion=${{ parameters.mdsPackageVersion }}
147-
--no-build
148-
-v n
149-
--filter "category=flaky"
150-
--collect "Code Coverage"
151-
--blame-hang
152-
--blame-hang-dump-type full
153-
--blame-hang-timeout 10m
154-
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
155-
continueOnError: true

‎eng/pipelines/common/templates/steps/run-all-tests-step.yml‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ parameters:
1818
- name: platform
1919
type: string
2020
default: $(Platform)
21-
21+
2222
- name: buildConfiguration
2323
type: string
2424
values:
@@ -39,11 +39,11 @@ parameters:
3939
values:
4040
- x64
4141
- x86
42-
42+
4343
- name: dotnetx86RootPath # full path to the x86 dotnet root folder with trailing slash
4444
type: string
4545
default: ''
46-
46+
4747
- name: operatingSystem
4848
type: string
4949
default: 'Windows'
@@ -68,7 +68,7 @@ steps:
6868
msbuildArchitecture: ${{parameters.msbuildArchitecture }}
6969
platform: '${{parameters.platform }}'
7070
configuration: '${{parameters.buildConfiguration }}'
71-
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
71+
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
7272
msbuildArguments: >-
7373
-t:RunUnitTests
7474
-p:TF=${{ parameters.targetFramework }}
@@ -88,20 +88,18 @@ steps:
8888
msbuildArchitecture: ${{parameters.msbuildArchitecture }}
8989
platform: '${{parameters.platform }}'
9090
configuration: '${{parameters.buildConfiguration }}'
91-
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
91+
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
9292
msbuildArguments: >-
93-
-t:RunUnitTests
93+
-t:RunFlakyUnitTests
9494
-p:TF=${{ parameters.targetFramework }}
9595
-p:MdsPackageVersion=${{ parameters.mdsPackageVersion }}
96-
-p:Filter="category=flaky"
9796
-p:CollectCodeCoverage=false
9897
${{ else }}: # x86
9998
msbuildArguments: >-
100-
-t:RunUnitTests
99+
-t:RunFlakyUnitTests
101100
-p:TF=${{ parameters.targetFramework }}
102101
-p:MdsPackageVersion=${{ parameters.mdsPackageVersion }}
103102
-p:DotnetPath=${{ parameters.dotnetx86RootPath }}
104-
-p:Filter="category=flaky"
105103
-p:CollectCodeCoverage=false
106104
continueOnError: true
107105

@@ -113,7 +111,7 @@ steps:
113111
msbuildArchitecture: ${{parameters.msbuildArchitecture }}
114112
platform: '${{parameters.platform }}'
115113
configuration: '${{parameters.buildConfiguration }}'
116-
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
114+
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
117115
msbuildArguments: >-
118116
-t:RunFunctionalTests
119117
-p:TF=${{ parameters.targetFramework }}
@@ -137,7 +135,7 @@ steps:
137135
msbuildArchitecture: ${{parameters.msbuildArchitecture }}
138136
platform: '${{parameters.platform }}'
139137
configuration: '${{parameters.buildConfiguration }}'
140-
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
138+
${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
141139
msbuildArguments: >-
142140
-t:RunFunctionalTests
143141
-p:TF=${{ parameters.targetFramework }}
@@ -232,7 +230,7 @@ steps:
232230
-p:Configuration=${{ parameters.buildConfiguration }}
233231
verbosityRestore: Detailed
234232
verbosityPack: Detailed
235-
233+
236234
- task: DotNetCoreCLI@2
237235
displayName: 'Run Flaky Unit Tests'
238236
condition: succeededOrFailed()
@@ -241,14 +239,13 @@ steps:
241239
projects: build.proj
242240
custom: msbuild
243241
arguments: >-
244-
-t:RunUnitTests
242+
-t:RunFlakyUnitTests
245243
-p:TF=${{ parameters.targetFramework }}
246244
-p:TestSet=${{ parameters.testSet }}
247245
-p:ReferenceType=${{ parameters.referenceType }}
248246
-p:MdsPackageVersion=${{ parameters.mdsPackageVersion }}
249247
-p:platform=${{ parameters.platform }}
250248
-p:Configuration=${{ parameters.buildConfiguration }}
251-
-p:Filter="category=flaky"
252249
verbosityRestore: Detailed
253250
verbosityPack: Detailed
254251
continueOnError: true

0 commit comments

Comments
 (0)