Skip to content

Commit 06fc394

Browse files
bryanbcookbryanbcook
authored
Support duplicates in test plans (#132)
* Support duplicates in test plans Related to #130 and #113, a Test Plan can contain multiple references to the same Test Case id. This change detects when multiple points refer to the same test case id * regression test for ado plan with duplicates * test outcome enabled setting --------- Co-authored-by: bryan cook <3217452+bryancook@users.noreply.github.com>
1 parent ed9296e commit 06fc394

5 files changed

Lines changed: 125 additions & 25 deletions

File tree

‎PublishTestPlanResultsV1/processing/TestResultProcessor.ts‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,18 +53,24 @@ export class TestResultProcessor {
5353

5454
// process matches
5555
if (matchingPoints && matchingPoints.length > 0) {
56-
if (matchingPoints.length == 1) {
57-
result.matches.set( matchingPoints[0].id, frameworkResult );
56+
57+
const singleTestCase : boolean = matchingPoints.length == 1 || this.allMatchSameTestCase(matchingPoints);
58+
59+
if (singleTestCase) {
60+
// record matches
61+
matchingPoints.forEach( p => {
62+
result.matches.set( p.id, frameworkResult );
63+
})
5864

5965
// strip matched points from subsequent evaluations
6066
var matchingPointIds = matchingPoints.map(p => p.id);
6167
testPoints = testPoints.filter(i => matchingPointIds.indexOf(i.id) === -1);
6268
} else {
63-
const testCaseNames = matchingPoints.map(item => JSON.stringify(item)).join("\n");
64-
this.logger.warn(`Multiple matches were found for test case: ${frameworkResult.name}. Matches:\n${testCaseNames}`);
65-
this.logger.info("To prevent this warning, adjust the duplicates or the testCaseMatchStrategy to be more specific.");
66-
}
67-
69+
// match strategy found too many matches
70+
const testCaseNames = matchingPoints.map(item => JSON.stringify(item)).join("\n");
71+
this.logger.warn(`Multiple matches were found for test case: ${frameworkResult.name}. Matches:\n${testCaseNames}`);
72+
this.logger.info("To prevent this warning, adjust the testCaseMatchStrategy to be more specific.");
73+
}
6874
} else {
6975
result.unmatched.push(frameworkResult);
7076
}
@@ -116,5 +122,18 @@ export class TestResultProcessor {
116122
return match;
117123
}
118124

125+
private allMatchSameTestCase( matches : TestPoint[] ) : boolean {
126+
127+
let firstTestCaseId = (matches[0] as TestPoint2).testCaseReference.id;
128+
129+
const allMatch = matches.map( p => p as TestPoint2).every( p => p.testCaseReference.id === firstTestCaseId);
130+
131+
if (allMatch && matches.length > 1) {
132+
this.logger.warn(`Test Plan contains duplicates for test case: ${firstTestCaseId}.`);
133+
}
134+
135+
return allMatch;
136+
}
137+
119138
}
120139

‎PublishTestPlanResultsV1/test/TestResultProcessor.specs.ts‎

Lines changed: 57 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,12 @@ describe('TestResultProcessor', () => {
2828
(ctx as any).testPlan = null;
2929
(ctx as any).projectId = null;
3030

31-
// setup test data
32-
var testResult1 = new TestFrameworkResult("First","PASS");
33-
testResult1.properties.set("TestCase","1000");
34-
35-
var testResult2 = new TestFrameworkResult("Second","PASS");
36-
testResult2.properties.set("TestCase","2000");
37-
38-
var testResult3 = new TestFrameworkResult("Third","PASS");
39-
testResult3.properties.set("TestCase","3000");
40-
41-
testresults = [ testResult1, testResult2, testResult3];
31+
// setup test framework results with properties for TestIdMatchStrategy
32+
testresults = [
33+
testUtil.newTestFrameworkResult("First", "PASS", new Map([["TestCase", "1000"]])),
34+
testUtil.newTestFrameworkResult("Second", "PASS", new Map([["TestCase", "2000"]])),
35+
testUtil.newTestFrameworkResult("Third", "PASS", new Map([["TestCase", "3000"]]))
36+
];
4237

4338
setupTestPoints([
4439
testUtil.newTestPoint(1, "Test Case 1000", "0", "1000"),
@@ -112,6 +107,55 @@ describe('TestResultProcessor', () => {
112107
ctx.getTestPoints.returns(points);
113108
}
114109

110+
// Although Test Points are unique, it's possible to import a test suite that contains
111+
// the same test case multiple times. When this happens, typically this should be viewed
112+
// as a mistake in the test plan design as this would represent duplication of testing efforts.
113+
// However, multiple users have requested to have the ability to support this scenario.
114+
context("Test Plan contains Duplicates", () => {
115+
116+
beforeEach(() => {
117+
// setup the test plan to have duplicate test points
118+
setupTestPoints(
119+
[
120+
testUtil.newTestPoint(10000, "Test Case 1234", "0", "1234"),
121+
testUtil.newTestPoint(10001, "Test Case 1234", "0", "1234"),
122+
testUtil.newTestPoint(10002, "Test Case 5678", "0", "5678"),
123+
testUtil.newTestPoint(10003, "Test Case 9012", "0", "9012")
124+
]
125+
);
126+
127+
testresults = [
128+
testUtil.newTestFrameworkResult("Test Case 1234", "PASS", new Map([["TestCase", "1234"]])), // should match 10000, 10001
129+
testUtil.newTestFrameworkResult("Test Case 5678", "PASS", new Map([["TestCase", "5678"]])) // should match 10002
130+
]
131+
})
132+
133+
it("Should log a warning that the test plan contains duplicates", async () => {
134+
// arrange
135+
subject.logger = sinon.createStubInstance(Logger);
136+
137+
// act
138+
var result = await subject.process(testresults);
139+
140+
// assert
141+
sinon.assert.called(subject.logger.warn as sinon.SinonSpy);
142+
var loggedMessage = (subject.logger.warn as sinon.SinonSpy).getCall(0).args[0] as string;
143+
expect(loggedMessage).to.contain("Test Plan contains duplicates for test case: 1234");
144+
});
145+
146+
it ("Should map test result to multiple test points", async () => {
147+
// arrange
148+
// act
149+
var result = await subject.process(testresults);
150+
151+
// assert
152+
expect(result.matches.size).to.eq(3);
153+
expect(result.matches.get(10000)?.name).to.eq("Test Case 1234");
154+
expect(result.matches.get(10001)?.name).to.eq("Test Case 1234");
155+
expect(result.matches.get(10002)?.name).to.eq("Test Case 5678");
156+
})
157+
})
158+
115159
context("Multiple Matches", () => {
116160

117161
it("Should log an issue when multiple matches are found", async () => {
@@ -124,9 +168,10 @@ describe('TestResultProcessor', () => {
124168

125169
// assert
126170
sinon.assert.called(subject.logger.warn as sinon.SinonSpy);
171+
var loggedMessage = (subject.logger.warn as sinon.SinonSpy).getCall(0).args[0] as string;
172+
expect(loggedMessage).to.contain("Multiple matches were found for test case");
127173
expect(result.matches.size).to.eq(0);
128174
})
129-
130175
})
131176

132177
})

‎PublishTestPlanResultsV1/test/testUtil.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,9 +151,14 @@ export function newShallowReference(id : string, name : string) {
151151
return <ShallowReference>{ id: id, name: name};
152152
}
153153

154-
export function newTestFrameworkResult(name : string = "Test1", outcome : string = "PASS") {
154+
export function newTestFrameworkResult(name : string = "Test1", outcome : string = "PASS", properties? : Map<string, string>) : TestFrameworkResult {
155155
let result = new TestFrameworkResult(name, outcome);
156156
result.duration = 1000;
157+
if (properties) {
158+
properties.forEach((value, key) => {
159+
result.properties.set(key, value);
160+
});
161+
}
157162
return result;
158163
}
159164

‎devops/pipelines/marketplace-extension/regression-test.yml‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,24 @@ parameters:
3535
format: nunit
3636
testResultsFile: '$(Build.SourcesDirectory)/examples/dotnet/NUnitExample/TestResults/TestResults.xml'
3737
failTaskOnSkippedTests: true
38+
- name: TestDuplicates
39+
format: xunit
40+
testResultsFile: '$(Build.SourcesDirectory)/examples/dotnet/xUnitResults/TestResults-adotestplan-prj.xml'
41+
projectName: 'ADO Test Automation'
42+
testplan: 'Plan w Duplicates'
43+
buildId: 2436
44+
expectedResults:
45+
- outcome: Passed
46+
count: 2
47+
- name: TestDuplicates_wTestOutcomeSettingEnabled
48+
format: xunit
49+
testResultsFile: '$(Build.SourcesDirectory)/examples/dotnet/xUnitResults/TestResults-adotestplan-prj.xml'
50+
projectName: 'ADO Test Automation'
51+
testplan: 'Plan w Duplicates and TestOutcome Setting Enabled'
52+
buildId: 2436
53+
expectedResults:
54+
- outcome: Passed
55+
count: 2
3856

3957
steps:
4058
- pwsh: |
@@ -72,19 +90,19 @@ steps:
7290
filePath: '$(Pipeline.Workspace)/build/Test-ExtensionLocally.ps1'
7391
arguments: >
7492
-CollectionUri $(System.CollectionUri)
75-
-ProjectName 'Test Plan Extension'
93+
-ProjectName '${{ iif( ne(test.projectName,''), test.projectName, 'Test Plan Extension' ) }}'
7694
-AccessToken $(System.AccessToken)
7795
-TestResultFormat ${{ test.format }}
7896
-TestResultFiles ${{ test.testResultsFile }}
79-
-TestPlan 'Primary Test Plan'
97+
-TestPlan '${{ iif( ne(test.testPlan,''), test.testPlan, 'Primary Test Plan' ) }}'
8098
-TestConfigFilter ''
8199
-TestConfigAliases ''
82100
-TestCaseMatchStrategy ''
83101
-TestCaseProperty 'TestCaseId'
84102
-TestCaseRegex 'TestCase(\d+)'
85103
-TestConfigProperty ''
86104
-TestRunTitle '$(Build.DefinitionName) - $(nodeVersion) - ${{ test.name }}'
87-
-BuildId $(Build.BuildId)
105+
-BuildId ${{ iif( ne(test.buildId,''), test.buildId, '$(Build.BuildId)' ) }}
88106
-ReleaseId ''
89107
-ReleaseEnvironmentId ''
90108
-failTaskOnFailedTests '${{ iif( ne(test.failTaskOnFailedTests,''), test.failTaskOnFailedTests, 'false' ) }}'
@@ -119,7 +137,7 @@ steps:
119137
120138
$auth64 = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$(System.AccessToken)"))
121139
$headers = @{ Authorization = "Basic $auth64" }
122-
$uri = "$(System.CollectionUri)/Test%20Plan%20Extension/_apis/test/runs/$($testRun)?api-version=7.1"
140+
$uri = "$(System.CollectionUri)/$($env:projectName)/_apis/test/runs/$($testRun)?api-version=7.1"
123141
124142
$response = Invoke-RestMethod -Uri $uri -Headers $headers -Method Get
125143
@@ -140,4 +158,5 @@ steps:
140158
env:
141159
expected: ${{ convertToJson(test.expectedResults) }}
142160
testrun: $(${{ replace(lower(test.name), '-','_') }}.TestRunId)
143-
testrunurl: $(${{ replace(lower(test.name), '-','_') }}.TestRunUrl)
161+
testrunurl: $(${{ replace(lower(test.name), '-','_') }}.TestRunUrl)
162+
projectName: ${{ iif( ne(test.projectName,''), test.projectName, 'Test Plan Extension' ) }}

‎examples/dotnet/xUnitResults/TestResults-adotestplan-prj.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<assemblies timestamp="03/24/2026 16:09:58">
2+
<assembly name="dummy" run-date="03/24/2026" run-time="16:09:58" total="1" passed="1" failed="0" skipped="0" time="0.377" errors="0">
3+
<errors />
4+
<collection total="1" passed="1" failed="0" skipped="0" name="Dummy" time="0.377">
5+
<test name="ADO-Test-Automation_Logging_Register-new-product" type="xUnitExample" method="ADO-Test-Automation_Logging_Register-new-product" time="0.0000188" result="Pass">
6+
<traits>
7+
<trait name="TestCaseId" value="180" />
8+
</traits>
9+
</test>
10+
</collection>
11+
</assembly>
12+
</assemblies>

0 commit comments

Comments
 (0)