Skip to content

Commit a4fd04e

Browse files
bryanbcookbryanbcook
authored
Fix for multi-configuration support (#93)
* improve debug logging for JSON framework data to show properties + attachments * Test improvements * added type coercion fix
1 parent b0a5cdb commit a4fd04e

7 files changed

Lines changed: 165 additions & 17 deletions

File tree

‎PublishTestPlanResultsV1/processing/TestResultProcessor.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { TestPoint, TestPlan } from "azure-devops-node-api/interfaces/TestInterf
22
import { TestResultContext } from "../context/TestResultContext";
33
import { TestFrameworkResult } from "../framework/TestFrameworkResult";
44
import { TestResultMatch, TestResultMatchStrategy } from "./TestResultMatchStrategy";
5+
import { JSONStringify } from "../services/JsonUtil";
56
import { ILogger, getLogger } from "../services/Logger"
67

78
export class TestResultProcessorResult {
@@ -43,7 +44,7 @@ export class TestResultProcessor {
4344
break;
4445
}
4546

46-
this.logger.debug(`evaluating '${frameworkResult.name}' (${JSON.stringify(frameworkResult)})`);
47+
this.logger.debug(`evaluating '${frameworkResult.name}' (${JSONStringify(frameworkResult)})`);
4748

4849
let matchingPoints = testPoints.filter(point => this.compare(frameworkResult, point));
4950

‎PublishTestPlanResultsV1/processing/TestResultProcessorFactory.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ export class TestConfigMatchStrategy implements TestResultMatchStrategy {
7272
let configIdToCompare = this.allowedConfigs.get(testResultConfig)?.id?.toString();
7373

7474
// compare against the test point
75-
if (configIdToCompare !== point.configuration.id) {
75+
if (configIdToCompare !== point.configuration.id?.toString()) {
7676
return TestResultMatch.Fail;
7777
}
7878
}

‎PublishTestPlanResultsV1/services/JsonUtil.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
function jsonStringifyReplacer(key: string, value: any) {
2+
if (value instanceof Map) {
3+
return Object.fromEntries(value);
4+
}
5+
if (Array.isArray(value)) {
6+
return [...value];
7+
}
8+
return value;
9+
}
10+
11+
export function JSONStringify(obj: any): string {
12+
return JSON.stringify(obj, jsonStringifyReplacer);
13+
}

‎PublishTestPlanResultsV1/test/JsonUtil.specs.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { expect } from "chai";
2+
import sinon from "sinon";
3+
import path from "path";
4+
import * as Contracts from 'azure-devops-node-api/interfaces/TestInterfaces'
5+
import { TestFrameworkResult } from "../framework/TestFrameworkResult";
6+
import { JSONStringify } from "../services/JsonUtil";
7+
8+
9+
10+
describe("JSONUtil", () => {
11+
12+
let resultItem: TestFrameworkResult
13+
14+
beforeEach(() => {
15+
resultItem = new TestFrameworkResult("TestName", "PASS")
16+
resultItem.duration = 1234.5678
17+
resultItem.stacktrace = "stacktrace"
18+
resultItem.failure = "failure"
19+
});
20+
21+
it("Can stringify test framework result", () => {
22+
// arrange
23+
// act
24+
const result = JSONStringify(resultItem);
25+
26+
// assert
27+
expect(result).to.contain('"name":"TestName"');
28+
expect(result).to.contain('"duration":1234.5678');
29+
expect(result).to.contain('"stacktrace":"stacktrace"');
30+
expect(result).to.contain('"failure":"failure"');
31+
expect(result).to.contain('"outcome":2');
32+
expect(result).to.contain('"properties":{}');
33+
expect(result).to.contain('"attachments":[]');
34+
});
35+
36+
it("Can stringify test framework result with properties", () => {
37+
// arrange
38+
resultItem.properties.set("TestID", "1234")
39+
resultItem.properties.set("TestConfig", "edge")
40+
41+
// act
42+
const result = JSONStringify(resultItem);
43+
44+
// assert
45+
expect(result).to.contain('"properties":{"TestID":"1234","TestConfig":"edge"}');
46+
});
47+
48+
it("Can stringify test framework result with attachments", () => {
49+
// arrange
50+
resultItem.attachments.push({ name: "TestAttachment", path: "filepath1" })
51+
resultItem.attachments.push({ name: "TestAttachment2", path: "filepath2" })
52+
53+
// act
54+
const result = JSONStringify(resultItem);
55+
56+
// assert
57+
expect(result).to.contain('"attachments":[{"name":"TestAttachment","path":"filepath1"},{"name":"TestAttachment2","path":"filepath2"}]');
58+
});
59+
60+
});

‎PublishTestPlanResultsV1/test/TestFrameworkResultReader.specs.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,18 @@ describe("TestFramework Results Reader", () => {
103103
expect(results[0].duration).to.eq(10000);
104104
});
105105

106+
// https://github.com/bryanbcook/azdevops-testplan-extension/issues/92
107+
it("Can read JUnit properties", async () => {
108+
// arrange
109+
files.push(path.join(baseDir, "junit/bug-92.xml"));
110+
111+
// act
112+
var results = await subject.read("junit", files);
113+
114+
// assert
115+
expect(results[0].properties.size).to.eq(3);
116+
});
117+
106118
it("Can read Cucumber results", async () => {
107119
// arrange
108120
files.push(path.join(baseDir, "cucumber/single-suite-single-test.json"));

‎PublishTestPlanResultsV1/test/TestResultMatchStrategy.specs.ts‎

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
import { expect } from "chai";
2-
32
import { ShallowReference, TestConfiguration, TestPoint, WorkItemReference } from "azure-devops-node-api/interfaces/TestInterfaces";
43
import { TestFrameworkResult } from "../framework/TestFrameworkResult";
54
import { TestAutomationPropertyMatchStrategy, TestConfigMatchStrategy, TestIdMatchStrategy, TestNameMatchStrategy, TestRegexMatchStrategy } from "../processing/TestResultProcessorFactory";
65
import { TestResultMatch } from "../processing/TestResultMatchStrategy";
7-
import * as util from './testUtil';
86

97
describe("TestCaseMatchStrategy", () => {
108

@@ -28,21 +26,22 @@ describe("TestCaseMatchStrategy", () => {
2826
point = <TestPoint>{ id: 1, configuration: <ShallowReference>{ id: "1", name:"Config1"} };
2927

3028
// setup subject
31-
subject = new TestConfigMatchStrategy(allowedConfigs,"Config");
29+
// The default testConfigProperty is "Config"
30+
subject = new TestConfigMatchStrategy(allowedConfigs, "Config");
3231
})
3332

34-
it('Should match if testpoint has the same category as filter', async () => {
33+
it('Should match if testpoint has a supported configuration', async () => {
3534
// arrange
36-
35+
test.properties.set("Config","Config1"); // dummy
3736
// act
3837
var result = subject.isMatch(test, point);
3938

4039
// assert
41-
expect( result).eq(TestResultMatch.None);
40+
expect(result).eq(TestResultMatch.None);
4241
});
4342

44-
// test point does not have same category
45-
it('Should not match if testpoint does not have the same category as filter', async() => {
43+
// test point does not have same configuration as allowed configurations
44+
it('Should not match if testpoint does not have supported configuration', async() => {
4645
// arrange
4746
point.configuration = <ShallowReference>{ id: "2", name: "Different Config"};
4847

@@ -53,6 +52,35 @@ describe("TestCaseMatchStrategy", () => {
5352
expect(result).eq(TestResultMatch.Fail);
5453
});
5554

55+
// assuming that testConfigProperty is opt-in and the majority of TestPoints won't have configurations
56+
// could translate into a bug if a TestPoint has multiple configurations but the TestFramework result doesn't
57+
it('Should match even if TestFramework result does not have a defined testConfigProperty', () => {
58+
// arrange
59+
test.properties.clear(); // no config property
60+
61+
// act
62+
var result = subject.isMatch(test, point);
63+
64+
// assert
65+
expect(result).eq(TestResultMatch.None);
66+
});
67+
68+
// https://github.com/bryanbcook/azdevops-testplan-extension/issues/92
69+
// looks like the API is returning a number for the id instead of a string in the shallowreference
70+
it('Should match if config id in shallow reference is not a string', async () => {
71+
// arrange
72+
test.properties.set("Config", "Config1")
73+
74+
// emulate API response as number by using <unknown> cast
75+
point.configuration = <ShallowReference><unknown>{ id: 1, name: "Config1" };
76+
77+
// act
78+
var result = subject.isMatch(test, point);
79+
80+
// assert
81+
expect(result).eq(TestResultMatch.None);
82+
});
83+
5684
it('Should match if test result has matching config', async () => {
5785
// arrange
5886
test.properties.set("Config", "1");
@@ -74,13 +102,6 @@ describe("TestCaseMatchStrategy", () => {
74102
// assert
75103
expect(result).eq(TestResultMatch.None);
76104
});
77-
// // arrange
78-
// // act
79-
// // assert
80-
// throw new Error("Not implemented");
81-
82-
83-
84105
});
85106

86107
context("TestCase Name Matcher", () => {

‎PublishTestPlanResultsV1/test/data/junit/bug-92.xml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<testsuites>
3+
<testsuite name="pytest" errors="0" failures="0" skipped="0" tests="6" time="271.885" timestamp="2025-05-06T17:11:00.804626-04:00" hostname="ROM-1GLRCX3">
4+
<testcase classname="tests.test_menu_elements" name="Test Menu Elements" time="11.656">
5+
<properties>
6+
<property name="TestConfig" value="chromium" />
7+
<property name="TestId" value="8207" />
8+
</properties>
9+
</testcase>
10+
<testcase classname="tests.test_menu_elements" name="Test Menu Elements" time="235.515">
11+
<properties>
12+
<property name="TestConfig" value="firefox" />
13+
<property name="TestId" value="8207" />
14+
</properties>
15+
</testcase>
16+
<testcase classname="tests.test_menu_elements" name="Test Menu Elements" time="5.432">
17+
<properties>
18+
<property name="TestConfig" value="edge" />
19+
<property name="TestId" value="8207" />
20+
</properties>
21+
</testcase>
22+
<testcase classname="tests.test_admin_tab" name="Test Admin Tab" time="5.243">
23+
<properties>
24+
<property name="TestConfig" value="chromium" />
25+
<property name="TestId" value="8173" />
26+
</properties>
27+
</testcase>
28+
<testcase classname="tests.test_admin_tab" name="Test Admin Tab" time="8.711">
29+
<properties>
30+
<property name="TestConfig" value="firefox" />
31+
<property name="TestId" value="8173" />
32+
</properties>
33+
</testcase>
34+
<testcase classname="tests.test_admin_tab" name="Test Admin Tab" time="5.310">
35+
<properties>
36+
<property name="TestConfig" value="edge" />
37+
<property name="TestId" value="8173" />
38+
</properties>
39+
</testcase>
40+
</testsuite>
41+
</testsuites>

0 commit comments

Comments
 (0)