Skip to content

Commit 16b5df0

Browse files
nccatoninccatoni
authored
Adding features to test optimization data (#7202)
1 parent 7f37103 commit 16b5df0

2 files changed

Lines changed: 124 additions & 104 deletions

File tree

‎conftest.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import types
1212
import xml.etree.ElementTree as ET
1313
from collections.abc import Generator, Sequence
14-
from typing import Any, Literal
14+
from typing import Any, Literal, TypedDict
1515

1616
import pytest
1717
from _pytest.junitxml import xml_key
@@ -273,7 +273,13 @@ def _collect_item_metadata(item: pytest.Item):
273273
if declaration is not None:
274274
logger.debug(f"{item.nodeid} => {declaration} => skipped")
275275

276-
metadata = {
276+
class Metadata(TypedDict):
277+
details: str | None
278+
testDeclaration: str | None
279+
features: list[int]
280+
owners: list[str]
281+
282+
metadata: Metadata = {
277283
"details": declaration if details is None else f"{declaration} ({details})",
278284
"testDeclaration": declaration,
279285
"features": [marker.kwargs["feature_id"] for marker in item.iter_markers("features")],
@@ -283,8 +289,9 @@ def _collect_item_metadata(item: pytest.Item):
283289
# decorate test for junit
284290
item.user_properties.append(("test.codeowners", json.dumps(metadata["owners"])))
285291

286-
# for feature_id in metadata["features"]:
287-
# item.user_properties.append(("dd_tags[test.feature_id]", str(feature_id)))
292+
for feature_id in metadata["features"]:
293+
if feature_id != NOT_REPORTED_FEATURE_ID:
294+
item.user_properties.append(("dd_tags[systest.case.feature_id]", str(feature_id)))
288295

289296
if declaration:
290297
item.user_properties.append(("dd_tags[systest.case.declaration]", declaration))

‎tests/test_the_test/reportJunit_expected.xml‎

Lines changed: 113 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,107 +1,120 @@
1+
<?xml version="1.0"?>
12
<testsuites>
2-
<testsuite name="system_tests_suite" errors="1" failures="1" skipped="6" tests="10" time="0.747" timestamp="2025-09-12T08:58:47.879135" hostname="COMP-C02FL1L0ML87">
3-
<properties>
4-
<property name="dd_tags[systest.suite.context.scenario]" value="MOCK_THE_TEST" />
5-
</properties>
6-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_error[spring]" time="0.000">
7-
<properties>
8-
<property name="dd_tags[systest.case.outcome]" value="error" />
9-
<property name="dd_tags[test.final_status]" value="fail"/>
10-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]" />
11-
</properties>
12-
<error message="failed on setup with &quot;TypeError: nope!&quot;">@pytest.fixture
3+
<testsuite name="system_tests_suite" errors="1" failures="1" skipped="6" tests="10" time="0.804" timestamp="2026-06-23T16:51:44.127970" hostname="COMP-MKHXWVJ25H">
4+
<properties>
5+
<property name="dd_tags[systest.suite.context.scenario]" value="MOCK_THE_TEST"/>
6+
</properties>
7+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_pass[spring]" time="0.000">
8+
<properties>
9+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
10+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
11+
<property name="dd_tags[systest.case.outcome]" value="passed"/>
12+
<property name="dd_tags[test.final_status]" value="pass"/>
13+
</properties>
14+
</testcase>
15+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_fail[spring]" time="0.000">
16+
<properties>
17+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
18+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
19+
<property name="dd_tags[systest.case.outcome]" value="failed"/>
20+
<property name="dd_tags[test.final_status]" value="fail"/>
21+
</properties>
22+
<failure message="Failed: dummy">self = &lt;tests.test_the_test.test_junit.Test_Cases object at 0x10cddf6b0&gt;
23+
24+
def test_fail(self):
25+
&gt; pytest.fail("dummy")
26+
E Failed: dummy
27+
28+
tests/test_the_test/test_junit.py:25: Failed</failure>
29+
</testcase>
30+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_skipped[spring]" time="0.000">
31+
<properties>
32+
<property name="dd_tags[systest.case.outcome]" value="skipped"/>
33+
<property name="dd_tags[test.final_status]" value="skip"/>
34+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
35+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
36+
<property name="dd_tags[systest.case.declaration]" value="irrelevant"/>
37+
</properties>
38+
<skipped type="pytest.skip" message="irrelevant">/Users/nicolas.catoni/repos/system-tests/tests/test_the_test/test_junit.py:27: irrelevant</skipped>
39+
</testcase>
40+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_xfail[spring]" time="0.000">
41+
<properties>
42+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
43+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
44+
<property name="dd_tags[systest.case.declaration]" value="bug"/>
45+
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
46+
<property name="dd_tags[systest.case.outcome]" value="xfailed"/>
47+
<property name="dd_tags[test.final_status]" value="skip"/>
48+
</properties>
49+
<skipped type="pytest.xfail" message="bug (APMRP-360)"/>
50+
</testcase>
51+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_xpass[spring]" time="0.000">
52+
<properties>
53+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
54+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
55+
<property name="dd_tags[systest.case.declaration]" value="bug"/>
56+
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
57+
<property name="dd_tags[systest.case.outcome]" value="xpassed"/>
58+
<property name="dd_tags[test.final_status]" value="pass"/>
59+
</properties>
60+
</testcase>
61+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_flaky[spring]" time="0.000">
62+
<properties>
63+
<property name="dd_tags[systest.case.outcome]" value="skipped"/>
64+
<property name="dd_tags[test.final_status]" value="skip"/>
65+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
66+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
67+
<property name="dd_tags[systest.case.declaration]" value="flaky"/>
68+
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
69+
</properties>
70+
<skipped type="pytest.skip" message="flaky (APMRP-360)">/Users/nicolas.catoni/repos/system-tests/tests/test_the_test/test_junit.py:39: flaky (APMRP-360)</skipped>
71+
</testcase>
72+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_error_on_bug[spring]" time="0.000">
73+
<properties>
74+
<property name="dd_tags[systest.case.outcome]" value="xfailed"/>
75+
<property name="dd_tags[test.final_status]" value="skip"/>
76+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
77+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
78+
<property name="dd_tags[systest.case.declaration]" value="bug"/>
79+
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
80+
</properties>
81+
<skipped type="pytest.xfail" message="bug (APMRP-360)"/>
82+
</testcase>
83+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_error_on_missing_feature[spring]" time="0.000">
84+
<properties>
85+
<property name="dd_tags[systest.case.outcome]" value="xfailed"/>
86+
<property name="dd_tags[test.final_status]" value="skip"/>
87+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
88+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
89+
<property name="dd_tags[systest.case.declaration]" value="missing_feature"/>
90+
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
91+
</properties>
92+
<skipped type="pytest.xfail" message="missing_feature (APMRP-360)"/>
93+
</testcase>
94+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_error[spring]" time="0.000">
95+
<properties>
96+
<property name="dd_tags[systest.case.outcome]" value="error"/>
97+
<property name="dd_tags[test.final_status]" value="fail"/>
98+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
99+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
100+
</properties>
101+
<error message="failed on setup with &quot;TypeError: nope!&quot;">@pytest.fixture
13102
def error_fixture():
14103
&gt; raise TypeError("nope!")
15104
E TypeError: nope!
16105

17106
tests/test_the_test/test_junit.py:15: TypeError</error>
18-
</testcase>
19-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_error_on_bug[spring]" time="0.000">
20-
<properties>
21-
<property name="dd_tags[systest.case.outcome]" value="xfailed" />
22-
<property name="dd_tags[test.final_status]" value="skip"/>
23-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]" />
24-
<property name="dd_tags[systest.case.declaration]" value="bug" />
25-
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360" />
26-
</properties>
27-
<skipped type="pytest.xfail" message="bug (APMRP-360)" />
28-
</testcase>
29-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_error_on_missing_feature[spring]" time="0.000">
30-
<properties>
31-
<property name="dd_tags[systest.case.outcome]" value="xfailed" />
32-
<property name="dd_tags[test.final_status]" value="skip"/>
33-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]" />
34-
<property name="dd_tags[systest.case.declaration]" value="missing_feature" />
35-
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360" />
36-
</properties>
37-
<skipped type="pytest.xfail" message="missing_feature (APMRP-360)" />
38-
</testcase>
39-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_fail[spring]" time="0.000">
40-
<properties>
41-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
42-
<property name="dd_tags[systest.case.outcome]" value="failed"/>
43-
<property name="dd_tags[test.final_status]" value="fail"/>
44-
</properties>
45-
<failure message="Failed: dummy">self = &lt;tests.test_the_test.test_junit.Test_Cases object at 0x10cf01eb0&gt;
46-
def test_fail(self):
47-
&gt; pytest.fail("dummy")
48-
E Failed: dummy
49-
tests/test_the_test/test_junit.py:20: Failed</failure>
50-
</testcase>
51-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_flaky[spring]" time="0.000">
52-
<properties>
53-
<property name="dd_tags[systest.case.outcome]" value="skipped"/>
54-
<property name="dd_tags[test.final_status]" value="skip"/>
55-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
56-
<property name="dd_tags[systest.case.declaration]" value="flaky"/>
57-
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
58-
</properties>
59-
<skipped type="pytest.skip" message="flaky (APMRP-360)">/Users/charles.debeauchesne/repos/system-tests/tests/test_the_test/test_junit.py:34: flaky (APMRP-360)</skipped>
60-
</testcase>
61-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_force_skip[spring]">
62-
<properties>
63-
<property name="dd_tags[systest.case.outcome]" value="skipped"/>
64-
<property name="dd_tags[test.final_status]" value="skip"/>
65-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
66-
<property name="dd_tags[systest.case.declaration]" value="bug"/>
67-
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
68-
</properties>
69-
<skipped message="bug (APMRP-360)" type="pytest.skip">system-tests/tests/test_the_test/test_junit.py:01: bug (APMRP-360)</skipped>
70-
</testcase>
71-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_pass[spring]">
72-
<properties>
73-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
74-
<property name="dd_tags[systest.case.outcome]" value="passed"/>
75-
<property name="dd_tags[test.final_status]" value="pass"/>
76-
</properties>
77-
</testcase>
78-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_skipped[spring]" time="0.000">
79-
<properties>
80-
<property name="dd_tags[systest.case.outcome]" value="skipped"/>
81-
<property name="dd_tags[test.final_status]" value="skip"/>
82-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
83-
<property name="dd_tags[systest.case.declaration]" value="irrelevant"/>
84-
</properties>
85-
<skipped type="pytest.skip" message="irrelevant">/Users/charles.debeauchesne/repos/system-tests/tests/test_the_test/test_junit.py:22: irrelevant</skipped>
86-
</testcase>
87-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_xfail[spring]" time="0.000">
88-
<properties>
89-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
90-
<property name="dd_tags[systest.case.declaration]" value="bug"/>
91-
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
92-
<property name="dd_tags[systest.case.outcome]" value="xfailed"/>
93-
<property name="dd_tags[test.final_status]" value="skip"/>
94-
</properties>
95-
<skipped type="pytest.xfail" message="bug (APMRP-360)" />
96-
</testcase>
97-
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_xpass[spring]" time="0.000">
98-
<properties>
99-
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
100-
<property name="dd_tags[systest.case.declaration]" value="bug"/>
101-
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
102-
<property name="dd_tags[systest.case.outcome]" value="xpassed"/>
103-
<property name="dd_tags[test.final_status]" value="pass"/>
104-
</properties>
105-
</testcase>
106-
</testsuite>
107+
</testcase>
108+
<testcase name="tests.test_the_test.test_junit.Test_Cases.test_force_skip[spring]" time="0.000">
109+
<properties>
110+
<property name="dd_tags[systest.case.outcome]" value="skipped"/>
111+
<property name="dd_tags[test.final_status]" value="skip"/>
112+
<property name="test.codeowners" value="[&quot;@DataDog/apm-sdk-capabilities&quot;]"/>
113+
<property name="dd_tags[systest.case.feature_id]" value="346"/>
114+
<property name="dd_tags[systest.case.declaration]" value="bug"/>
115+
<property name="dd_tags[systest.case.declarationDetails]" value="APMRP-360"/>
116+
</properties>
117+
<skipped type="pytest.skip" message="bug (APMRP-360)">/Users/nicolas.catoni/repos/system-tests/tests/test_the_test/test_junit.py:50: bug (APMRP-360)</skipped>
118+
</testcase>
119+
</testsuite>
107120
</testsuites>

0 commit comments

Comments
 (0)