Skip to content

Commit 03abb49

Browse files
mhliddmhlidd
authored
Add Attributes to OTLP Metrics Test (#7039)
1 parent 56306b6 commit 03abb49

3 files changed

Lines changed: 224 additions & 89 deletions

File tree

‎manifests/java.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4260,7 +4260,10 @@ manifest:
42604260
play: irrelevant (integration injects Date header after bytecode injection occurs)
42614261
tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
42624262
tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics::test_dd_metrics_are_absent: bug (APMAPI-1952)
4263-
tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics::test_otel_metrics_are_present: missing_feature
4263+
tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics::test_otel_metrics_are_present_and_attributed:
4264+
- weblog_declaration:
4265+
"*": v1.63.0-SNAPSHOT
4266+
spring-boot-3-native: irrelevant (GraalVM. Tracing support only)
42644267
tests/test_profiling.py::Test_Profile:
42654268
- weblog_declaration:
42664269
akka-http: v1.22.0

‎tests/test_otlp_runtime_metrics.py‎

Lines changed: 219 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -9,133 +9,264 @@
99
from utils import context, features, interfaces, scenarios, weblog
1010

1111

12-
# All OTel semconv metric names that MUST be present per language.
13-
# These are the complete instrument sets from each tracer's OTLP runtime metrics implementation.
14-
EXPECTED_METRICS = {
15-
"dotnet": [
16-
"dotnet.assembly.count",
17-
"dotnet.exceptions",
18-
"dotnet.gc.collections",
19-
"dotnet.gc.heap.total_allocated",
20-
"dotnet.gc.last_collection.heap.fragmentation.size",
21-
"dotnet.gc.last_collection.heap.size",
22-
"dotnet.gc.last_collection.memory.committed_size",
23-
"dotnet.gc.pause.time",
24-
"dotnet.jit.compilation.time",
25-
"dotnet.jit.compiled_il.size",
26-
"dotnet.jit.compiled_methods",
27-
"dotnet.monitor.lock_contentions",
28-
"dotnet.process.cpu.count",
29-
"dotnet.process.cpu.time",
30-
"dotnet.process.memory.working_set",
31-
"dotnet.thread_pool.queue.length",
32-
"dotnet.thread_pool.thread.count",
33-
"dotnet.thread_pool.work_item.count",
34-
"dotnet.timer.count",
35-
],
36-
"golang": [
37-
"go.config.gogc",
38-
"go.goroutine.count",
39-
"go.memory.allocated",
40-
"go.memory.allocations",
41-
"go.memory.gc.goal",
42-
"go.memory.limit",
43-
"go.memory.used",
44-
"go.processor.limit",
45-
],
46-
"nodejs": [
47-
"nodejs.eventloop.delay.max",
48-
"nodejs.eventloop.delay.mean",
49-
"nodejs.eventloop.delay.min",
50-
"nodejs.eventloop.delay.p50",
51-
"nodejs.eventloop.delay.p90",
52-
"nodejs.eventloop.delay.p99",
53-
"nodejs.eventloop.utilization",
54-
"process.cpu.utilization",
55-
"process.memory.usage",
56-
"v8js.memory.heap.limit",
57-
"v8js.memory.heap.space.available_size",
58-
"v8js.memory.heap.space.physical_size",
59-
"v8js.memory.heap.used",
60-
],
61-
"java": [
62-
"jvm.buffer.count",
63-
"jvm.buffer.memory.limit",
64-
"jvm.buffer.memory.used",
65-
"jvm.class.count",
66-
"jvm.class.loaded",
67-
"jvm.class.unloaded",
68-
"jvm.cpu.count",
69-
"jvm.cpu.recent_utilization",
70-
"jvm.cpu.time",
71-
"jvm.memory.committed",
72-
"jvm.memory.init",
73-
"jvm.memory.limit",
74-
"jvm.memory.used",
75-
"jvm.memory.used_after_last_gc",
76-
"jvm.thread.count",
77-
],
12+
# Maps each expected metric to its attribute constraints:
13+
# "all" — attribute keys that must appear on every emitted data point.
14+
# "some" — attribute keys that must appear on at least one data point.
15+
#
16+
# Use "some" when a metric emits both per-dimension points (which carry the attribute)
17+
# and aggregate rollup points (which don't). For example, jvm.memory.used emits one
18+
# point per memory pool (with jvm.memory.pool.name) as well as heap/non-heap totals
19+
# (without it), so pool.name goes in "some" while jvm.memory.type, which is present
20+
# on every point, goes in "all".
21+
EXPECTED_METRICS: dict[str, dict[str, dict[str, list[str]]]] = {
22+
"dotnet": {
23+
"dotnet.assembly.count": {"all": []},
24+
"dotnet.exceptions": {"all": []},
25+
"dotnet.gc.collections": {"all": []},
26+
"dotnet.gc.heap.total_allocated": {"all": []},
27+
"dotnet.gc.last_collection.heap.fragmentation.size": {"all": []},
28+
"dotnet.gc.last_collection.heap.size": {"all": []},
29+
"dotnet.gc.last_collection.memory.committed_size": {"all": []},
30+
"dotnet.gc.pause.time": {"all": []},
31+
"dotnet.jit.compilation.time": {"all": []},
32+
"dotnet.jit.compiled_il.size": {"all": []},
33+
"dotnet.jit.compiled_methods": {"all": []},
34+
"dotnet.monitor.lock_contentions": {"all": []},
35+
"dotnet.process.cpu.count": {"all": []},
36+
"dotnet.process.cpu.time": {"all": []},
37+
"dotnet.process.memory.working_set": {"all": []},
38+
"dotnet.thread_pool.queue.length": {"all": []},
39+
"dotnet.thread_pool.thread.count": {"all": []},
40+
"dotnet.thread_pool.work_item.count": {"all": []},
41+
"dotnet.timer.count": {"all": []},
42+
},
43+
"golang": {
44+
"go.config.gogc": {"all": []},
45+
"go.goroutine.count": {"all": []},
46+
"go.memory.allocated": {"all": []},
47+
"go.memory.allocations": {"all": []},
48+
"go.memory.gc.goal": {"all": []},
49+
"go.memory.limit": {"all": []},
50+
"go.memory.used": {"all": []},
51+
"go.processor.limit": {"all": []},
52+
},
53+
"nodejs": {
54+
"nodejs.eventloop.delay.max": {"all": []},
55+
"nodejs.eventloop.delay.mean": {"all": []},
56+
"nodejs.eventloop.delay.min": {"all": []},
57+
"nodejs.eventloop.delay.p50": {"all": []},
58+
"nodejs.eventloop.delay.p90": {"all": []},
59+
"nodejs.eventloop.delay.p99": {"all": []},
60+
"nodejs.eventloop.utilization": {"all": []},
61+
"process.cpu.utilization": {"all": []},
62+
"process.memory.usage": {"all": []},
63+
"v8js.memory.heap.limit": {"all": []},
64+
"v8js.memory.heap.space.available_size": {"all": []},
65+
"v8js.memory.heap.space.physical_size": {"all": []},
66+
"v8js.memory.heap.used": {"all": []},
67+
},
68+
"java": {
69+
"jvm.buffer.count": {"all": ["jvm.buffer.pool.name"]},
70+
"jvm.buffer.memory.limit": {"all": ["jvm.buffer.pool.name"]},
71+
"jvm.buffer.memory.used": {"all": ["jvm.buffer.pool.name"]},
72+
"jvm.class.count": {"all": []},
73+
"jvm.class.loaded": {"all": []},
74+
"jvm.class.unloaded": {"all": []},
75+
"jvm.cpu.count": {"all": []},
76+
"jvm.cpu.recent_utilization": {"all": []},
77+
"jvm.cpu.time": {"all": []},
78+
# jvm.gc.duration is a histogram; the agent surfaces it as .count/.sum/.min/.max series.
79+
"jvm.gc.duration.count": {"all": ["jvm.gc.name", "jvm.gc.action", "jvm.gc.cause"]},
80+
"jvm.gc.duration.sum": {"all": ["jvm.gc.name", "jvm.gc.action", "jvm.gc.cause"]},
81+
"jvm.gc.duration.min": {"all": ["jvm.gc.name", "jvm.gc.action", "jvm.gc.cause"]},
82+
"jvm.gc.duration.max": {"all": ["jvm.gc.name", "jvm.gc.action", "jvm.gc.cause"]},
83+
# memory metrics emit per-pool points (with pool.name) and aggregate totals (without),
84+
# so jvm.memory.type is required on all points and pool.name on at least one.
85+
"jvm.memory.committed": {"all": ["jvm.memory.type"], "some": ["jvm.memory.pool.name"]},
86+
"jvm.memory.init": {"all": ["jvm.memory.type"], "some": ["jvm.memory.pool.name"]},
87+
"jvm.memory.limit": {"all": ["jvm.memory.type"], "some": ["jvm.memory.pool.name"]},
88+
"jvm.memory.used": {"all": ["jvm.memory.type"], "some": ["jvm.memory.pool.name"]},
89+
# used_after_last_gc only emits per-pool points — both attributes are always present.
90+
"jvm.memory.used_after_last_gc": {"all": ["jvm.memory.pool.name", "jvm.memory.type"]},
91+
"jvm.thread.count": {"all": ["jvm.thread.daemon", "jvm.thread.state"]},
92+
# experimental metrics (on by default); no domain-specific attributes.
93+
"jvm.system.cpu.utilization": {"all": []},
94+
"jvm.system.cpu.load_1m": {"all": []},
95+
"jvm.file_descriptor.count": {"all": []},
96+
"jvm.file_descriptor.limit": {"all": []},
97+
},
7898
}
7999

80-
# DD-proprietary prefixes that should NOT appear when OTLP metrics are active
81-
DD_PROPRIETARY_PREFIXES = {
100+
# Valid value domains for attributes. For closed enums (jvm.memory.type, jvm.thread.*) these are
101+
# exhaustive. For open-ended attributes (pool names, GC names) these are supersets covering all
102+
# known JVM/GC implementations — the assertion is that observed values fall within the known universe.
103+
EXPECTED_METRIC_ATTRIBUTE_VALUES: dict[str, dict[str, frozenset[str]]] = {
104+
"java": {
105+
"jvm.memory.type": frozenset({"heap", "non_heap"}),
106+
"jvm.thread.daemon": frozenset({"true", "false"}),
107+
"jvm.thread.state": frozenset({"new", "runnable", "blocked", "waiting", "timed_waiting", "terminated"}),
108+
# Pool names vary by GC algorithm; superset across G1GC, ZGC, Shenandoah, ParallelGC, SerialGC, CMS.
109+
# Values are emitted as raw JMX strings (mixed case, spaces, quotes preserved).
110+
"jvm.memory.pool.name": frozenset(
111+
{
112+
# G1GC
113+
"G1 Eden Space",
114+
"G1 Survivor Space",
115+
"G1 Old Gen",
116+
# ZGC
117+
"ZHeap",
118+
# Shenandoah
119+
"Shenandoah",
120+
# ParallelGC
121+
"PS Eden Space",
122+
"PS Survivor Space",
123+
"PS Old Gen",
124+
# SerialGC / CMS young
125+
"Eden Space",
126+
"Survivor Space",
127+
# SerialGC old
128+
"Tenured Gen",
129+
# CMS
130+
"Par Eden Space",
131+
"Par Survivor Space",
132+
"CMS Old Gen",
133+
# Non-heap regions common across all GCs
134+
"Metaspace",
135+
"Compressed Class Space",
136+
# Code Cache (monolithic, JDK < 9 or -XX:-SegmentedCodeCache)
137+
"Code Cache",
138+
# Segmented Code Cache (JDK 9+)
139+
"CodeHeap 'non-nmethods'",
140+
"CodeHeap 'profiled nmethods'",
141+
"CodeHeap 'non-profiled nmethods'",
142+
}
143+
),
144+
# Buffer pool names are stable across JVM versions.
145+
"jvm.buffer.pool.name": frozenset(
146+
{
147+
"direct",
148+
"mapped",
149+
"mapped - 'non-volatile memory'", # JDK 14+ non-volatile MappedByteBuffer
150+
}
151+
),
152+
# GC collector names vary by algorithm.
153+
"jvm.gc.name": frozenset(
154+
{
155+
# G1GC
156+
"G1 Young Generation",
157+
"G1 Old Generation",
158+
"G1 Concurrent GC",
159+
# ZGC
160+
"ZGC",
161+
"ZGC Pauses",
162+
"ZGC Cycles",
163+
# Shenandoah
164+
"Shenandoah Cycles",
165+
"Shenandoah Pauses",
166+
# ParallelGC
167+
"PS Scavenge",
168+
"PS MarkSweep",
169+
# SerialGC
170+
"Copy",
171+
"MarkSweepCompact",
172+
# CMS (deprecated but still encountered)
173+
"ParNew",
174+
"ConcurrentMarkSweep",
175+
}
176+
),
177+
"jvm.gc.action": frozenset({"end of minor GC", "end of major GC", "end of GC cycle"}),
178+
},
179+
}
180+
181+
# DD-proprietary prefixes that should NOT appear when OTLP metrics are active.
182+
DD_PROPRIETARY_PREFIXES: dict[str, str] = {
82183
"dotnet": "runtime.dotnet.",
83184
"golang": "runtime.go.",
84185
"nodejs": "runtime.node.",
85186
"java": "jvm.heap_memory",
86187
}
87188

88189

89-
def get_runtime_metric_names():
90-
"""Extract runtime metric names from the agent interface (agent -> backend series).
190+
def get_runtime_metrics_by_name() -> dict[str, list[dict[str, str]]]:
191+
"""Return observed runtime metrics grouped by name.
91192
92-
Uses interfaces.agent.get_metrics() — the same approach as
93-
Test_Config_RuntimeMetrics_Enabled in test_config_consistency.py.
193+
Each entry maps metric name -> list of tag dicts, one per data point.
194+
Tags are parsed from "key:value" strings in the agent series.
94195
"""
95-
metric_names = set()
196+
result: dict[str, list[dict[str, str]]] = {}
96197
for _, metric in interfaces.agent.get_metrics():
97-
metric_names.add(metric["metric"])
98-
return metric_names
198+
name: str = metric["metric"]
199+
tags: dict[str, str] = dict(tag.split(":", 1) for tag in metric.get("tags", []) if ":" in tag)
200+
result.setdefault(name, []).append(tags)
201+
return result
99202

100203

101204
@scenarios.otlp_runtime_metrics
102205
@features.runtime_metrics
103206
class Test_OtlpRuntimeMetrics:
104207
"""Verify runtime metrics are sent via OTLP with OTel names, not DD-proprietary names."""
105208

106-
def setup_otel_metrics_are_present(self):
209+
def setup_otel_metrics_are_present_and_attributed(self) -> None:
107210
self.req = weblog.get("/")
108211

109-
def test_otel_metrics_are_present(self):
212+
def test_otel_metrics_are_present_and_attributed(self) -> None:
110213
assert self.req.status_code == 200
111214

112215
library = context.library.name
113216
if library not in EXPECTED_METRICS:
114217
return
115218

116-
metric_names = get_runtime_metric_names()
219+
observed = get_runtime_metrics_by_name()
220+
attribute_values = EXPECTED_METRIC_ATTRIBUTE_VALUES.get(library, {})
117221

118-
expected = EXPECTED_METRICS[library]
119-
for expected_name in expected:
120-
assert expected_name in metric_names, (
121-
f"Expected OTel runtime metric '{expected_name}' not found for {library}. "
122-
f"Got metrics: {sorted(metric_names)}"
222+
for metric_name, constraints in EXPECTED_METRICS[library].items():
223+
assert metric_name in observed, (
224+
f"Expected OTel runtime metric '{metric_name}' not found for {library}. "
225+
f"Got metrics: {sorted(observed.keys())}"
123226
)
124227

125-
def setup_dd_metrics_are_absent(self):
228+
points = observed[metric_name]
229+
all_keys = constraints.get("all", [])
230+
some_keys = constraints.get("some", [])
231+
232+
for point_tags in points:
233+
for key in all_keys:
234+
assert key in point_tags, (
235+
f"Metric '{metric_name}' data point missing required attribute '{key}' "
236+
f"for {library}. Got tags: {point_tags}"
237+
)
238+
if key in attribute_values:
239+
assert point_tags[key] in attribute_values[key], (
240+
f"Metric '{metric_name}' attribute '{key}' has invalid value "
241+
f"'{point_tags[key]}' for {library}. "
242+
f"Expected one of: {sorted(attribute_values[key])}"
243+
)
244+
245+
for key in some_keys:
246+
assert any(key in point_tags for point_tags in points), (
247+
f"Metric '{metric_name}' has no data point with attribute '{key}' for {library}."
248+
)
249+
for point_tags in points:
250+
if key in point_tags and key in attribute_values:
251+
assert point_tags[key] in attribute_values[key], (
252+
f"Metric '{metric_name}' attribute '{key}' has invalid value "
253+
f"'{point_tags[key]}' for {library}. "
254+
f"Expected one of: {sorted(attribute_values[key])}"
255+
)
256+
257+
def setup_dd_metrics_are_absent(self) -> None:
126258
self.req = weblog.get("/")
127259

128-
def test_dd_metrics_are_absent(self):
260+
def test_dd_metrics_are_absent(self) -> None:
129261
assert self.req.status_code == 200
130262

131263
library = context.library.name
132264
dd_prefix = DD_PROPRIETARY_PREFIXES.get(library)
133265
if not dd_prefix:
134266
return
135267

136-
metric_names = get_runtime_metric_names()
137-
138-
dd_named_metrics = [n for n in metric_names if n.startswith(dd_prefix)]
268+
observed = get_runtime_metrics_by_name()
269+
dd_named_metrics = [n for n in observed if n.startswith(dd_prefix)]
139270
assert len(dd_named_metrics) == 0, (
140271
f"Found DD-proprietary metric names for {library}: {dd_named_metrics}. Expected OTel-native names only."
141272
)

‎utils/_context/_scenarios/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1234,6 +1234,7 @@ class _Scenarios:
12341234
"OTLP_RUNTIME_METRICS",
12351235
weblog_env={
12361236
"DD_METRICS_OTEL_ENABLED": "true",
1237+
"DD_DOGSTATSD_START_DELAY": "0",
12371238
"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
12381239
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/metrics",
12391240
"OTEL_EXPORTER_OTLP_METRICS_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent",

0 commit comments

Comments
 (0)