Add OTLP runtime metrics end-to-end scenario and test

link04 · claude · link04 · commit f74c21d2e5b2 · 2026-04-22T11:28:54.000-04:00
Add OTLP_RUNTIME_METRICS scenario that validates dd-trace libraries
export runtime metrics via OTLP with OTel semantic convention names
instead of DD-proprietary names.

- New scenario routes OTLP metrics through proxy to agent using
  standard dd-otlp-path=agent header pattern
- Test reads from interfaces.agent.get_metrics(), matching how
  RUNTIME_METRICS_ENABLED tests work
- Expected metrics defined per language (dotnet, go, nodejs, java),
  all set to missing_feature until tracer PRs land
- Registered in CI workflow and orchestrator
- Fix parametric test_otel_metrics leaking runtime metrics by
  setting DD_RUNTIME_METRICS_ENABLED=false in DEFAULT_ENVVARS

Co-Authored-By: Claude Opus 4.6 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml
@@ -261,6 +261,9 @@ jobs:
     - name: Run LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID"')
       run: ./run.sh LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID
+    - name: Run OTLP_RUNTIME_METRICS scenario
+      if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"OTLP_RUNTIME_METRICS"')
+      run: ./run.sh OTLP_RUNTIME_METRICS
     - name: Run RUNTIME_METRICS_ENABLED scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"RUNTIME_METRICS_ENABLED"')
       run: ./run.sh RUNTIME_METRICS_ENABLED
diff --git a/manifests/cpp.yml b/manifests/cpp.yml
@@ -310,5 +310,6 @@ manifest:
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_required_headers: missing_feature
diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml
@@ -1124,6 +1124,7 @@ manifest:
   tests/test_library_conf.py::Test_HeaderTags_Wildcard_Response_Headers: missing_feature
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_profiling.py::Test_Profile:
     - weblog_declaration:
         "*": v1.0  # real version not known
diff --git a/manifests/golang.yml b/manifests/golang.yml
@@ -1395,6 +1395,7 @@ manifest:
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_profiling.py::Test_Profile::test_process_tags_svc: missing_feature
   tests/test_protobuf.py: missing_feature
   tests/test_resource_renaming.py:
diff --git a/manifests/java.yml b/manifests/java.yml
@@ -4220,6 +4220,7 @@ manifest:
         akka-http: irrelevant (integration injects Date header after bytecode injection occurs)
         play: irrelevant (integration injects Date header after bytecode injection occurs)
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_profiling.py::Test_Profile:
     - weblog_declaration:
         akka-http: v1.22.0
diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml
@@ -2311,6 +2311,7 @@ manifest:
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_profiling.py::Test_Profile: *ref_5_16_0
   tests/test_profiling.py::Test_Profile::test_process_tags: missing_feature
   tests/test_profiling.py::Test_Profile::test_process_tags_svc: missing_feature
diff --git a/manifests/php.yml b/manifests/php.yml
@@ -997,6 +997,7 @@ manifest:
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_profiling.py::Test_Profile:
     - declaration: missing_feature (profiling seems not to be activated)
       component_version: <1.16.0
diff --git a/manifests/python.yml b/manifests/python.yml
@@ -2153,6 +2153,7 @@ manifest:
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_profiling.py::Test_Profile:
     - weblog_declaration:
         "*": v0.1  # actual version unknown
diff --git a/manifests/ruby.yml b/manifests/ruby.yml
@@ -1961,6 +1961,7 @@ manifest:
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_otlp_runtime_metrics.py::Test_OtlpRuntimeMetrics: missing_feature
   tests/test_profiling.py::Test_Profile:  # Modified by easy win activation script
     - declaration: missing_feature (temporary fix, scenario not working on dd-trace-rb CI)
       component_version: <2.24.0
diff --git a/tests/parametric/test_otel_metrics.py b/tests/parametric/test_otel_metrics.py
@@ -43,6 +43,7 @@
 #   CORECLR_ENABLE_PROFILING=1 is required in .NET to enable auto-instrumentation
 DEFAULT_ENVVARS = {
     "DD_METRICS_OTEL_ENABLED": "true",
+    "DD_RUNTIME_METRICS_ENABLED": "false",  # Prevent runtime metrics (System.Runtime, Microsoft.AspNetCore.*) from leaking into custom metric tests
     "OTEL_METRIC_EXPORT_INTERVAL": "60000",  # Mitigate test flake by increasing the interval so that the only time new metrics are exported are when we manually flush them
     "CORECLR_ENABLE_PROFILING": "1",
 }
diff --git a/tests/test_otlp_runtime_metrics.py b/tests/test_otlp_runtime_metrics.py
@@ -0,0 +1,135 @@
+"""Test that runtime metrics are exported via OTLP using OTel semantic convention names.
+
+When DD_RUNTIME_METRICS_ENABLED=true and DD_METRICS_OTEL_ENABLED=true, dd-trace-*
+libraries should send runtime metrics via OTLP with OTel-native naming (dotnet.*,
+jvm.*, go.*, v8js.*, etc.) instead of DD-proprietary naming (runtime.dotnet.*,
+runtime.go.*, runtime.node.*, etc.).
+"""
+
+from utils import context, features, interfaces, scenarios, weblog
+
+
+# All OTel semconv metric names that MUST be present per language.
+# These are the complete instrument sets from each tracer's OTLP runtime metrics implementation.
+EXPECTED_METRICS = {
+    "dotnet": [
+        "dotnet.assembly.count",
+        "dotnet.exceptions",
+        "dotnet.gc.collections",
+        "dotnet.gc.heap.total_allocated",
+        "dotnet.gc.last_collection.heap.fragmentation.size",
+        "dotnet.gc.last_collection.heap.size",
+        "dotnet.gc.last_collection.memory.committed_size",
+        "dotnet.gc.pause.time",
+        "dotnet.jit.compilation.time",
+        "dotnet.jit.compiled_il.size",
+        "dotnet.jit.compiled_methods",
+        "dotnet.monitor.lock_contentions",
+        "dotnet.process.cpu.count",
+        "dotnet.process.cpu.time",
+        "dotnet.process.memory.working_set",
+        "dotnet.thread_pool.queue.length",
+        "dotnet.thread_pool.thread.count",
+        "dotnet.thread_pool.work_item.count",
+        "dotnet.timer.count",
+    ],
+    "golang": [
+        "go.config.gogc",
+        "go.goroutine.count",
+        "go.memory.allocated",
+        "go.memory.allocations",
+        "go.memory.gc.goal",
+        "go.memory.limit",
+        "go.memory.used",
+        "go.processor.limit",
+    ],
+    "nodejs": [
+        "nodejs.eventloop.delay.max",
+        "nodejs.eventloop.delay.mean",
+        "nodejs.eventloop.delay.min",
+        "nodejs.eventloop.delay.p50",
+        "nodejs.eventloop.delay.p90",
+        "nodejs.eventloop.delay.p99",
+        "nodejs.eventloop.utilization",
+        "process.cpu.utilization",
+        "process.memory.usage",
+        "v8js.memory.heap.limit",
+        "v8js.memory.heap.space.available_size",
+        "v8js.memory.heap.space.physical_size",
+        "v8js.memory.heap.used",
+    ],
+    "java": [
+        "jvm.buffer.count",
+        "jvm.buffer.memory.limit",
+        "jvm.buffer.memory.used",
+        "jvm.class.count",
+        "jvm.class.loaded",
+        "jvm.class.unloaded",
+        "jvm.cpu.count",
+        "jvm.cpu.recent_utilization",
+        "jvm.cpu.time",
+        "jvm.file_descriptor.count",
+        "jvm.file_descriptor.limit",
+        "jvm.memory.committed",
+        "jvm.memory.init",
+        "jvm.memory.limit",
+        "jvm.memory.used",
+        "jvm.memory.used_after_last_gc",
+        "jvm.system.cpu.utilization",
+        "jvm.thread.count",
+    ],
+}
+
+# DD-proprietary prefixes that should NOT appear when OTLP metrics are active
+DD_PROPRIETARY_PREFIXES = {
+    "dotnet": "runtime.dotnet.",
+    "golang": "runtime.go.",
+    "nodejs": "runtime.node.",
+    "java": "jvm.heap_memory",
+}
+
+
+def get_runtime_metric_names():
+    """Extract runtime metric names from the agent interface (agent -> backend series).
+
+    Uses interfaces.agent.get_metrics() — the same approach as
+    Test_Config_RuntimeMetrics_Enabled in test_config_consistency.py.
+    """
+    metric_names = set()
+    for _, metric in interfaces.agent.get_metrics():
+        metric_names.add(metric["metric"])
+    return metric_names
+
+
+@scenarios.otlp_runtime_metrics
+@features.runtime_metrics
+class Test_OtlpRuntimeMetrics:
+    """Verify runtime metrics are sent via OTLP with OTel names, not DD-proprietary names."""
+
+    def setup_main(self):
+        self.req = weblog.get("/")
+
+    def test_main(self):
+        assert self.req.status_code == 200
+
+        library = context.library.name
+        if library not in EXPECTED_METRICS:
+            return
+
+        metric_names = get_runtime_metric_names()
+
+        # All expected OTel-named metrics must be present
+        expected = EXPECTED_METRICS[library]
+        for expected_name in expected:
+            assert expected_name in metric_names, (
+                f"Expected OTel runtime metric '{expected_name}' not found for {library}. "
+                f"Got metrics: {sorted(metric_names)}"
+            )
+
+        # DD-proprietary names must NOT be present
+        dd_prefix = DD_PROPRIETARY_PREFIXES.get(library)
+        if dd_prefix:
+            dd_named_metrics = [n for n in metric_names if n.startswith(dd_prefix)]
+            assert len(dd_named_metrics) == 0, (
+                f"Found DD-proprietary metric names for {library}: {dd_named_metrics}. Expected OTel-native names only."
+            )
diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py
@@ -1188,6 +1188,20 @@ class _Scenarios:
         doc="Test runtime metrics",
     )
 
+    otlp_runtime_metrics = EndToEndScenario(
+        "OTLP_RUNTIME_METRICS",
+        weblog_env={
+            "DD_METRICS_OTEL_ENABLED": "true",
+            "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
+            "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/metrics",
+            "OTEL_EXPORTER_OTLP_METRICS_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent",
+        },
+        runtime_metrics_enabled=True,
+        include_opentelemetry=True,
+        library_interface_timeout=20,
+        doc="Test runtime metrics exported via OTLP with OTel semantic convention names",
+    )
+
     # Appsec Lambda Scenarios
     appsec_lambda_default = LambdaScenario(
         "APPSEC_LAMBDA_DEFAULT",
diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py
@@ -635,6 +635,7 @@ def _is_supported(library: str, weblog: str, scenario: str, _ci_environment: str
             "IPV6",
             "LIBRARY_CONF_CUSTOM_HEADER_TAGS",
             "LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID",
+            "OTLP_RUNTIME_METRICS",
             "PERFORMANCES",
             "PROFILING",
             "REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD",

Original file line number	Diff line number	Diff line change
`@@ -43,6 +43,7 @@`
`43`	`43`	`# CORECLR_ENABLE_PROFILING=1 is required in .NET to enable auto-instrumentation`
`44`	`44`	`DEFAULT_ENVVARS = {`
`45`	`45`	`"DD_METRICS_OTEL_ENABLED": "true",`
	`46`	`+ "DD_RUNTIME_METRICS_ENABLED": "false", # Prevent runtime metrics (System.Runtime, Microsoft.AspNetCore.*) from leaking into custom metric tests`
`46`	`47`	`"OTEL_METRIC_EXPORT_INTERVAL": "60000", # Mitigate test flake by increasing the interval so that the only time new metrics are exported are when we manually flush them`
`47`	`48`	`"CORECLR_ENABLE_PROFILING": "1",`
`48`	`49`	`}`