Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
b0c9cbc
Adding otlp metrics config and polyfill
link04 Apr 14, 2026
5c97823
Adding tests including integration one
link04 Apr 14, 2026
79db8cd
Adding integration tests
link04 Apr 15, 2026
5df066e
Merge branch 'master' into maximo/otlp-runtime-metrics
link04 Apr 15, 2026
e5737b4
Fixing broken unit tests
link04 Apr 15, 2026
855e32f
Fixing metric handling
link04 Apr 20, 2026
314f287
Updating test snapshot and configs
link04 Apr 21, 2026
f541511
Fix broken tests snapshots
link04 Apr 21, 2026
a6b7850
Merge branch 'master' into maximo/otlp-runtime-metrics
link04 Apr 21, 2026
07d2628
Moving and updating test
link04 Apr 21, 2026
e427f9f
Merge branch 'master' into maximo/otlp-runtime-metrics
link04 Apr 21, 2026
151a895
Updating test based on merged fix
link04 Apr 21, 2026
88132cd
Trying to fix broken CI test
link04 Apr 22, 2026
df39724
Removing custom meters to test only Runtime Metrics
link04 Apr 22, 2026
1b582a2
Merge branch 'master' into maximo/otlp-runtime-metrics
link04 Apr 22, 2026
b8baa65
Applying codex fixes
link04 Apr 22, 2026
22ed38a
Merge branch 'maximo/otlp-runtime-metrics' of github.com:DataDog/dd-t…
link04 Apr 22, 2026
3e0c3b5
Addressing most comments
link04 Apr 29, 2026
5578d15
Removing method duplication for getting GetGcPauseTime
link04 Apr 29, 2026
2b2124a
Merge branch 'master' into maximo/otlp-runtime-metrics
link04 Apr 29, 2026
c662ee4
Fixing description of metrics to match System.Runtime
link04 Apr 30, 2026
a8441ae
Fixing snapshot once again... gosh
link04 Apr 30, 2026
42368b6
Adding .NET 7/8 CreateObservableUpDownCounter reflection
link04 May 4, 2026
109909d
Updating test and snapshot
link04 May 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@
</assembly>
<assembly fullname="System.ComponentModel.Primitives">
<type fullname="System.ComponentModel.BrowsableAttribute" />
<type fullname="System.ComponentModel.Component" />
<type fullname="System.ComponentModel.DescriptionAttribute" />
</assembly>
<assembly fullname="System.ComponentModel.TypeConverter">
Expand Down Expand Up @@ -261,10 +262,15 @@
<type fullname="System.Diagnostics.ActivityTraceId" />
<type fullname="System.Diagnostics.DiagnosticListener" />
<type fullname="System.Diagnostics.DistributedContextPropagator" />
<type fullname="System.Diagnostics.Metrics.Counter`1" />
<type fullname="System.Diagnostics.Metrics.Instrument" />
<type fullname="System.Diagnostics.Metrics.Measurement`1" />
<type fullname="System.Diagnostics.Metrics.MeasurementCallback`1" />
<type fullname="System.Diagnostics.Metrics.Meter" />
<type fullname="System.Diagnostics.Metrics.MeterListener" />
<type fullname="System.Diagnostics.Metrics.ObservableCounter`1" />
<type fullname="System.Diagnostics.Metrics.ObservableGauge`1" />
<type fullname="System.Diagnostics.Metrics.ObservableUpDownCounter`1" />
</assembly>
<assembly fullname="System.Diagnostics.Process">
<type fullname="System.Diagnostics.Process" />
Expand Down Expand Up @@ -919,6 +925,7 @@
<type fullname="System.Runtime.InteropServices.GCHandleType" />
<type fullname="System.Runtime.InteropServices.InAttribute" />
<type fullname="System.Runtime.InteropServices.SafeHandle" />
<type fullname="System.Runtime.JitInfo" />
<type fullname="System.Runtime.Serialization.IFormatterConverter" />
<type fullname="System.Runtime.Serialization.ISerializable" />
<type fullname="System.Runtime.Serialization.OnDeserializedAttribute" />
Expand Down
9 changes: 9 additions & 0 deletions tracer/src/Datadog.Trace/Configuration/TracerSettings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,8 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) =
? new HashSet<string>(TrimSplitString(enabledMeters, commaSeparator), StringComparer.Ordinal)
: new HashSet<string>(StringComparer.Ordinal);

OtlpRuntimeMetricsEnabled = OpenTelemetryMetricsEnabled && OtelMetricsExporterEnabled && RuntimeMetricsEnabled;

var disabledActivitySources = config.WithKeys(ConfigurationKeys.DisabledActivitySources).AsString();

DisabledActivitySources = !string.IsNullOrEmpty(disabledActivitySources) ? TrimSplitString(disabledActivitySources, commaSeparator) : [];
Expand Down Expand Up @@ -1152,6 +1154,13 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) =
/// <seealso cref="ConfigurationKeys.RuntimeMetricsDiagnosticsMetricsApiEnabled"/>
internal bool RuntimeMetricsDiagnosticsMetricsApiEnabled { get; }

/// <summary>
/// Gets a value indicating whether runtime metrics should be collected in OTEL format and exported via OTLP.
/// True when runtime metrics are enabled AND (DD_METRICS_OTEL_ENABLED=true AND OTEL_METRICS_EXPORTER=otlp).
/// When true, OTLP takes precedence over DogStatsD for runtime metrics.
/// </summary>
internal bool OtlpRuntimeMetricsEnabled { get; }

/// <summary>
/// Gets a value indicating whether libdatadog data pipeline
/// is enabled.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,15 @@ public MetricPoint CreateSnapshotAndReset()
var previousCumulative = double.IsNaN(_lastObservedCumulative) ? 0 : _lastObservedCumulative;
var delta = _runningDoubleValue - previousCumulative;

// OTel spec: for monotonic counters (ObservableCounter), a negative delta
// indicates a counter reset (e.g. process restart). Report currentValue
// as the delta, as if the previous cumulative was 0.
// ObservableUpDownCounter is non-monotonic so negative deltas are expected.
if (delta < 0 && InstrumentType is InstrumentType.ObservableCounter)
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a thought, and not strictly related to this PR: Do we need a similar guard somewhere for negative values in InstrumentTrype.Counter too? 🤔 I don't think it needs to be right here (because this method is all about observable counters), but I believe we could have the same wrapping case to handle.

I think we may also need to handle this in our statsd case too - I'll look into it!

{
delta = _runningDoubleValue;
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Had to work this one through to convince myself it's correct, so included here so others don't need to 😅

In an overflow scenario - given that you would have (for example)

  • _runningDoubleValue is small (due to overflow) e.g. 10
  • _lastObservedCumulative is huge e.g. 1,000,000,000,000

Then

  • _delta = _runningDoubleValue - previousCumulative ~-1,000,000,000,000

So we hit this branch and set

  • _delta = _runningDoubleValue = ~10

so LGTM!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aay thank you for adding this investigation here!

}

sumForSnapshot = AggregationTemporality == Metrics.AggregationTemporality.Delta
? delta
: _runningDoubleValue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ public void OnInstrumentPublished(Instrument instrument, MeterListener listener)
shouldEnable = !meterName.StartsWith("System.", StringComparison.Ordinal) && !meterName.StartsWith("Microsoft.", StringComparison.Ordinal);
}

if (!shouldEnable && _settings.OtlpRuntimeMetricsEnabled)
{
shouldEnable = meterName is "System.Runtime";
}

if (!shouldEnable)
{
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@

#nullable enable

using System;
using System.Threading;
using System.Threading.Tasks;
using Datadog.Trace.Configuration;
using Datadog.Trace.Logging;
using Datadog.Trace.RuntimeMetrics;

namespace Datadog.Trace.OpenTelemetry.Metrics
{
Expand All @@ -19,21 +22,44 @@ namespace Datadog.Trace.OpenTelemetry.Metrics
internal static class MetricsRuntime
{
private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(MetricsRuntime));
private static readonly object StartLock = new();
private static OtelMetricsPipeline? _instance;
private static RuntimeMetricsPolyfill? _runtimeMetricsPolyfill;

public static void Start(TracerSettings settings)
{
if (_instance != null)
Comment thread
link04 marked this conversation as resolved.
{
Log.Debug("MetricsRuntime already started");
return;
}

var exporter = new OtlpExporter(settings, settings.Manager.InitialExporterSettings);
_instance = new OtelMetricsPipeline(settings, exporter);
_instance.Start();
lock (StartLock)
{
if (_instance != null)
{
Log.Debug("MetricsRuntime already started");
return;
}

if (settings.OtlpRuntimeMetricsEnabled && FrameworkDescription.Instance.RuntimeVersion.Major < 9)
{
try
{
_runtimeMetricsPolyfill = new RuntimeMetricsPolyfill();
Comment thread
link04 marked this conversation as resolved.
Log.Debug("Started RuntimeMetricsPolyfill for .NET {Version} (System.Runtime meter instruments not natively available until .NET 9)", FrameworkDescription.Instance.RuntimeVersion);
}
catch (Exception ex)
{
Log.Error(ex, "Failed to initialize RuntimeMetricsPolyfill for OTLP export");
}
}

var exporter = new OtlpExporter(settings, settings.Manager.InitialExporterSettings);
_instance = new OtelMetricsPipeline(settings, exporter);
_instance.Start();

LifetimeManager.Instance.AddAsyncShutdownTask((_) => StopAsync());
LifetimeManager.Instance.AddAsyncShutdownTask((_) => StopAsync());
}
}

public static Task ForceFlushAsync()
Expand All @@ -54,6 +80,9 @@ public static async Task StopAsync()
}

await _instance.StopAsync().ConfigureAwait(false);

_runtimeMetricsPolyfill?.Dispose();
Comment thread
link04 marked this conversation as resolved.
_runtimeMetricsPolyfill = null;
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Reflection;
using System.Threading;
using Datadog.Trace.DogStatsd;
using Datadog.Trace.Logging;
Expand Down Expand Up @@ -65,29 +64,12 @@ public DiagnosticsMetricsRuntimeMetricsListener(IStatsdManager statsd)
// System.Runtime metrics are only available on .NET 9+, but the only one we need it for is GC pause time
_getGcPauseTimeFunc = GetGcPauseTime_RuntimeMetrics;
}
else if (version.Major > 6
|| version is { Major: 6, Build: >= 21 })
{
// .NET 6.0.21 introduced the GC.GetTotalPauseDuration() method https://github.com/dotnet/runtime/pull/87143
// Which is what OTel uses where required: https://github.com/open-telemetry/opentelemetry-dotnet-contrib/blob/5aa6d868/src/OpenTelemetry.Instrumentation.Runtime/RuntimeMetrics.cs#L105C40-L107
// We could use ducktyping instead of reflection, but this is such a simple case that it's kind of easier
// to just go with the delegate approach
var methodInfo = typeof(GC).GetMethod("GetTotalPauseDuration", BindingFlags.Public | BindingFlags.Static);
if (methodInfo is null)
{
// strange, but we failed to get the delegate
_getGcPauseTimeFunc = GetGcPauseTime_Noop;
}
else
{
var getTotalPauseDuration = methodInfo.CreateDelegate<Func<TimeSpan>>();
_getGcPauseTimeFunc = _ => getTotalPauseDuration().TotalMilliseconds;
}
}
else
{
// can't get pause time
_getGcPauseTimeFunc = GetGcPauseTime_Noop;
var del = GcPauseTimeReflection.TryCreateDelegate();
_getGcPauseTimeFunc = del is null
? GetGcPauseTime_Noop
: (_ => del().TotalMilliseconds);
}

// The .NET runtime instruments we listen to only produce long or double values
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// <copyright file="GcPauseTimeReflection.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>

#if NET6_0_OR_GREATER

#nullable enable

using System;
using System.Reflection;
using Datadog.Trace.Logging;

namespace Datadog.Trace.RuntimeMetrics;

/// <summary>
/// Shared helper for resolving <c>GC.GetTotalPauseDuration()</c> via reflection.
/// Used by both the OTLP polyfill and the DogStatsD runtime metrics listener on .NET 6–8.
/// Each caller is responsible for converting the <see cref="TimeSpan"/> to its preferred unit
/// (seconds for OTel, milliseconds for DogStatsD).
/// </summary>
internal static class GcPauseTimeReflection
{
private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(GcPauseTimeReflection));

/// <summary>
/// Returns a delegate for <c>GC.GetTotalPauseDuration()</c>, or <c>null</c> if the method
/// is not available on the current runtime (i.e. .NET versions older than 6.0.21).
/// </summary>
/// <remarks>
/// <c>GC.GetTotalPauseDuration()</c> was introduced in .NET 6.0.21:
/// https://github.com/dotnet/runtime/pull/87143
/// This is also what the OTel runtime instrumentation package uses:
/// https://github.com/open-telemetry/opentelemetry-dotnet-contrib/blob/5aa6d868/src/OpenTelemetry.Instrumentation.Runtime/RuntimeMetrics.cs#L105C40-L107
/// We use reflection rather than duck typing because this is a simple static method with no overloads.
/// </remarks>
public static Func<TimeSpan>? TryCreateDelegate()
{
var version = FrameworkDescription.Instance.RuntimeVersion;
if (version.Major <= 6 && version is not { Major: 6, Build: >= 21 })
{
return null;
}

var methodInfo = typeof(GC).GetMethod("GetTotalPauseDuration", BindingFlags.Public | BindingFlags.Static);
if (methodInfo is null)
{
Log.Debug("GC.GetTotalPauseDuration() is not available on this runtime version; gc pause time will not be reported.");
return null;
}

return methodInfo.CreateDelegate<Func<TimeSpan>>();
}
}

#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// <copyright file="MeterObservableUpDownCounterReflection.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>

#if NET6_0_OR_GREATER

#nullable enable

using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Reflection;

namespace Datadog.Trace.RuntimeMetrics;

/// <summary>
/// Resolves <c>Meter.CreateObservableUpDownCounter</c> overloads via reflection.
/// <c>Datadog.Trace</c> compiles against the net6.0 ref assembly (no UpDownCounter API), but on
/// .NET 7+ host processes roll-forward provides <c>System.Diagnostics.DiagnosticSource</c> 7.0+
/// where the API exists. On .NET 6 hosts callers should fall back to <c>ObservableGauge</c>.
/// </summary>
internal static class MeterObservableUpDownCounterReflection
{
// Pin to the 4-parameter overloads (string name, Func<...>, string? unit, string? description).
// .NET 7's DiagnosticSource 7.0 only ships the 4-param shape; .NET 8+ adds 5-param overloads with
// a required `tags` argument alongside the original 4-param ones. Pinning to Length == 4 picks
// an overload that exists on every supported host and avoids ambiguity on .NET 8+.
private static readonly MethodInfo? FuncOfTMethod = FindOverload(secondParam =>
secondParam.IsGenericType
&& secondParam.GetGenericTypeDefinition() == typeof(Func<>)
&& secondParam.GetGenericArguments()[0].IsGenericMethodParameter);

private static readonly MethodInfo? FuncOfMeasurementsMethod = FindOverload(secondParam =>
{
if (!secondParam.IsGenericType || secondParam.GetGenericTypeDefinition() != typeof(Func<>))
{
return false;
}

var inner = secondParam.GetGenericArguments()[0];
if (!inner.IsGenericType || inner.GetGenericTypeDefinition() != typeof(IEnumerable<>))
{
return false;
}

var measurement = inner.GetGenericArguments()[0];
return measurement.IsGenericType && measurement.GetGenericTypeDefinition() == typeof(Measurement<>);
});

public static bool TryRegister<T>(Meter meter, string name, Func<T> observeValue, string unit, string description)
where T : struct
=> Invoke(FuncOfTMethod, meter, typeof(T), name, observeValue, unit, description);

public static bool TryRegisterMulti<T>(Meter meter, string name, Func<IEnumerable<Measurement<T>>> observeValues, string unit, string description)
where T : struct
=> Invoke(FuncOfMeasurementsMethod, meter, typeof(T), name, observeValues, unit, description);

private static bool Invoke(MethodInfo? method, Meter meter, Type genericArg, string name, object observe, string unit, string description)
{
if (method is null)
{
return false;
}

method.MakeGenericMethod(genericArg).Invoke(meter, [name, observe, unit, description]);
return true;
}

private static MethodInfo? FindOverload(Func<Type, bool> matchesSecondParam) =>
typeof(Meter).GetMethods(BindingFlags.Public | BindingFlags.Instance)
.FirstOrDefault(m =>
m.Name == "CreateObservableUpDownCounter"
&& m.IsGenericMethodDefinition
&& m.GetParameters() is { Length: 4 } p
&& p[0].ParameterType == typeof(string)
&& matchesSecondParam(p[1].ParameterType));
}

#endif
Loading
Loading