From caecce0156508564849eff30245b255acc513a3f Mon Sep 17 00:00:00 2001 From: dudik <8845578+dudikeleti@users.noreply.github.com> Date: Sat, 18 Apr 2026 01:50:36 +0200 Subject: [PATCH] Add debugger global rate limiter and dispose adaptive samplers and trim rate-limit overhead --- .../Configurations/ConfigurationUpdater.cs | 40 ++- .../ProbeConfigurationComparer.cs | 3 +- .../Datadog.Trace/Debugger/DebuggerFactory.cs | 7 +- .../Debugger/DynamicInstrumentation.cs | 7 +- .../Debugger/Expressions/ProbeProcessor.cs | 39 ++- .../Debugger/RateLimiting/AdaptiveSampler.cs | 25 +- .../RateLimiting/AdaptiveSamplerLifetime.cs | 41 +++ .../RateLimiting/DebuggerGlobalRateLimiter.cs | 114 +++++++++ .../Debugger/RateLimiting/IAdaptiveSampler.cs | 5 +- .../IDebuggerGlobalRateLimiter.cs | 21 ++ .../RateLimiting/NopAdaptiveSampler.cs | 4 + .../Debugger/RateLimiting/ProbeRateLimiter.cs | 46 +++- .../Debugger/ConfigurationUpdaterTests.cs | 125 +++++++++ .../DebuggerGlobalRateLimiterTests.cs | 164 ++++++++++++ .../Debugger/DynamicInstrumentationTests.cs | 86 ++++++- .../ProbeConfigurationComparerTests.cs | 24 +- .../Debugger/ProbeProcessorTests.cs | 237 ++++++++++++++++++ .../Debugger/ProbeRateLimiterTests.cs | 71 ++++++ 18 files changed, 1013 insertions(+), 46 deletions(-) create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSamplerLifetime.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/DebuggerGlobalRateLimiter.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/IDebuggerGlobalRateLimiter.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/ConfigurationUpdaterTests.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/DebuggerGlobalRateLimiterTests.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/ProbeProcessorTests.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/ProbeRateLimiterTests.cs diff --git a/tracer/src/Datadog.Trace/Debugger/Configurations/ConfigurationUpdater.cs b/tracer/src/Datadog.Trace/Debugger/Configurations/ConfigurationUpdater.cs index bd7fcd9c5747..8314f421e08e 100644 --- a/tracer/src/Datadog.Trace/Debugger/Configurations/ConfigurationUpdater.cs +++ b/tracer/src/Datadog.Trace/Debugger/Configurations/ConfigurationUpdater.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Linq; using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.RateLimiting; using Datadog.Trace.Logging; using Datadog.Trace.RemoteConfigurationManagement; @@ -20,20 +21,35 @@ internal sealed class ConfigurationUpdater private readonly string? _env; private readonly string? _version; private readonly int _maxProbesPerType; + private readonly IDebuggerGlobalRateLimiter _globalRateLimiter; private ProbeConfiguration _currentConfiguration; - private ConfigurationUpdater(string? env, string? version, int maxProbesPerType) + private ConfigurationUpdater(string? env, string? version, int maxProbesPerType, IDebuggerGlobalRateLimiter? globalRateLimiter) { _env = env; _version = version; _maxProbesPerType = maxProbesPerType; + _globalRateLimiter = globalRateLimiter ?? DebuggerGlobalRateLimiter.Instance; _currentConfiguration = new ProbeConfiguration(); } - public static ConfigurationUpdater Create(string? environment, string? serviceVersion, int maxProbesPerType) + public static ConfigurationUpdater Create(string? environment, string? serviceVersion, int maxProbesPerType, IDebuggerGlobalRateLimiter? globalRateLimiter = null) { - return new ConfigurationUpdater(environment, serviceVersion, maxProbesPerType); + return new ConfigurationUpdater(environment, serviceVersion, maxProbesPerType, globalRateLimiter); + } + + private static bool IsProbeConfigurationPath(RemoteConfigurationPath path) + { + return path.Id.StartsWith(DefinitionPaths.LogProbe) + || path.Id.StartsWith(DefinitionPaths.MetricProbe) + || path.Id.StartsWith(DefinitionPaths.SpanProbe) + || path.Id.StartsWith(DefinitionPaths.SpanDecorationProbe); + } + + private static bool IsServiceConfigurationPath(RemoteConfigurationPath path) + { + return path.Id.StartsWith(DefinitionPaths.ServiceConfiguration); } public List AcceptAdded(ProbeConfiguration configuration) @@ -49,7 +65,7 @@ public List AcceptAdded(ProbeConfiguration configuration) if (comparer.HasRateLimitChanged) { - HandleRateLimitChanged(comparer); + HandleRateLimitChanged(filteredConfiguration); } _currentConfiguration = configuration; @@ -61,7 +77,17 @@ public void AcceptRemoved(List paths) { try { - HandleRemovedProbesChanges(paths); + if (paths.Any(IsServiceConfigurationPath)) + { + _currentConfiguration.ServiceConfiguration = null; + _globalRateLimiter.ResetRate(); + } + + var probePaths = paths.Where(IsProbeConfigurationPath).ToList(); + if (probePaths.Count > 0) + { + HandleRemovedProbesChanges(probePaths); + } } catch (Exception ex) { @@ -127,9 +153,9 @@ private void HandleRemovedProbesChanges(List paths) DebuggerManager.Instance.DynamicInstrumentation?.UpdateRemovedProbeInstrumentations(paths); } - private void HandleRateLimitChanged(ProbeConfigurationComparer comparer) + private void HandleRateLimitChanged(ProbeConfiguration configuration) { - // todo handle rate limited changes + _globalRateLimiter.SetRate(configuration.ServiceConfiguration?.Sampling?.SnapshotsPerSecond); } internal sealed record UpdateResult(string Id, string? Error); diff --git a/tracer/src/Datadog.Trace/Debugger/Configurations/ProbeConfigurationComparer.cs b/tracer/src/Datadog.Trace/Debugger/Configurations/ProbeConfigurationComparer.cs index dfcfd63cc5e2..bcc6ac519161 100644 --- a/tracer/src/Datadog.Trace/Debugger/Configurations/ProbeConfigurationComparer.cs +++ b/tracer/src/Datadog.Trace/Debugger/Configurations/ProbeConfigurationComparer.cs @@ -33,8 +33,7 @@ public ProbeConfigurationComparer(ProbeConfiguration currentConfiguration, Probe HasProbeRelatedChanges = AddedDefinitions.Any() || isFilteredListChanged; HasRateLimitChanged = - (!currentConfiguration.ServiceConfiguration?.Sampling?.Equals(incomingConfiguration.ServiceConfiguration?.Sampling) ?? incomingConfiguration.ServiceConfiguration?.Sampling != null) - || HasProbeRelatedChanges; + (!currentConfiguration.ServiceConfiguration?.Sampling?.Equals(incomingConfiguration.ServiceConfiguration?.Sampling) ?? incomingConfiguration.ServiceConfiguration?.Sampling != null); } public IReadOnlyList AddedDefinitions { get; } diff --git a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs index 114d523c1e0a..0092ecb54689 100644 --- a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs +++ b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs @@ -10,6 +10,7 @@ using Datadog.Trace.Configuration; using Datadog.Trace.Debugger.Configurations; using Datadog.Trace.Debugger.ProbeStatuses; +using Datadog.Trace.Debugger.RateLimiting; using Datadog.Trace.Debugger.Sink; using Datadog.Trace.Debugger.Snapshots; using Datadog.Trace.Debugger.Symbols; @@ -29,6 +30,7 @@ internal sealed class DebuggerFactory internal static DynamicInstrumentation CreateDynamicInstrumentation(IDiscoveryService discoveryService, IRcmSubscriptionManager remoteConfigurationManager, TracerSettings tracerSettings, Func serviceNameProvider, DebuggerSettings debuggerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider) { + var globalRateLimiter = DebuggerGlobalRateLimiter.Instance; var snapshotSlicer = SnapshotSlicer.Create(debuggerSettings); var snapshotSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer); var logSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer); @@ -39,7 +41,7 @@ internal static DynamicInstrumentation CreateDynamicInstrumentation(IDiscoverySe var diagnosticsUploader = CreateDiagnosticsUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, true), diagnosticsSink); var lineProbeResolver = LineProbeResolver.Create(debuggerSettings.ThirdPartyDetectionExcludes, debuggerSettings.ThirdPartyDetectionIncludes); var probeStatusPoller = ProbeStatusPoller.Create(diagnosticsSink, debuggerSettings); - var configurationUpdater = ConfigurationUpdater.Create(tracerSettings.Manager.InitialMutableSettings.Environment, tracerSettings.Manager.InitialMutableSettings.ServiceVersion, debuggerSettings.MaxProbesPerType); + var configurationUpdater = ConfigurationUpdater.Create(tracerSettings.Manager.InitialMutableSettings.Environment, tracerSettings.Manager.InitialMutableSettings.ServiceVersion, debuggerSettings.MaxProbesPerType, globalRateLimiter); var statsd = GetDogStatsd(tracerSettings); @@ -53,7 +55,8 @@ internal static DynamicInstrumentation CreateDynamicInstrumentation(IDiscoverySe diagnosticsUploader: diagnosticsUploader, probeStatusPoller: probeStatusPoller, configurationUpdater: configurationUpdater, - dogStats: statsd); + dogStats: statsd, + globalRateLimiter: globalRateLimiter); } private static IDogStatsd GetDogStatsd(TracerSettings tracerSettings) diff --git a/tracer/src/Datadog.Trace/Debugger/DynamicInstrumentation.cs b/tracer/src/Datadog.Trace/Debugger/DynamicInstrumentation.cs index fb478e0663e6..70d1a3a04416 100644 --- a/tracer/src/Datadog.Trace/Debugger/DynamicInstrumentation.cs +++ b/tracer/src/Datadog.Trace/Debugger/DynamicInstrumentation.cs @@ -47,6 +47,7 @@ internal sealed class DynamicInstrumentation : IDisposable private readonly ConfigurationUpdater _configurationUpdater; private readonly IDogStatsd _dogStats; private readonly DebuggerSettings _settings; + private readonly IDebuggerGlobalRateLimiter _globalRateLimiter; private readonly object _instanceLock = new(); private int _disposeState; @@ -60,7 +61,8 @@ internal DynamicInstrumentation( IDebuggerUploader diagnosticsUploader, IProbeStatusPoller probeStatusPoller, ConfigurationUpdater configurationUpdater, - IDogStatsd dogStats) + IDogStatsd dogStats, + IDebuggerGlobalRateLimiter? globalRateLimiter = null) { Log.Information("Initializing Dynamic Instrumentation"); _settings = settings; @@ -74,7 +76,9 @@ internal DynamicInstrumentation( _subscriptionManager = remoteConfigurationManager; _configurationUpdater = configurationUpdater; _dogStats = dogStats; + _globalRateLimiter = globalRateLimiter ?? DebuggerGlobalRateLimiter.Instance; _unboundProbes = new List(); + _globalRateLimiter.ResetRate(); _subscription = new Subscription( (updates, removals) => { @@ -744,6 +748,7 @@ public void Dispose() // On master, _dogStats was disposed via SafeDisposal.Add() which called sync // Dispose() — itself fire-and-forget internally via Task.Run(). _dogStats?.DisposeAsync().ContinueWith(t => Log.Error(t.Exception, "Error waiting for StatsD disposal"), TaskContinuationOptions.OnlyOnFaulted); + _globalRateLimiter.Dispose(); } } } diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs index 8e9264cd9da6..24567a374260 100644 --- a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs @@ -28,6 +28,10 @@ internal sealed class ProbeProcessor : IProbeProcessor private const string DynamicPrefix = "_dd.di."; private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ProbeProcessor)); + private readonly IDebuggerGlobalRateLimiter _globalRateLimiter; + private string _probeId = string.Empty; + private ProbeType _probeType; + private bool _shouldApplyGlobalRateLimit; private ProbeExpressionEvaluator? _evaluator; private DebuggerExpression?[]? _templates; private DebuggerExpression? _condition; @@ -41,7 +45,13 @@ internal sealed class ProbeProcessor : IProbeProcessor /// If probe type or probe location is from unsupported type /// Exceptions should be caught and logged by the caller internal ProbeProcessor(ProbeDefinition probe) + : this(probe, DebuggerGlobalRateLimiter.Instance) { + } + + internal ProbeProcessor(ProbeDefinition probe, IDebuggerGlobalRateLimiter globalRateLimiter) + { + _globalRateLimiter = globalRateLimiter ?? throw new ArgumentNullException(nameof(globalRateLimiter)); InitializeProbeProcessor(probe); } @@ -73,6 +83,9 @@ private void InitializeProbeProcessor(ProbeDefinition probe) }; SetExpressions(probe); + _probeId = probe.Id; + _probeType = probeType; + _shouldApplyGlobalRateLimit = probeType is ProbeType.Snapshot or ProbeType.Log; var capture = (probe as LogProbe)?.Capture; var maxInfo = capture != null @@ -164,9 +177,15 @@ private ProbeExpressionEvaluator GetOrCreateEvaluator() return _evaluator; } + private bool SamplePayload(IAdaptiveSampler sampler) + { + return (!_shouldApplyGlobalRateLimit || _globalRateLimiter.ShouldSample(_probeType, _probeId)) + && sampler.Sample(); + } + public bool ShouldProcess(in ProbeData probeData) { - return HasCondition() || probeData.Sampler.Sample(); + return HasCondition() || SamplePayload(probeData.Sampler); } public bool Process(ref CaptureInfo info, IDebuggerSnapshotCreator inSnapshotCreator, in ProbeData probeData) @@ -382,14 +401,28 @@ private ExpressionEvaluationResult Evaluate(DebuggerSnapshotCreator snapshotCrea } if (evaluationResult.Condition != null && // i.e. not a metric, span probe, or span decoration - (evaluationResult.Condition is false || - !sampler.Sample())) + evaluationResult.Condition is false) { // if the expression evaluated to false, or there is a rate limit, stop capture shouldStopCapture = true; return evaluationResult; } + if (evaluationResult.Condition != null && + _shouldApplyGlobalRateLimit && + !_globalRateLimiter.ShouldSample(_probeType, _probeId)) + { + shouldStopCapture = true; + return evaluationResult; + } + + if (evaluationResult.Condition != null && + !sampler.Sample()) + { + shouldStopCapture = true; + return evaluationResult; + } + return evaluationResult; } diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs index 8a2f6849d50b..f60d42399ee0 100644 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +#nullable enable + using System; using System.Threading; using Datadog.Trace.Logging; @@ -64,15 +66,16 @@ internal sealed class AdaptiveSampler : IAdaptiveSampler private int _countsSlotIndex; private Counts[] _countsSlots; - private Timer _timer; - private Action _rollWindowCallback; + private Timer? _timer; + private Action? _rollWindowCallback; + private int _disposeState; internal AdaptiveSampler( TimeSpan windowDuration, int samplesPerWindow, int averageLookback, int budgetLookback, - Action rollWindowCallback) + Action? rollWindowCallback) { _timer = new Timer(state => RollWindow(), state: null, windowDuration, windowDuration); _totalCountRunningAverage = 0; @@ -134,6 +137,17 @@ public double NextDouble() return ThreadSafeRandom.Shared.NextDouble(); } + public void Dispose() + { + if (Interlocked.CompareExchange(ref _disposeState, 1, 0) != 0) + { + return; + } + + Interlocked.Exchange(ref _timer, null)?.Dispose(); + _rollWindowCallback = null; + } + private double ComputeIntervalAlpha(int lookback) { return 1 - Math.Pow(lookback, -1.0 / lookback); @@ -191,10 +205,7 @@ internal void RollWindow() counts.Reset(); - if (_rollWindowCallback != null) - { - _rollWindowCallback(); - } + _rollWindowCallback?.Invoke(); } catch (Exception e) { diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSamplerLifetime.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSamplerLifetime.cs new file mode 100644 index 000000000000..3ef647baa2eb --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSamplerLifetime.cs @@ -0,0 +1,41 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Threading; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal static class AdaptiveSamplerLifetime + { + private const int AverageLookback = 180; + private const int BudgetLookback = 16; + + private static readonly TimeSpan WindowDuration = TimeSpan.FromSeconds(1); + + public static IAdaptiveSampler Create(int samplesPerSecond) + { + return new AdaptiveSampler(WindowDuration, samplesPerSecond, AverageLookback, BudgetLookback, rollWindowCallback: null); + } + + public static void Replace(ref IAdaptiveSampler sampler, IAdaptiveSampler replacement) + { + Dispose(Interlocked.Exchange(ref sampler, replacement)); + } + + public static void Dispose(ref IAdaptiveSampler sampler) + { + Dispose(Interlocked.Exchange(ref sampler, NopAdaptiveSampler.Instance)); + } + + public static void Dispose(IAdaptiveSampler sampler) + { + if (!ReferenceEquals(sampler, NopAdaptiveSampler.Instance)) + { + sampler.Dispose(); + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/DebuggerGlobalRateLimiter.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/DebuggerGlobalRateLimiter.cs new file mode 100644 index 000000000000..b5b0f5a1c823 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/DebuggerGlobalRateLimiter.cs @@ -0,0 +1,114 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.CompilerServices; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal sealed class DebuggerGlobalRateLimiter : IDebuggerGlobalRateLimiter + { + internal const int DefaultSnapshotSamplesPerSecond = 100; + internal const int DefaultLogSamplesPerSecond = 5000; + + private const int LogCooldownSeconds = 60; + + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(DebuggerGlobalRateLimiter)); + + private readonly Func _samplerFactory; + private readonly ILogRateLimiter _logRateLimiter; + + private IAdaptiveSampler _snapshotSampler; + private IAdaptiveSampler _logSampler; + + internal DebuggerGlobalRateLimiter() + : this(AdaptiveSamplerLifetime.Create, new LogRateLimiter(LogCooldownSeconds)) + { + } + + internal DebuggerGlobalRateLimiter(Func samplerFactory, ILogRateLimiter logRateLimiter) + { + _samplerFactory = samplerFactory ?? throw new ArgumentNullException(nameof(samplerFactory)); + _logRateLimiter = logRateLimiter ?? throw new ArgumentNullException(nameof(logRateLimiter)); + _snapshotSampler = NopAdaptiveSampler.Instance; + _logSampler = NopAdaptiveSampler.Instance; + ResetRate(); + } + + internal static DebuggerGlobalRateLimiter Instance { get; } = new(); + + public bool ShouldSample(ProbeType probeType, string probeId) + { + var sampler = probeType switch + { + ProbeType.Snapshot => _snapshotSampler, + ProbeType.Log => _logSampler, + _ => null + }; + + if (sampler == null || sampler.Sample()) + { + return true; + } + + LogDrop(probeType, probeId); + return false; + } + + public void SetRate(double? samplesPerSecond) + { + if (!samplesPerSecond.HasValue) + { + ResetRate(); + return; + } + + var configuredRate = Math.Max((int)samplesPerSecond.Value, 0); + ReplaceSamplers(configuredRate, configuredRate); + } + + public void ResetRate() + { + ReplaceSamplers(DefaultSnapshotSamplesPerSecond, DefaultLogSamplesPerSecond); + } + + public void Dispose() + { + AdaptiveSamplerLifetime.Dispose(ref _snapshotSampler); + AdaptiveSamplerLifetime.Dispose(ref _logSampler); + } + + private void ReplaceSamplers(int snapshotSamplesPerSecond, int logSamplesPerSecond) + { + AdaptiveSamplerLifetime.Replace(ref _snapshotSampler, _samplerFactory(snapshotSamplesPerSecond)); + AdaptiveSamplerLifetime.Replace(ref _logSampler, _samplerFactory(logSamplesPerSecond)); + } + + private void LogDrop(ProbeType probeType, string probeId, [CallerFilePath] string sourceFile = "", [CallerLineNumber] int sourceLine = 0) + { + if (!_logRateLimiter.ShouldLog(sourceFile, sourceLine, out var skipCount)) + { + return; + } + + var probeTypeName = probeType == ProbeType.Snapshot ? "snapshot" : "log"; + const string message = "Global debugger rate limit reached for {ProbeType} probes. Dropping capture for ProbeId={ProbeId}"; + const string messageWithSkipCount = "Global debugger rate limit reached for {ProbeType} probes. Dropping capture for ProbeId={ProbeId}, {SkipCount} additional messages skipped"; + + if (skipCount > 0) + { + Log.Warning(messageWithSkipCount, probeTypeName, probeId, skipCount); + } + else + { + Log.Warning(message, probeTypeName, probeId); + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs index b6d72153d35f..ae10fd64b405 100644 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs @@ -4,9 +4,12 @@ // #nullable enable + +using System; + namespace Datadog.Trace.Debugger.RateLimiting { - internal interface IAdaptiveSampler + internal interface IAdaptiveSampler : IDisposable { bool Sample(); diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/IDebuggerGlobalRateLimiter.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IDebuggerGlobalRateLimiter.cs new file mode 100644 index 000000000000..58f47f90445d --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IDebuggerGlobalRateLimiter.cs @@ -0,0 +1,21 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using Datadog.Trace.Debugger.Expressions; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal interface IDebuggerGlobalRateLimiter : IDisposable + { + bool ShouldSample(ProbeType probeType, string probeId); + + void SetRate(double? samplesPerSecond); + + void ResetRate(); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs index 86d1325c991f..3fe94618f337 100644 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs @@ -29,5 +29,9 @@ public double NextDouble() { return 1.0; } + + public void Dispose() + { + } } } diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs index c500b42ddc2f..753314280672 100644 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs @@ -22,8 +22,19 @@ internal sealed class ProbeRateLimiter private static ProbeRateLimiter _instance; + private readonly Func _samplerFactory; private readonly ConcurrentDictionary _samplers = new(); + internal ProbeRateLimiter() + : this(AdaptiveSamplerLifetime.Create) + { + } + + internal ProbeRateLimiter(Func samplerFactory) + { + _samplerFactory = samplerFactory ?? throw new ArgumentNullException(nameof(samplerFactory)); + } + internal static ProbeRateLimiter Instance { get @@ -35,17 +46,34 @@ internal static ProbeRateLimiter Instance } } - private static AdaptiveSampler CreateSampler(int samplesPerSecond = DefaultSamplesPerSecond) => - new(TimeSpan.FromSeconds(1), samplesPerSecond, 180, 16, null); - public IAdaptiveSampler GerOrAddSampler(string probeId) { - return _samplers.GetOrAdd(probeId, _ => CreateSampler(1)); + while (true) + { + if (_samplers.TryGetValue(probeId, out var sampler)) + { + return sampler; + } + + var createdSampler = _samplerFactory(DefaultSamplesPerSecond); + if (_samplers.TryAdd(probeId, createdSampler)) + { + return createdSampler; + } + + AdaptiveSamplerLifetime.Dispose(createdSampler); + } } public bool TryAddSampler(string probeId, IAdaptiveSampler sampler) { - return _samplers.TryAdd(probeId, sampler); + if (_samplers.TryAdd(probeId, sampler)) + { + return true; + } + + AdaptiveSamplerLifetime.Dispose(sampler); + return false; } public void SetRate(string probeId, int samplesPerSecond) @@ -58,16 +86,20 @@ public void SetRate(string probeId, int samplesPerSecond) return; } - var adaptiveSampler = CreateSampler(samplesPerSecond); + var adaptiveSampler = _samplerFactory(samplesPerSecond); if (!_samplers.TryAdd(probeId, adaptiveSampler)) { + AdaptiveSamplerLifetime.Dispose(adaptiveSampler); Log.Information("Adaptive sampler already exist for {ProbeID}", probeId); } } public void ResetRate(string probeId) { - _samplers.TryRemove(probeId, out _); + if (_samplers.TryRemove(probeId, out var sampler)) + { + AdaptiveSamplerLifetime.Dispose(sampler); + } } } } diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/ConfigurationUpdaterTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/ConfigurationUpdaterTests.cs new file mode 100644 index 000000000000..ed793727f543 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/ConfigurationUpdaterTests.cs @@ -0,0 +1,125 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.Debugger.Configurations; +using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.RateLimiting; +using Datadog.Trace.RemoteConfigurationManagement; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger; + +public class ConfigurationUpdaterTests +{ + [Fact] + public void AcceptAdded_ServiceConfigurationOnly_UpdatesGlobalRateLimiter() + { + var globalRateLimiter = new GlobalRateLimiterMock(); + var updater = ConfigurationUpdater.Create("env", "version", 0, globalRateLimiter); + + updater.AcceptAdded( + new ProbeConfiguration + { + ServiceConfiguration = new ServiceConfiguration + { + Sampling = new Datadog.Trace.Debugger.Configurations.Models.Sampling { SnapshotsPerSecond = 42 } + } + }); + + Assert.Equal(1, globalRateLimiter.SetRateCallCount); + Assert.Equal(42, globalRateLimiter.LastRate); + } + + [Fact] + public void AcceptAdded_ProbeOnlyChange_DoesNotResetGlobalRateLimiter() + { + var globalRateLimiter = new GlobalRateLimiterMock(); + var updater = ConfigurationUpdater.Create("env", "version", 0, globalRateLimiter); + updater.AcceptAdded( + new ProbeConfiguration + { + ServiceConfiguration = new ServiceConfiguration + { + Sampling = new Datadog.Trace.Debugger.Configurations.Models.Sampling { SnapshotsPerSecond = 42 } + } + }); + globalRateLimiter.ResetCounters(); + + updater.AcceptAdded( + new ProbeConfiguration + { + ServiceConfiguration = new ServiceConfiguration + { + Sampling = new Datadog.Trace.Debugger.Configurations.Models.Sampling { SnapshotsPerSecond = 42 } + }, + LogProbes = + [ + new LogProbe + { + Id = "log-probe", + Where = new Where { MethodName = "TestMethod" }, + Tags = [], + } + ] + }); + + Assert.Equal(0, globalRateLimiter.SetRateCallCount); + Assert.Equal(0, globalRateLimiter.ResetRateCallCount); + } + + [Fact] + public void AcceptRemoved_ServiceConfiguration_ResetsGlobalRateLimiter() + { + var globalRateLimiter = new GlobalRateLimiterMock(); + var updater = ConfigurationUpdater.Create("env", "version", 0, globalRateLimiter); + updater.AcceptAdded( + new ProbeConfiguration + { + ServiceConfiguration = new ServiceConfiguration + { + Sampling = new Datadog.Trace.Debugger.Configurations.Models.Sampling { SnapshotsPerSecond = 42 } + } + }); + globalRateLimiter.ResetCounters(); + + updater.AcceptRemoved([RemoteConfigurationPath.FromPath("datadog/123/LIVE_DEBUGGING/serviceConfig_/config")]); + + Assert.Equal(0, globalRateLimiter.SetRateCallCount); + Assert.Equal(1, globalRateLimiter.ResetRateCallCount); + } + + private sealed class GlobalRateLimiterMock : IDebuggerGlobalRateLimiter + { + public double? LastRate { get; private set; } + + public int SetRateCallCount { get; private set; } + + public int ResetRateCallCount { get; private set; } + + public bool ShouldSample(ProbeType probeType, string probeId) => true; + + public void SetRate(double? samplesPerSecond) + { + SetRateCallCount++; + LastRate = samplesPerSecond; + } + + public void ResetRate() + { + ResetRateCallCount++; + } + + public void ResetCounters() + { + SetRateCallCount = 0; + ResetRateCallCount = 0; + } + + public void Dispose() + { + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/DebuggerGlobalRateLimiterTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/DebuggerGlobalRateLimiterTests.cs new file mode 100644 index 000000000000..e1a2657722ea --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/DebuggerGlobalRateLimiterTests.cs @@ -0,0 +1,164 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.RateLimiting; +using Datadog.Trace.Logging; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger; + +public class DebuggerGlobalRateLimiterTests +{ + [Fact] + public void Constructor_UsesFallbackRates() + { + var factory = new RecordingSamplerFactory(); + + _ = new DebuggerGlobalRateLimiter(factory.Create, new NullLogRateLimiter()); + + Assert.Equal( + [DebuggerGlobalRateLimiter.DefaultSnapshotSamplesPerSecond, DebuggerGlobalRateLimiter.DefaultLogSamplesPerSecond], + factory.RequestedRates); + } + + [Fact] + public void SetRate_UsesConfiguredRateForSnapshotAndLogSamplers() + { + var factory = new RecordingSamplerFactory(); + var limiter = new DebuggerGlobalRateLimiter(factory.Create, new NullLogRateLimiter()); + + limiter.SetRate(42); + + Assert.Equal( + [DebuggerGlobalRateLimiter.DefaultSnapshotSamplesPerSecond, DebuggerGlobalRateLimiter.DefaultLogSamplesPerSecond, 42, 42], + factory.RequestedRates); + } + + [Fact] + public void SetRate_DisposesPreviousSamplers() + { + var factory = new RecordingSamplerFactory(); + var limiter = new DebuggerGlobalRateLimiter(factory.Create, new NullLogRateLimiter()); + var initialSnapshotSampler = factory.Samplers[0]; + var initialLogSampler = factory.Samplers[1]; + + limiter.SetRate(42); + + Assert.Equal(1, initialSnapshotSampler.DisposeCallCount); + Assert.Equal(1, initialLogSampler.DisposeCallCount); + } + + [Fact] + public void ResetRate_RestoresFallbackRates() + { + var factory = new RecordingSamplerFactory(); + var limiter = new DebuggerGlobalRateLimiter(factory.Create, new NullLogRateLimiter()); + limiter.SetRate(42); + + limiter.ResetRate(); + + Assert.Equal( + [DebuggerGlobalRateLimiter.DefaultSnapshotSamplesPerSecond, DebuggerGlobalRateLimiter.DefaultLogSamplesPerSecond, 42, 42, DebuggerGlobalRateLimiter.DefaultSnapshotSamplesPerSecond, DebuggerGlobalRateLimiter.DefaultLogSamplesPerSecond], + factory.RequestedRates); + } + + [Fact] + public void Dispose_DisposesCurrentSamplers() + { + var factory = new RecordingSamplerFactory(); + var limiter = new DebuggerGlobalRateLimiter(factory.Create, new NullLogRateLimiter()); + limiter.SetRate(42); + var currentSnapshotSampler = factory.Samplers[2]; + var currentLogSampler = factory.Samplers[3]; + + limiter.Dispose(); + + Assert.Equal(1, currentSnapshotSampler.DisposeCallCount); + Assert.Equal(1, currentLogSampler.DisposeCallCount); + } + + [Fact] + public void SnapshotProbesShareOneGlobalBudget() + { + var factory = new RecordingSamplerFactory(); + var limiter = new DebuggerGlobalRateLimiter(factory.Create, new NullLogRateLimiter()); + factory.Samplers[0].SetResults(true, false); + + var firstResult = limiter.ShouldSample(ProbeType.Snapshot, "snapshot-1"); + var secondResult = limiter.ShouldSample(ProbeType.Snapshot, "snapshot-2"); + + Assert.True(firstResult); + Assert.False(secondResult); + Assert.Equal(2, factory.Samplers[0].SampleCallCount); + Assert.Equal(0, factory.Samplers[1].SampleCallCount); + } + + [Fact] + public void NonPayloadProbesAreUnaffected() + { + var factory = new RecordingSamplerFactory(); + var limiter = new DebuggerGlobalRateLimiter(factory.Create, new NullLogRateLimiter()); + + var metricResult = limiter.ShouldSample(ProbeType.Metric, "metric"); + var spanDecorationResult = limiter.ShouldSample(ProbeType.SpanDecoration, "span"); + + Assert.True(metricResult); + Assert.True(spanDecorationResult); + Assert.Equal(0, factory.Samplers[0].SampleCallCount); + Assert.Equal(0, factory.Samplers[1].SampleCallCount); + } + + private sealed class RecordingSamplerFactory + { + public List RequestedRates { get; } = []; + + public List Samplers { get; } = []; + + public IAdaptiveSampler Create(int samplesPerSecond) + { + RequestedRates.Add(samplesPerSecond); + var sampler = new TestAdaptiveSampler(); + Samplers.Add(sampler); + return sampler; + } + } + + private sealed class TestAdaptiveSampler : IAdaptiveSampler + { + private readonly Queue _results = new(); + + public int DisposeCallCount { get; private set; } + + public int SampleCallCount { get; private set; } + + public void SetResults(params bool[] results) + { + _results.Clear(); + foreach (var result in results) + { + _results.Enqueue(result); + } + } + + public bool Sample() + { + SampleCallCount++; + return _results.Count == 0 || _results.Dequeue(); + } + + public bool Keep() => true; + + public bool Drop() => false; + + public double NextDouble() => 0; + + public void Dispose() + { + DisposeCallCount++; + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs index 0bd0dc195216..b8647bd1b78e 100644 --- a/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs @@ -13,8 +13,10 @@ using Datadog.Trace.Debugger; using Datadog.Trace.Debugger.Configurations; using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Expressions; using Datadog.Trace.Debugger.Models; using Datadog.Trace.Debugger.ProbeStatuses; +using Datadog.Trace.Debugger.RateLimiting; using Datadog.Trace.Debugger.Sink; using Datadog.Trace.DogStatsd; using Datadog.Trace.RemoteConfigurationManagement; @@ -28,6 +30,57 @@ namespace Datadog.Trace.Tests.Debugger; public class DynamicInstrumentationTests { + [Fact] + public void DynamicInstrumentation_ResetsGlobalRateLimiterOnConstruction() + { + var settings = DebuggerSettings.FromSource( + new NameValueConfigurationSource(new() { { ConfigurationKeys.Debugger.DynamicInstrumentationEnabled, "0" }, }), + NullConfigurationTelemetry.Instance); + + var globalRateLimiter = new GlobalRateLimiterMock(); + + _ = new DynamicInstrumentation( + settings, + new DiscoveryServiceMock(), + new RcmSubscriptionManagerMock(), + new LineProbeResolverMock(), + new SnapshotUploaderMock(), + new LogUploaderMock(), + new UploaderMock(), + new ProbeStatusPollerMock(), + ConfigurationUpdater.Create(string.Empty, string.Empty, 0, globalRateLimiter), + NoOpStatsd.Instance, + globalRateLimiter); + + globalRateLimiter.ResetRateCallCount.Should().Be(1); + } + + [Fact] + public void DynamicInstrumentation_DisposesGlobalRateLimiterOnDispose() + { + var settings = DebuggerSettings.FromSource( + new NameValueConfigurationSource(new() { { ConfigurationKeys.Debugger.DynamicInstrumentationEnabled, "0" }, }), + NullConfigurationTelemetry.Instance); + + var globalRateLimiter = new GlobalRateLimiterMock(); + var debugger = new DynamicInstrumentation( + settings, + new DiscoveryServiceMock(), + new RcmSubscriptionManagerMock(), + new LineProbeResolverMock(), + new SnapshotUploaderMock(), + new LogUploaderMock(), + new UploaderMock(), + new ProbeStatusPollerMock(), + ConfigurationUpdater.Create(string.Empty, string.Empty, 0, globalRateLimiter), + NoOpStatsd.Instance, + globalRateLimiter); + + debugger.Dispose(); + + globalRateLimiter.DisposeCallCount.Should().Be(1); + } + [Fact] public async Task DynamicInstrumentationEnabled_ServicesCalled() { @@ -42,9 +95,10 @@ public async Task DynamicInstrumentationEnabled_ServicesCalled() var logUploader = new LogUploaderMock(); var diagnosticsUploader = new UploaderMock(); var probeStatusPoller = new ProbeStatusPollerMock(); - var updater = ConfigurationUpdater.Create("env", "version", 0); + var globalRateLimiter = new GlobalRateLimiterMock(); + var updater = ConfigurationUpdater.Create("env", "version", 0, globalRateLimiter); - var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance); + var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance, globalRateLimiter); debugger.Initialize(); // Wait for async initialization to complete @@ -79,9 +133,10 @@ public void DynamicInstrumentationDisabled_ServicesNotCalled() var logUploader = new LogUploaderMock(); var diagnosticsUploader = new UploaderMock(); var probeStatusPoller = new ProbeStatusPollerMock(); - var updater = ConfigurationUpdater.Create(string.Empty, string.Empty, 0); + var globalRateLimiter = new GlobalRateLimiterMock(); + var updater = ConfigurationUpdater.Create(string.Empty, string.Empty, 0, globalRateLimiter); - var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance); + var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance, globalRateLimiter); debugger.Initialize(); lineProbeResolver.Called.Should().BeFalse(); probeStatusPoller.Called.Should().BeFalse(); @@ -257,4 +312,27 @@ public void Dispose() { } } + + private class GlobalRateLimiterMock : IDebuggerGlobalRateLimiter + { + internal int DisposeCallCount { get; private set; } + + internal int ResetRateCallCount { get; private set; } + + public bool ShouldSample(ProbeType probeType, string probeId) => true; + + public void SetRate(double? samplesPerSecond) + { + } + + public void ResetRate() + { + ResetRateCallCount++; + } + + public void Dispose() + { + DisposeCallCount++; + } + } } diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/ProbeConfigurationComparerTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/ProbeConfigurationComparerTests.cs index fcdaaa305a4b..babf8938d687 100644 --- a/tracer/test/Datadog.Trace.Tests/Debugger/ProbeConfigurationComparerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Debugger/ProbeConfigurationComparerTests.cs @@ -76,7 +76,7 @@ public void CurrentFilterNull_IncomingFilterEmpty_FilterRelatedChanged() var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -98,7 +98,7 @@ public void CurrentFilterNotEmpty_IncomingFilterEmpty_FilterRelatedChanged() var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -120,7 +120,7 @@ public void CurrentFilterValue_IncomingFilterDifferentValue_FilterRelatedNotChan var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -131,7 +131,7 @@ public void CurrentSnaphotsEmpty_IncomingSnapshotsWithDefault_ProbeRelatedChange var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -153,7 +153,7 @@ public void CurrentSnaphotsValue_IncomingSnapshotsBaseValue_ProbeRelatedChanged( var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -164,7 +164,7 @@ public void CurrentSnaphotsValue_IncomingSnapshotsTypeValue_ProbeRelatedChanged( var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -186,7 +186,7 @@ public void CurrentSnaphotsWhereValue_IncomingSnapshotsWhereAnotherValue_ProbeRe var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -197,7 +197,7 @@ public void CurrentMetricsEmpty_IncomingMetricsWithDefault_ProbeRelatedChanged() var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -219,7 +219,7 @@ public void CurrentMetricsValue_IncomingMetricsBaseValue_ProbeRelatedChanged() var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -230,7 +230,7 @@ public void CurrentMetricsValue_IncomingMetricsTypeValue_ProbeRelatedChanged() var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -252,7 +252,7 @@ public void CurrentMetricsWhereValue_IncomingMetricsWhereAnotherValue_ProbeRelat var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] @@ -263,7 +263,7 @@ public void CurrentMetricsValue_IncomingMetricsAnotherValue_ProbeRelatedChanged( var comparer = new ProbeConfigurationComparer(current, incoming); comparer.HasProbeRelatedChanges.Should().BeTrue(); - comparer.HasRateLimitChanged.Should().BeTrue(); + comparer.HasRateLimitChanged.Should().BeFalse(); } [Fact] diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/ProbeProcessorTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/ProbeProcessorTests.cs new file mode 100644 index 000000000000..5003395d9586 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/ProbeProcessorTests.cs @@ -0,0 +1,237 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using System.Reflection; +using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.Debugger.Instrumentation.Collections; +using Datadog.Trace.Debugger.RateLimiting; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger; + +public class ProbeProcessorTests +{ + private const string FalseConditionJson = @"{ ""eq"": [1, 0] }"; + private const string TrueConditionJson = @"{ ""eq"": [1, 1] }"; + + [Fact] + public void ShouldProcess_UnconditionalLogProbe_SamplesGlobalBeforePerProbe() + { + var globalRateLimiter = new GlobalRateLimiterMock(false); + var perProbeSampler = new AdaptiveSamplerMock(true); + var probe = CreateLogProbe("log-probe", captureSnapshot: false); + var processor = new ProbeProcessor(probe, globalRateLimiter); + var probeData = new ProbeData(probe.Id, perProbeSampler, processor); + + var shouldProcess = processor.ShouldProcess(in probeData); + + Assert.False(shouldProcess); + Assert.Equal(1, globalRateLimiter.ShouldSampleCallCount); + Assert.Equal(ProbeType.Log, globalRateLimiter.LastProbeType); + Assert.Equal("log-probe", globalRateLimiter.LastProbeId); + Assert.Equal(0, perProbeSampler.SampleCallCount); + } + + [Fact] + public void Process_ConditionalProbe_EvaluatesConditionBeforeRateLimiting() + { + var globalRateLimiter = new GlobalRateLimiterMock(false); + var perProbeSampler = new AdaptiveSamplerMock(true); + var probe = CreateConditionalLogProbe("conditional-false", FalseConditionJson, captureSnapshot: true); + var processor = new ProbeProcessor(probe, globalRateLimiter); + var probeData = new ProbeData(probe.Id, perProbeSampler, processor); + var snapshotCreator = processor.CreateSnapshotCreator(); + var captureInfo = CreateAsyncEvaluateCaptureInfo(); + + var result = processor.Process(ref captureInfo, snapshotCreator, in probeData); + + Assert.False(result); + Assert.Equal(0, globalRateLimiter.ShouldSampleCallCount); + Assert.Equal(0, perProbeSampler.SampleCallCount); + } + + [Fact] + public void Process_ConditionalProbe_SamplesGlobalBeforePerProbe() + { + var globalRateLimiter = new GlobalRateLimiterMock(false); + var perProbeSampler = new AdaptiveSamplerMock(true); + var probe = CreateConditionalLogProbe("conditional-true", TrueConditionJson, captureSnapshot: true); + var processor = new ProbeProcessor(probe, globalRateLimiter); + var probeData = new ProbeData(probe.Id, perProbeSampler, processor); + var snapshotCreator = processor.CreateSnapshotCreator(); + var captureInfo = CreateAsyncEvaluateCaptureInfo(); + + var result = processor.Process(ref captureInfo, snapshotCreator, in probeData); + + Assert.False(result); + Assert.Equal(1, globalRateLimiter.ShouldSampleCallCount); + Assert.Equal(ProbeType.Snapshot, globalRateLimiter.LastProbeType); + Assert.Equal("conditional-true", globalRateLimiter.LastProbeId); + Assert.Equal(0, perProbeSampler.SampleCallCount); + } + + [Fact] + public void ShouldProcess_MetricProbe_DoesNotUseGlobalLimiter() + { + var globalRateLimiter = new GlobalRateLimiterMock(false); + var perProbeSampler = new AdaptiveSamplerMock(true); + var probe = new MetricProbe + { + Id = "metric-probe", + MetricName = "metric", + Kind = MetricKind.COUNT, + Where = new Where { MethodName = nameof(TestMethod) }, + Tags = [], + }; + var processor = new ProbeProcessor(probe, globalRateLimiter); + var probeData = new ProbeData(probe.Id, perProbeSampler, processor); + + var shouldProcess = processor.ShouldProcess(in probeData); + + Assert.True(shouldProcess); + Assert.Equal(0, globalRateLimiter.ShouldSampleCallCount); + Assert.Equal(1, perProbeSampler.SampleCallCount); + } + + [Fact] + public void ShouldProcess_SpanDecorationProbe_DoesNotUseGlobalLimiter() + { + var globalRateLimiter = new GlobalRateLimiterMock(false); + var perProbeSampler = new AdaptiveSamplerMock(true); + var probe = new SpanDecorationProbe + { + Id = "span-probe", + Decorations = [], + TargetSpan = TargetSpan.Active, + Where = new Where { MethodName = nameof(TestMethod) }, + Tags = [], + }; + var processor = new ProbeProcessor(probe, globalRateLimiter); + var probeData = new ProbeData(probe.Id, perProbeSampler, processor); + + var shouldProcess = processor.ShouldProcess(in probeData); + + Assert.True(shouldProcess); + Assert.Equal(0, globalRateLimiter.ShouldSampleCallCount); + Assert.Equal(1, perProbeSampler.SampleCallCount); + } + + private static CaptureInfo CreateAsyncEvaluateCaptureInfo() + { + return new CaptureInfo( + methodMetadataIndex: 0, + methodState: MethodState.EntryAsync, + value: new object(), + method: typeof(ProbeProcessorTests).GetMethod(nameof(TestMethod), BindingFlags.NonPublic | BindingFlags.Static)!, + invocationTargetType: typeof(ProbeProcessorTests), + memberKind: ScopeMemberKind.Argument, + type: typeof(object), + name: "argument", + localsCount: 0, + argumentsCount: 0, + asyncCaptureInfo: new AsyncCaptureInfo( + moveNextInvocationTarget: new object(), + kickoffInvocationTarget: new object(), + kickoffInvocationTargetType: typeof(object), + hoistedArgs: [], + hoistedLocals: [])); + } + + private static LogProbe CreateLogProbe(string probeId, bool captureSnapshot) + { + return new LogProbe + { + Id = probeId, + CaptureSnapshot = captureSnapshot, + EvaluateAt = EvaluateAt.Entry, + Where = new Where { MethodName = nameof(TestMethod) }, + Tags = [], + }; + } + + private static LogProbe CreateConditionalLogProbe(string probeId, string conditionJson, bool captureSnapshot) + { + var probe = CreateLogProbe(probeId, captureSnapshot); + probe.When = new SnapshotSegment(dsl: string.Empty, json: conditionJson, str: null); + return probe; + } + + private static void TestMethod() + { + } + + private sealed class GlobalRateLimiterMock : IDebuggerGlobalRateLimiter + { + private readonly Queue _results = new(); + + public GlobalRateLimiterMock(params bool[] results) + { + foreach (var result in results) + { + _results.Enqueue(result); + } + } + + public int ShouldSampleCallCount { get; private set; } + + public string LastProbeId { get; private set; } = string.Empty; + + public ProbeType? LastProbeType { get; private set; } + + public bool ShouldSample(ProbeType probeType, string probeId) + { + ShouldSampleCallCount++; + LastProbeType = probeType; + LastProbeId = probeId; + return _results.Count == 0 || _results.Dequeue(); + } + + public void SetRate(double? samplesPerSecond) + { + } + + public void ResetRate() + { + } + + public void Dispose() + { + } + } + + private sealed class AdaptiveSamplerMock : IAdaptiveSampler + { + private readonly Queue _results = new(); + + public AdaptiveSamplerMock(params bool[] results) + { + foreach (var result in results) + { + _results.Enqueue(result); + } + } + + public int SampleCallCount { get; private set; } + + public bool Sample() + { + SampleCallCount++; + return _results.Count == 0 || _results.Dequeue(); + } + + public bool Keep() => true; + + public bool Drop() => false; + + public double NextDouble() => 0; + + public void Dispose() + { + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/ProbeRateLimiterTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/ProbeRateLimiterTests.cs new file mode 100644 index 000000000000..1561f9a8f2a9 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/ProbeRateLimiterTests.cs @@ -0,0 +1,71 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using Datadog.Trace.Debugger.RateLimiting; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger; + +public class ProbeRateLimiterTests +{ + [Fact] + public void ResetRate_DisposesRemovedSampler() + { + var factory = new RecordingSamplerFactory(); + var limiter = new ProbeRateLimiter(factory.Create); + + _ = limiter.GerOrAddSampler("probe"); + var sampler = factory.Samplers[0]; + + limiter.ResetRate("probe"); + + Assert.Equal(1, sampler.DisposeCallCount); + } + + [Fact] + public void TryAddSampler_DisposesRejectedSampler() + { + var factory = new RecordingSamplerFactory(); + var limiter = new ProbeRateLimiter(factory.Create); + _ = limiter.GerOrAddSampler("probe"); + var rejectedSampler = new TestAdaptiveSampler(); + + var added = limiter.TryAddSampler("probe", rejectedSampler); + + Assert.False(added); + Assert.Equal(1, rejectedSampler.DisposeCallCount); + } + + private sealed class RecordingSamplerFactory + { + public List Samplers { get; } = []; + + public IAdaptiveSampler Create(int samplesPerSecond) + { + var sampler = new TestAdaptiveSampler(); + Samplers.Add(sampler); + return sampler; + } + } + + private sealed class TestAdaptiveSampler : IAdaptiveSampler + { + public int DisposeCallCount { get; private set; } + + public bool Sample() => true; + + public bool Keep() => true; + + public bool Drop() => false; + + public double NextDouble() => 0; + + public void Dispose() + { + DisposeCallCount++; + } + } +}