From 778f9785e1d71ac51b53a673afa85a48231290ba Mon Sep 17 00:00:00 2001 From: bm1549 Date: Tue, 10 Mar 2026 21:42:45 -0400 Subject: [PATCH 1/6] Add _dd.p.ksr propagated tag for Knuth sampling rate Co-Authored-By: Claude Opus 4.6 --- tracer/src/Datadog.Trace/Tags.cs | 7 ++ tracer/src/Datadog.Trace/TraceContext.cs | 11 ++ .../TraceContextTests_KnuthSamplingRate.cs | 109 ++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs diff --git a/tracer/src/Datadog.Trace/Tags.cs b/tracer/src/Datadog.Trace/Tags.cs index fcd8fed5539c..ea8c1d0a6ec4 100644 --- a/tracer/src/Datadog.Trace/Tags.cs +++ b/tracer/src/Datadog.Trace/Tags.cs @@ -852,6 +852,13 @@ internal static class Propagated /// A two char hex string with the product being the trace source /// internal const string TraceSource = "_dd.p.ts"; + + /// + /// Tag used to propagate the Knuth sampling rate applied to the trace. + /// Set when a sampling decision is made using agent-based or rule-based sampling. + /// The value is the applied sampling rate formatted as a string with up to 6 significant digits. + /// + internal const string KnuthSamplingRate = "_dd.p.ksr"; } } } diff --git a/tracer/src/Datadog.Trace/TraceContext.cs b/tracer/src/Datadog.Trace/TraceContext.cs index aa6293909a4c..e50d815f7c9f 100644 --- a/tracer/src/Datadog.Trace/TraceContext.cs +++ b/tracer/src/Datadog.Trace/TraceContext.cs @@ -8,6 +8,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Globalization; using System.Runtime.CompilerServices; using System.Threading; using Datadog.Trace.Agent; @@ -329,6 +330,16 @@ public void SetSamplingPriority( Tags.RemoveTag(Trace.Tags.Propagated.DecisionMaker); } + // set Knuth sampling rate as a propagated tag for agent and rule-based sampling + if (rate is { } samplingRate && mechanism is Sampling.SamplingMechanism.AgentRate + or Sampling.SamplingMechanism.LocalTraceSamplingRule + or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule + or Sampling.SamplingMechanism.RemoteUserSamplingRule) + { + // format with up to 6 significant digits, no trailing zeros + Tags.TryAddTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("G6", CultureInfo.InvariantCulture)); + } + if (notifyDistributedTracer) { DistributedTracer.Instance.SetSamplingPriority(priority); diff --git a/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs new file mode 100644 index 000000000000..39b134eb745e --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs @@ -0,0 +1,109 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.Sampling; +using Datadog.Trace.Tests.Util; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests; + +public class TraceContextTests_KnuthSamplingRate +{ + [Theory] + [InlineData(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, 0.5f, "0.5")] + [InlineData(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, 1.0f, "1")] + [InlineData(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, 0.1f, "0.1")] + [InlineData(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, 0.7654321f, "0.765432")] + [InlineData(SamplingPriorityValues.UserKeep, SamplingMechanism.LocalTraceSamplingRule, 0.25f, "0.25")] + [InlineData(SamplingPriorityValues.UserKeep, SamplingMechanism.RemoteUserSamplingRule, 0.75f, "0.75")] + [InlineData(SamplingPriorityValues.UserKeep, SamplingMechanism.RemoteAdaptiveSamplingRule, 0.333333f, "0.333333")] + public void SetSamplingPriority_SetsKsrTag_ForApplicableMechanisms( + int samplingPriority, string samplingMechanism, float rate, string expectedKsr) + { + var tracer = new StubDatadogTracer(); + var traceContext = new TraceContext(tracer); + + traceContext.SetSamplingPriority(samplingPriority, samplingMechanism, rate); + + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be(expectedKsr); + } + + [Theory] + [InlineData(SamplingPriorityValues.AutoReject, SamplingMechanism.AgentRate, 0.5f)] + [InlineData(SamplingPriorityValues.UserReject, SamplingMechanism.LocalTraceSamplingRule, 0.25f)] + [InlineData(SamplingPriorityValues.UserReject, SamplingMechanism.RemoteUserSamplingRule, 0.75f)] + [InlineData(SamplingPriorityValues.UserReject, SamplingMechanism.RemoteAdaptiveSamplingRule, 0.1f)] + public void SetSamplingPriority_SetsKsrTag_EvenForDropDecisions( + int samplingPriority, string samplingMechanism, float rate) + { + var tracer = new StubDatadogTracer(); + var traceContext = new TraceContext(tracer); + + traceContext.SetSamplingPriority(samplingPriority, samplingMechanism, rate); + + // KSR should be set regardless of keep/drop since it uses TryAddTag + // and records the original sampling rate + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().NotBeNull(); + } + + [Theory] + [InlineData(SamplingPriorityValues.UserKeep, SamplingMechanism.Manual, 0.5f)] + [InlineData(SamplingPriorityValues.UserKeep, SamplingMechanism.Asm, 0.5f)] + [InlineData(SamplingPriorityValues.AutoKeep, SamplingMechanism.Default, 0.5f)] + public void SetSamplingPriority_DoesNotSetKsrTag_ForNonApplicableMechanisms( + int samplingPriority, string samplingMechanism, float rate) + { + var tracer = new StubDatadogTracer(); + var traceContext = new TraceContext(tracer); + + traceContext.SetSamplingPriority(samplingPriority, samplingMechanism, rate); + + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().BeNull(); + } + + [Fact] + public void SetSamplingPriority_DoesNotSetKsrTag_WhenRateIsNull() + { + var tracer = new StubDatadogTracer(); + var traceContext = new TraceContext(tracer); + + traceContext.SetSamplingPriority(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, rate: null); + + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().BeNull(); + } + + [Fact] + public void SetSamplingPriority_KeepsFirstKsrTag() + { + var tracer = new StubDatadogTracer(); + var traceContext = new TraceContext(tracer); + + traceContext.SetSamplingPriority(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, 0.5f); + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.5"); + + // second call should not override the first KSR value (TryAddTag semantics) + traceContext.SetSamplingPriority(SamplingPriorityValues.UserKeep, SamplingMechanism.LocalTraceSamplingRule, 0.75f); + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.5"); + } + + [Theory] + [InlineData(0.0f, "0")] + [InlineData(1.0f, "1")] + [InlineData(0.5f, "0.5")] + [InlineData(0.1f, "0.1")] + [InlineData(0.123456f, "0.123456")] + [InlineData(0.1234567f, "0.123457")] + [InlineData(0.00001f, "1E-05")] + public void KsrTag_FormattedWithSixSignificantDigits(float rate, string expected) + { + var tracer = new StubDatadogTracer(); + var traceContext = new TraceContext(tracer); + + traceContext.SetSamplingPriority(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, rate); + + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be(expected); + } +} From 2de801725937f67133197b5e61d0a60602d49eeb Mon Sep 17 00:00:00 2001 From: bm1549 Date: Tue, 10 Mar 2026 22:42:34 -0400 Subject: [PATCH 2/6] Use SetTag instead of TryAddTag for _dd.p.ksr to always overwrite The Knuth sampling rate tag (_dd.p.ksr) was using TryAddTag which has first-write semantics, preserving any existing upstream value. This is inconsistent with all other tracers which unconditionally overwrite ksr with the locally-computed value. Switch to SetTag (replaceIfExists: true) so the local sampling rate always takes precedence. Co-Authored-By: Claude Opus 4.6 (1M context) --- tracer/src/Datadog.Trace/TraceContext.cs | 2 +- .../TraceContextTests_KnuthSamplingRate.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/TraceContext.cs b/tracer/src/Datadog.Trace/TraceContext.cs index e50d815f7c9f..f394fdbe3b91 100644 --- a/tracer/src/Datadog.Trace/TraceContext.cs +++ b/tracer/src/Datadog.Trace/TraceContext.cs @@ -337,7 +337,7 @@ or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule or Sampling.SamplingMechanism.RemoteUserSamplingRule) { // format with up to 6 significant digits, no trailing zeros - Tags.TryAddTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("G6", CultureInfo.InvariantCulture)); + Tags.SetTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("G6", CultureInfo.InvariantCulture)); } if (notifyDistributedTracer) diff --git a/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs index 39b134eb745e..eecff214105d 100644 --- a/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs +++ b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs @@ -76,7 +76,7 @@ public void SetSamplingPriority_DoesNotSetKsrTag_WhenRateIsNull() } [Fact] - public void SetSamplingPriority_KeepsFirstKsrTag() + public void SetSamplingPriority_OverwritesExistingKsrTag() { var tracer = new StubDatadogTracer(); var traceContext = new TraceContext(tracer); @@ -84,9 +84,9 @@ public void SetSamplingPriority_KeepsFirstKsrTag() traceContext.SetSamplingPriority(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, 0.5f); traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.5"); - // second call should not override the first KSR value (TryAddTag semantics) + // second call should overwrite the first KSR value (SetTag semantics) traceContext.SetSamplingPriority(SamplingPriorityValues.UserKeep, SamplingMechanism.LocalTraceSamplingRule, 0.75f); - traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.5"); + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.75"); } [Theory] From 62749778fcc3035b4e6db45b97f7f2ace2e7c0b9 Mon Sep 17 00:00:00 2001 From: bm1549 Date: Tue, 10 Mar 2026 23:28:13 -0400 Subject: [PATCH 3/6] chore: retrigger CI Co-Authored-By: Claude Opus 4.6 From 16eaa8a188d084c7fed85c617e449f934c92664f Mon Sep 17 00:00:00 2001 From: bm1549 Date: Wed, 11 Mar 2026 00:27:13 -0400 Subject: [PATCH 4/6] Fix integration test failures caused by unexpected _dd.p.ksr tag The new _dd.p.ksr propagated tag added in the ksr PR was not listed as an optional tag in DefaultTagAssertions, causing ValidateIntegrationSpans to fail with "Expected to have no remaining tags" in AspNetMvc4Tests and other Windows IIS integration tests. Co-Authored-By: Claude Sonnet 4.6 --- tracer/test/Datadog.Trace.TestHelpers/SpanTagAssertion.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tracer/test/Datadog.Trace.TestHelpers/SpanTagAssertion.cs b/tracer/test/Datadog.Trace.TestHelpers/SpanTagAssertion.cs index 37388deb3f16..a1a6d11e3943 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/SpanTagAssertion.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/SpanTagAssertion.cs @@ -28,6 +28,7 @@ public static void DefaultTagAssertions(SpanTagAssertion s) => s .IsOptional("version") .IsOptional("_dd.p.dm") // "decision maker", but contains the sampling mechanism .IsOptional("_dd.p.tid") // contains the upper 64 bits of a 128-bit trace id + .IsOptional("_dd.p.ksr") // Knuth sampling rate, propagated tag added by the tracer .IsOptional("_dd.parent_id") // Contains the 16 length hex decoded last found parent_id found set from either exising `p` tag on tracestate or headers .IsOptional("error.msg") .IsOptional("error.type") From c4b337e52a66149c2113370df0e7d3e48045cffa Mon Sep 17 00:00:00 2001 From: bm1549 Date: Mon, 23 Mar 2026 17:36:59 -0400 Subject: [PATCH 5/6] Use TryAddTag for _dd.p.ksr to preserve original sampling rate Address Codex review feedback: change SetTag to TryAddTag for the _dd.p.ksr propagated tag so it follows the same first-write-wins semantics as AppliedSamplingRate and SamplingMechanism. This prevents inconsistency where _dd.p.ksr could reflect a newer rate while _dd.agent_psr/_dd.rule_psr still come from the original rate. Co-Authored-By: Claude Opus 4.6 (1M context) --- tracer/src/Datadog.Trace/TraceContext.cs | 5 +++-- .../TraceContextTests_KnuthSamplingRate.cs | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tracer/src/Datadog.Trace/TraceContext.cs b/tracer/src/Datadog.Trace/TraceContext.cs index f394fdbe3b91..67762093405d 100644 --- a/tracer/src/Datadog.Trace/TraceContext.cs +++ b/tracer/src/Datadog.Trace/TraceContext.cs @@ -330,14 +330,15 @@ public void SetSamplingPriority( Tags.RemoveTag(Trace.Tags.Propagated.DecisionMaker); } - // set Knuth sampling rate as a propagated tag for agent and rule-based sampling + // set Knuth sampling rate as a propagated tag for agent and rule-based sampling. + // use TryAddTag to preserve the original rate, consistent with AppliedSamplingRate ??= rate above. if (rate is { } samplingRate && mechanism is Sampling.SamplingMechanism.AgentRate or Sampling.SamplingMechanism.LocalTraceSamplingRule or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule or Sampling.SamplingMechanism.RemoteUserSamplingRule) { // format with up to 6 significant digits, no trailing zeros - Tags.SetTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("G6", CultureInfo.InvariantCulture)); + Tags.TryAddTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("G6", CultureInfo.InvariantCulture)); } if (notifyDistributedTracer) diff --git a/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs index eecff214105d..1d4d3a9f1b3b 100644 --- a/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs +++ b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs @@ -76,7 +76,7 @@ public void SetSamplingPriority_DoesNotSetKsrTag_WhenRateIsNull() } [Fact] - public void SetSamplingPriority_OverwritesExistingKsrTag() + public void SetSamplingPriority_PreservesOriginalKsrTag() { var tracer = new StubDatadogTracer(); var traceContext = new TraceContext(tracer); @@ -84,9 +84,10 @@ public void SetSamplingPriority_OverwritesExistingKsrTag() traceContext.SetSamplingPriority(SamplingPriorityValues.AutoKeep, SamplingMechanism.AgentRate, 0.5f); traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.5"); - // second call should overwrite the first KSR value (SetTag semantics) + // second call should NOT overwrite — TryAddTag preserves the original rate, + // consistent with AppliedSamplingRate ??= rate semantics traceContext.SetSamplingPriority(SamplingPriorityValues.UserKeep, SamplingMechanism.LocalTraceSamplingRule, 0.75f); - traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.75"); + traceContext.Tags.GetTag(Tags.Propagated.KnuthSamplingRate).Should().Be("0.5"); } [Theory] From 61998e7a5e6e9b9d63b1e22ad1f382382a33109f Mon Sep 17 00:00:00 2001 From: bm1549 Date: Mon, 23 Mar 2026 19:42:26 -0400 Subject: [PATCH 6/6] Use decimal format for _dd.p.ksr instead of scientific notation Change format specifier from "G6" (6 significant digits, allows scientific notation) to "0.######" (up to 6 decimal digits) per RFC. This fixes 0.00001 being formatted as "1E-05" instead of "0.00001". Co-Authored-By: Claude Opus 4.6 (1M context) --- tracer/src/Datadog.Trace/TraceContext.cs | 4 ++-- .../TraceContextTests_KnuthSamplingRate.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/TraceContext.cs b/tracer/src/Datadog.Trace/TraceContext.cs index 67762093405d..13aafc1bb12f 100644 --- a/tracer/src/Datadog.Trace/TraceContext.cs +++ b/tracer/src/Datadog.Trace/TraceContext.cs @@ -337,8 +337,8 @@ or Sampling.SamplingMechanism.LocalTraceSamplingRule or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule or Sampling.SamplingMechanism.RemoteUserSamplingRule) { - // format with up to 6 significant digits, no trailing zeros - Tags.TryAddTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("G6", CultureInfo.InvariantCulture)); + // format with up to 6 decimal digits, no trailing zeros (per RFC) + Tags.TryAddTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("0.######", CultureInfo.InvariantCulture)); } if (notifyDistributedTracer) diff --git a/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs index 1d4d3a9f1b3b..9d35ef134b5a 100644 --- a/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs +++ b/tracer/test/Datadog.Trace.Tests/TraceContextTests_KnuthSamplingRate.cs @@ -97,8 +97,8 @@ public void SetSamplingPriority_PreservesOriginalKsrTag() [InlineData(0.1f, "0.1")] [InlineData(0.123456f, "0.123456")] [InlineData(0.1234567f, "0.123457")] - [InlineData(0.00001f, "1E-05")] - public void KsrTag_FormattedWithSixSignificantDigits(float rate, string expected) + [InlineData(0.00001f, "0.00001")] + public void KsrTag_FormattedWithUpToSixDecimalDigits(float rate, string expected) { var tracer = new StubDatadogTracer(); var traceContext = new TraceContext(tracer);