From 2b0e03a25728526c6502fb5c49f86a6192d21b2d Mon Sep 17 00:00:00 2001 From: bm1549 Date: Tue, 10 Mar 2026 22:45:40 -0400 Subject: [PATCH 1/5] fix(ddtrace/tracer): only set _dd.p.ksr when agent rates have been received The priority sampler was unconditionally setting the _dd.p.ksr tag on every span, even when using the initial client-side default rate (1.0) before any agent response was received. This is inconsistent with other Datadog tracers, which only propagate ksr when actual agent rates or sampling rules are applied. Add an agentRatesLoaded flag to prioritySampler that is set to true the first time readRatesJSON processes an agent response. The apply method now only sets keyKnuthSamplingRate when this flag is true. Rule-based sampling (applyTraceRuleSampling) is unaffected and continues to set ksr unconditionally, which is the correct behavior. Co-Authored-By: Claude Opus 4.6 (1M context) --- ddtrace/tracer/sampler.go | 20 +++++++++--- ddtrace/tracer/sampler_test.go | 60 ++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/sampler.go b/ddtrace/tracer/sampler.go index 5a70082d6e..b6b1df33df 100644 --- a/ddtrace/tracer/sampler.go +++ b/ddtrace/tracer/sampler.go @@ -133,9 +133,10 @@ type serviceEnvKey struct { // prioritySampler holds a set of per-service sampling rates and applies // them to spans. type prioritySampler struct { - mu locking.RWMutex - rates map[serviceEnvKey]float64 // +checklocks:mu - defaultRate float64 // +checklocks:mu + mu locking.RWMutex + rates map[serviceEnvKey]float64 // +checklocks:mu + defaultRate float64 // +checklocks:mu + agentRatesLoaded bool // +checklocks:mu } func newPrioritySampler() *prioritySampler { @@ -176,6 +177,7 @@ func (ps *prioritySampler) readRatesJSON(rc io.ReadCloser) error { } ps.mu.Lock() defer ps.mu.Unlock() + ps.agentRatesLoaded = true ps.rates = rates if v, ok := ps.rates[defaultRateKey]; ok { ps.defaultRate = v @@ -208,6 +210,14 @@ func (ps *prioritySampler) apply(spn *Span) { spn.setSamplingPriority(ext.PriorityAutoReject, samplernames.AgentRate) } spn.SetTag(keySamplingPriorityRate, rate) - // Set the Knuth sampling rate tag when sampled by agent rate - spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate)) + // Only set the Knuth sampling rate tag when actual agent rates have been + // received. The initial default rate (1.0) is a client-side fallback that + // does not represent an agent-configured rate, so it must not propagate + // as _dd.p.ksr to stay consistent with other tracers. + ps.mu.RLock() + loaded := ps.agentRatesLoaded + ps.mu.RUnlock() + if loaded { + spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate)) + } } diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 4bff852e50..2ea3d457a5 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -201,6 +201,66 @@ func TestPrioritySampler(t *testing.T) { assert.EqualValues(ext.PriorityAutoReject, priority) assert.EqualValues(0.5, rate) }) + + t.Run("ksr-not-set-without-agent-rates", func(t *testing.T) { + // When no agent rates have been received, the priority sampler uses + // its initial default rate (1.0). This is a client-side fallback and + // should NOT propagate as _dd.p.ksr to stay consistent with other + // Datadog tracers. + assert := assert.New(t) + ps := newPrioritySampler() + spn := newBasicSpan("http.request") + spn.service = "my-service" + spn.traceID = 1 + + ps.apply(spn) + _, ok := getMeta(spn, keyKnuthSamplingRate) + assert.False(ok, "_dd.p.ksr must not be set when no agent rates have been received") + + // Sampling priority and rate metric should still be set normally + priority, _ := getMetric(spn, keySamplingPriority) + assert.EqualValues(ext.PriorityAutoKeep, priority) + rate, _ := getMetric(spn, keySamplingPriorityRate) + assert.EqualValues(1.0, rate) + }) + + t.Run("ksr-set-after-agent-rates-received", func(t *testing.T) { + // After agent rates are received via readRatesJSON, apply should + // set _dd.p.ksr — even when the span falls through to the default + // rate provided by the agent. + assert := assert.New(t) + ps := newPrioritySampler() + assert.NoError(ps.readRatesJSON( + io.NopCloser(strings.NewReader( + `{ + "rate_by_service":{ + "service:,env:":0.8, + "service:obfuscate.http,env:":0.5 + } + }`, + )), + )) + + // Span matching a per-service rate + spn1 := newBasicSpan("http.request") + spn1.service = "obfuscate.http" + spn1.traceID = 1 + + ps.apply(spn1) + ksr, ok := getMeta(spn1, keyKnuthSamplingRate) + assert.True(ok, "_dd.p.ksr must be set when agent rates have been received (per-service)") + assert.Equal("0.5", ksr) + + // Span falling through to agent-provided default rate + spn2 := newBasicSpan("http.request") + spn2.service = "unknown-service" + spn2.traceID = 1 + + ps.apply(spn2) + ksr, ok = getMeta(spn2, keyKnuthSamplingRate) + assert.True(ok, "_dd.p.ksr must be set when agent rates have been received (default)") + assert.Equal("0.8", ksr) + }) } func BenchmarkPrioritySamplerGetRate(b *testing.B) { From 47123ddbcd4071637e6fa896581285764389acb2 Mon Sep 17 00:00:00 2001 From: bm1549 Date: Wed, 18 Mar 2026 19:59:50 -0400 Subject: [PATCH 2/5] refactor(sampler): consolidate lock acquisitions in prioritySampler.apply Extract getRateLocked() so apply() acquires ps.mu.RLock only once to read both the rate and agentRatesLoaded, addressing review feedback. Co-Authored-By: Claude Opus 4.6 --- ddtrace/tracer/sampler.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/sampler.go b/ddtrace/tracer/sampler.go index d929287470..26a031da9e 100644 --- a/ddtrace/tracer/sampler.go +++ b/ddtrace/tracer/sampler.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" "github.com/DataDog/dd-trace-go/v2/internal/locking" + "github.com/DataDog/dd-trace-go/v2/internal/locking/assert" "github.com/DataDog/dd-trace-go/v2/internal/samplernames" ) @@ -227,9 +228,17 @@ func (ps *prioritySampler) readRatesJSON(rc io.ReadCloser) error { // guard the span. // +checklocksignore — Called during initialization in StartSpan, span not yet shared. func (ps *prioritySampler) getRate(spn *Span) float64 { - key := serviceEnvKey{service: spn.service, env: spn.meta[ext.Environment]} ps.mu.RLock() defer ps.mu.RUnlock() + return ps.getRateLocked(spn) +} + +// getRateLocked returns the sampling rate for the given span. +// Caller must hold ps.mu (at least RLock). +// +checklocksignore — Called during initialization in StartSpan, span not yet shared. +func (ps *prioritySampler) getRateLocked(spn *Span) float64 { + assert.RWMutexRLocked(&ps.mu) + key := serviceEnvKey{service: spn.service, env: spn.meta[ext.Environment]} if rate, ok := ps.rates[key]; ok { return rate } @@ -247,7 +256,10 @@ func (ps *prioritySampler) getDefaultRate() float64 { // to modify the span. // +checklocksignore — Called during initialization in StartSpan, span not yet shared. func (ps *prioritySampler) apply(spn *Span) { - rate := ps.getRate(spn) + ps.mu.RLock() + rate := ps.getRateLocked(spn) + loaded := ps.agentRatesLoaded + ps.mu.RUnlock() if sampledByRate(spn.traceID, rate) { spn.setSamplingPriority(ext.PriorityAutoKeep, samplernames.AgentRate) } else { @@ -258,9 +270,6 @@ func (ps *prioritySampler) apply(spn *Span) { // received. The initial default rate (1.0) is a client-side fallback that // does not represent an agent-configured rate, so it must not propagate // as _dd.p.ksr to stay consistent with other tracers. - ps.mu.RLock() - loaded := ps.agentRatesLoaded - ps.mu.RUnlock() if loaded { spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate)) } From 8254210de01b6a1406785ddd051eb3ad4d183e47 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 19 Mar 2026 11:02:03 -0400 Subject: [PATCH 3/5] Update ddtrace/tracer/sampler.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Dario Castañé --- ddtrace/tracer/sampler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/sampler.go b/ddtrace/tracer/sampler.go index 26a031da9e..36060d3ec4 100644 --- a/ddtrace/tracer/sampler.go +++ b/ddtrace/tracer/sampler.go @@ -270,7 +270,8 @@ func (ps *prioritySampler) apply(spn *Span) { // received. The initial default rate (1.0) is a client-side fallback that // does not represent an agent-configured rate, so it must not propagate // as _dd.p.ksr to stay consistent with other tracers. - if loaded { - spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate)) + if !loaded { + return } + spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate)) } From 1828d1f03ce04664cb847a7df1182fd64fa5e5ac Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 19 Mar 2026 16:05:56 -0400 Subject: [PATCH 4/5] Update ddtrace/tracer/sampler.go Co-authored-by: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> --- ddtrace/tracer/sampler.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/sampler.go b/ddtrace/tracer/sampler.go index 36060d3ec4..7be919b53f 100644 --- a/ddtrace/tracer/sampler.go +++ b/ddtrace/tracer/sampler.go @@ -270,8 +270,7 @@ func (ps *prioritySampler) apply(spn *Span) { // received. The initial default rate (1.0) is a client-side fallback that // does not represent an agent-configured rate, so it must not propagate // as _dd.p.ksr to stay consistent with other tracers. - if !loaded { - return + if fromAgent { + spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate)) } - spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate)) } From cfbe029c31cabae534fb0ea071369a588f0b0f6c Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 19 Mar 2026 16:06:04 -0400 Subject: [PATCH 5/5] Update ddtrace/tracer/sampler.go Co-authored-by: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> --- ddtrace/tracer/sampler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/sampler.go b/ddtrace/tracer/sampler.go index 7be919b53f..0a3391948c 100644 --- a/ddtrace/tracer/sampler.go +++ b/ddtrace/tracer/sampler.go @@ -258,7 +258,7 @@ func (ps *prioritySampler) getDefaultRate() float64 { func (ps *prioritySampler) apply(spn *Span) { ps.mu.RLock() rate := ps.getRateLocked(spn) - loaded := ps.agentRatesLoaded + fromAgent := ps.agentRatesLoaded ps.mu.RUnlock() if sampledByRate(spn.traceID, rate) { spn.setSamplingPriority(ext.PriorityAutoKeep, samplernames.AgentRate)