fix(tracer): only set _dd.p.ksr after agent rates are received (#4523)

### What does this PR do?

Fixes `_dd.p.ksr` (Knuth Sampling Rate) to only be set on spans when the
agent has provided sampling rates via `readRatesJSON()`. Previously, ksr
was unconditionally set in `prioritySampler.apply()`, including when the
rate was the initial client-side default (1.0) before any agent response
arrived.

Also refactors `prioritySampler` to consolidate lock acquisitions:
extracts `getRateLocked()` so `apply()` acquires `ps.mu.RLock` only once
to read both the rate and `agentRatesLoaded`.

### Motivation

Cross-language consistency: Python, Java, PHP, and other tracers only
set ksr when actual agent rates or sampling rules are applied, not for
the default fallback. This aligns Go with that behavior.

See RFC: "Transmit Knuth sampling rate to backend"

### Additional Notes

- Added `agentRatesLoaded` bool field to `prioritySampler`, set to
`true` in `readRatesJSON()`
- `apply()` now gates ksr behind `agentRatesLoaded` check
- Extracted `getRateLocked()` to avoid double lock acquisition in
`apply()`
- Rule-based sampling path (`applyTraceRuleSampling` in span.go)
unchanged — correctly always sets ksr
- Tests added: `ksr-not-set-without-agent-rates` and
`ksr-set-after-agent-rates-received`

Related PRs across tracers:
- Java: DataDog/dd-trace-java#10802
- .NET: DataDog/dd-trace-dotnet#8287
- Ruby: DataDog/dd-trace-rb#5436
- Node.js: DataDog/dd-trace-js#7741
- PHP: DataDog/dd-trace-php#3701
- Rust: DataDog/dd-trace-rs#180
- C++: DataDog/dd-trace-cpp#288
- System tests: DataDog/system-tests#6466

### Reviewer's Checklist

- [x] Changed code has unit tests for its functionality at or near 100%
coverage.
- [x] [System-Tests](https://github.com/DataDog/system-tests/) covering
this feature have been added and enabled with the va.b.c-dev version
tag.
- [ ] There is a benchmark for any new code, or changes to existing
code.
- [x] If this interacts with the agent in a new way, a system test has
been added.
- [x] New code is free of linting errors. You can check this by running
`make lint` locally.
- [x] New code doesn't break existing tests. You can check this by
running `make test` locally.
- [ ] Add an appropriate team label so this PR gets put in the right
place for the release notes.
- [ ] All generated files are up to date. You can check this by running
`make generate` locally.
- [ ] Non-trivial go.mod changes, e.g. adding new modules, are reviewed
by @DataDog/dd-trace-go-guild. Make sure all nested modules are up to
date by running `make fix-modules` locally.

Unsure? Have a question? Request a review!

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Dario Castañé <dario.castane@datadoghq.com>
Co-authored-by: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com>

Author: 4 people

ddtrace/tracer/sampler.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/DataDog/dd-trace-go/v2/ddtrace/ext"
 	"github.com/DataDog/dd-trace-go/v2/internal/locking"
+	"github.com/DataDog/dd-trace-go/v2/internal/locking/assert"
 	"github.com/DataDog/dd-trace-go/v2/internal/samplernames"
 )
 
@@ -137,10 +138,11 @@ const rampUpInterval = time.Second
 // prioritySampler holds a set of per-service sampling rates and applies
 // them to spans.
 type prioritySampler struct {
-	mu          locking.RWMutex
-	rates       map[serviceEnvKey]float64 // +checklocks:mu
-	defaultRate float64                   // +checklocks:mu
-	lastCapped  time.Time                 // +checklocks:mu
+	mu               locking.RWMutex
+	rates            map[serviceEnvKey]float64 // +checklocks:mu
+	defaultRate      float64                   // +checklocks:mu
+	agentRatesLoaded bool                      // +checklocks:mu
+	lastCapped       time.Time                 // +checklocks:mu
 }
 
 func newPrioritySampler() *prioritySampler {
@@ -198,6 +200,7 @@ func (ps *prioritySampler) readRatesJSON(rc io.ReadCloser) error {
 	}
 	ps.mu.Lock()
 	defer ps.mu.Unlock()
+	ps.agentRatesLoaded = true
 	now := time.Now()
 	canIncrease := ps.lastCapped.IsZero() || now.Sub(ps.lastCapped) >= rampUpInterval
 	capApplied := false
@@ -225,9 +228,17 @@ func (ps *prioritySampler) readRatesJSON(rc io.ReadCloser) error {
 // guard the span.
 // +checklocksignore — Called during initialization in StartSpan, span not yet shared.
 func (ps *prioritySampler) getRate(spn *Span) float64 {
-	key := serviceEnvKey{service: spn.service, env: spn.meta[ext.Environment]}
 	ps.mu.RLock()
 	defer ps.mu.RUnlock()
+	return ps.getRateLocked(spn)
+}
+
+// getRateLocked returns the sampling rate for the given span.
+// Caller must hold ps.mu (at least RLock).
+// +checklocksignore — Called during initialization in StartSpan, span not yet shared.
+func (ps *prioritySampler) getRateLocked(spn *Span) float64 {
+	assert.RWMutexRLocked(&ps.mu)
+	key := serviceEnvKey{service: spn.service, env: spn.meta[ext.Environment]}
 	if rate, ok := ps.rates[key]; ok {
 		return rate
 	}
@@ -245,13 +256,21 @@ func (ps *prioritySampler) getDefaultRate() float64 {
 // to modify the span.
 // +checklocksignore — Called during initialization in StartSpan, span not yet shared.
 func (ps *prioritySampler) apply(spn *Span) {
-	rate := ps.getRate(spn)
+	ps.mu.RLock()
+	rate := ps.getRateLocked(spn)
+	fromAgent := ps.agentRatesLoaded
+	ps.mu.RUnlock()
 	if sampledByRate(spn.traceID, rate) {
 		spn.setSamplingPriority(ext.PriorityAutoKeep, samplernames.AgentRate)
 	} else {
 		spn.setSamplingPriority(ext.PriorityAutoReject, samplernames.AgentRate)
 	}
 	spn.SetTag(keySamplingPriorityRate, rate)
-	// Set the Knuth sampling rate tag when sampled by agent rate
-	spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate))
+	// Only set the Knuth sampling rate tag when actual agent rates have been
+	// received. The initial default rate (1.0) is a client-side fallback that
+	// does not represent an agent-configured rate, so it must not propagate
+	// as _dd.p.ksr to stay consistent with other tracers.
+	if fromAgent {
+		spn.SetTag(keyKnuthSamplingRate, formatKnuthSamplingRate(rate))
+	}
 }

ddtrace/tracer/sampler_test.go

@@ -202,6 +202,66 @@ func TestPrioritySampler(t *testing.T) {
 		assert.EqualValues(ext.PriorityAutoReject, priority)
 		assert.EqualValues(0.5, rate)
 	})
+
+	t.Run("ksr-not-set-without-agent-rates", func(t *testing.T) {
+		// When no agent rates have been received, the priority sampler uses
+		// its initial default rate (1.0). This is a client-side fallback and
+		// should NOT propagate as _dd.p.ksr to stay consistent with other
+		// Datadog tracers.
+		assert := assert.New(t)
+		ps := newPrioritySampler()
+		spn := newBasicSpan("http.request")
+		spn.service = "my-service"
+		spn.traceID = 1
+
+		ps.apply(spn)
+		_, ok := getMeta(spn, keyKnuthSamplingRate)
+		assert.False(ok, "_dd.p.ksr must not be set when no agent rates have been received")
+
+		// Sampling priority and rate metric should still be set normally
+		priority, _ := getMetric(spn, keySamplingPriority)
+		assert.EqualValues(ext.PriorityAutoKeep, priority)
+		rate, _ := getMetric(spn, keySamplingPriorityRate)
+		assert.EqualValues(1.0, rate)
+	})
+
+	t.Run("ksr-set-after-agent-rates-received", func(t *testing.T) {
+		// After agent rates are received via readRatesJSON, apply should
+		// set _dd.p.ksr — even when the span falls through to the default
+		// rate provided by the agent.
+		assert := assert.New(t)
+		ps := newPrioritySampler()
+		assert.NoError(ps.readRatesJSON(
+			io.NopCloser(strings.NewReader(
+				`{
+					"rate_by_service":{
+						"service:,env:":0.8,
+						"service:obfuscate.http,env:":0.5
+					}
+				}`,
+			)),
+		))
+
+		// Span matching a per-service rate
+		spn1 := newBasicSpan("http.request")
+		spn1.service = "obfuscate.http"
+		spn1.traceID = 1
+
+		ps.apply(spn1)
+		ksr, ok := getMeta(spn1, keyKnuthSamplingRate)
+		assert.True(ok, "_dd.p.ksr must be set when agent rates have been received (per-service)")
+		assert.Equal("0.5", ksr)
+
+		// Span falling through to agent-provided default rate
+		spn2 := newBasicSpan("http.request")
+		spn2.service = "unknown-service"
+		spn2.traceID = 1
+
+		ps.apply(spn2)
+		ksr, ok = getMeta(spn2, keyKnuthSamplingRate)
+		assert.True(ok, "_dd.p.ksr must be set when agent rates have been received (default)")
+		assert.Equal("0.8", ksr)
+	})
 }
 
 func BenchmarkPrioritySamplerGetRate(b *testing.B) {