Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -271,24 +271,38 @@ class OpenTelemetryTest extends InstrumentationSpecification {
httpPropagator.inject(context, textMap, new TextMapSetter())

then:
def expectedTraceparent = "00-${span.delegate.traceId.toHexStringPadded(32)}" +
"-${DDSpanId.toHexStringPadded(span.delegate.spanId)}" +
def traceId = span.delegate.traceId as DDTraceId
def spanId = span.delegate.spanId
def expectedTraceparent = "00-${traceId.toHexStringPadded(32)}" +
"-${DDSpanId.toHexStringPadded(spanId)}" +
"-" + (propagatedPriority > 0 ? "01" : "00")
def expectedTracestate = "dd=s:${propagatedPriority};p:${DDSpanId.toHexStringPadded(span.delegate.spanId)}"
def expectedDatadogTags = null
def expectedTracestate = "dd=s:${propagatedPriority};p:${DDSpanId.toHexStringPadded(spanId)}"
def datadogTags = []
Comment thread
bm1549 marked this conversation as resolved.
Outdated
if (propagatedMechanism != UNKNOWN) {
expectedDatadogTags = "_dd.p.dm=-" + propagatedMechanism
expectedTracestate+= ";t.dm:-" + propagatedMechanism
datadogTags << "_dd.p.dm=-" + propagatedMechanism
expectedTracestate += ";t.dm:-" + propagatedMechanism
}
if (traceId.toHighOrderLong() != 0) {
expectedTracestate += ";t.tid:${traceId.toHexStringPadded(32).substring(0, 16)}"
}
if (contextPriority == UNSET) {
expectedTracestate += ";t.ksr:1"
}
if (traceId.toHighOrderLong() != 0) {
datadogTags << "_dd.p.tid=" + traceId.toHexStringPadded(32).substring(0, 16)
}
if (contextPriority == UNSET) {
datadogTags << "_dd.p.ksr=1"
}
def expectedTextMap = [
"x-datadog-trace-id" : "$span.delegate.traceId",
"x-datadog-parent-id" : "$span.delegate.spanId",
"x-datadog-trace-id" : "$traceId",
"x-datadog-parent-id" : "$spanId",
"x-datadog-sampling-priority": propagatedPriority.toString(),
"traceparent" : expectedTraceparent,
"tracestate" : expectedTracestate,
]
if (expectedDatadogTags != null) {
expectedTextMap.put("x-datadog-tags", expectedDatadogTags)
if (!datadogTags.empty) {
expectedTextMap.put("x-datadog-tags", datadogTags.join(','))
}
textMap == expectedTextMap

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ class DatadogPropagatorTest extends AgentPropagatorTest {
if (traceId.length() == 32) {
tags+= '_dd.p.tid='+ traceId.substring(0, 16)
}
if (sampling == UNSET) {
tags+= '_dd.p.ksr=1'
}
assert headers['x-datadog-tags'] == tags.join(',')
assert headers['x-datadog-sampling-priority'] == samplingPriority
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -284,22 +284,30 @@ class OpenTracing31Test extends InstrumentationSpecification {
"-${DDSpanId.toHexStringPadded(context.delegate.spanId)}" +
"-" + (propagatedPriority > 0 ? "01" : "00")
def expectedTracestate = "dd=s:${propagatedPriority};p:${DDSpanId.toHexStringPadded(context.delegate.spanId)}"
def expectedDatadogTags = null
def datadogTags = []
if (propagatedPriority > 0) {
def effectiveSamplingMechanism = contextPriority == UNSET ? AGENT_RATE : samplingMechanism
expectedDatadogTags = "_dd.p.dm=-" + effectiveSamplingMechanism
expectedTracestate+= ";t.dm:-" + effectiveSamplingMechanism
datadogTags << "_dd.p.dm=-" + effectiveSamplingMechanism
expectedTracestate += ";t.dm:-" + effectiveSamplingMechanism
}
def traceId = context.delegate.traceId as DDTraceId
if (traceId.toHighOrderLong() != 0) {
expectedTracestate += ";t.tid:${traceId.toHexStringPadded(32).substring(0, 16)}"
datadogTags << "_dd.p.tid=" + traceId.toHexStringPadded(32).substring(0, 16)
}
if (contextPriority == UNSET) {
expectedTracestate += ";t.ksr:1"
datadogTags << "_dd.p.ksr=1"
}

def expectedTextMap = [
"x-datadog-trace-id" : "$context.delegate.traceId",
"x-datadog-parent-id" : "$context.delegate.spanId",
"x-datadog-sampling-priority": propagatedPriority.toString(),
"traceparent" : expectedTraceparent,
"tracestate" : expectedTracestate,
]
if (expectedDatadogTags != null) {
expectedTextMap.put("x-datadog-tags", expectedDatadogTags)
if (!datadogTags.empty) {
expectedTextMap.put("x-datadog-tags", datadogTags.join(','))
}
textMap == expectedTextMap

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import datadog.trace.agent.test.InstrumentationSpecification
import datadog.trace.api.DDSpanId
import datadog.trace.api.DDTags
import datadog.trace.api.DDTraceId
import datadog.trace.api.internal.util.LongStringUtils
import datadog.trace.api.interceptor.MutableSpan
import datadog.trace.bootstrap.instrumentation.api.AgentTracer
import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities
Expand Down Expand Up @@ -299,21 +300,30 @@ class OpenTracing32Test extends InstrumentationSpecification {
"-${DDSpanId.toHexStringPadded(context.delegate.spanId)}" +
"-" + (propagatedPriority > 0 ? "01" : "00")
def expectedTracestate = "dd=s:${propagatedPriority};p:${DDSpanId.toHexStringPadded(context.delegate.spanId)}"
def expectedDatadogTags = null
def datadogTags = []
if (propagatedPriority > 0) {
def effectiveSamplingMechanism = contextPriority == UNSET ? AGENT_RATE : samplingMechanism
expectedDatadogTags = "_dd.p.dm=-" + effectiveSamplingMechanism
datadogTags << "_dd.p.dm=-" + effectiveSamplingMechanism
expectedTracestate+= ";t.dm:-" + effectiveSamplingMechanism
}
def traceId = context.delegate.traceId as DDTraceId
if (traceId.toHighOrderLong() != 0) {
expectedTracestate+= ";t.tid:${traceId.toHexStringPadded(32).substring(0, 16)}"
datadogTags << "_dd.p.tid=" + LongStringUtils.toHexStringPadded(traceId.toHighOrderLong(), 16)
}
if (contextPriority == UNSET) {
expectedTracestate+= ";t.ksr:1"
datadogTags << "_dd.p.ksr=1"
}
def expectedTextMap = [
"x-datadog-trace-id" : "$context.delegate.traceId",
"x-datadog-parent-id" : "$context.delegate.spanId",
"x-datadog-sampling-priority": propagatedPriority.toString(),
"traceparent" : expectedTraceparent,
"tracestate" : expectedTracestate
]
if (expectedDatadogTags != null) {
expectedTextMap.put("x-datadog-tags", expectedDatadogTags)
if (!datadogTags.empty) {
expectedTextMap.put("x-datadog-tags", datadogTags.join(','))
}
textMap == expectedTextMap

Expand Down
28 changes: 28 additions & 0 deletions dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java
Original file line number Diff line number Diff line change
Expand Up @@ -646,10 +646,38 @@ public DDSpan setSamplingPriority(
int samplingPriority, CharSequence rate, double sampleRate, int samplingMechanism) {
if (context.setSamplingPriority(samplingPriority, samplingMechanism)) {
setMetric(rate, sampleRate);
if (samplingMechanism == SamplingMechanism.AGENT_RATE
|| samplingMechanism == SamplingMechanism.LOCAL_USER_RULE
|| samplingMechanism == SamplingMechanism.REMOTE_USER_RULE
|| samplingMechanism == SamplingMechanism.REMOTE_ADAPTIVE_RULE) {
context.getPropagationTags().updateKnuthSamplingRate(formatKnuthSamplingRate(sampleRate));
}
}
return this;
}

/**
* Formats the sampling rate for the _dd.p.ksr propagated tag. Uses up to 6 significant digits
* with no trailing zeros, matching the Go/Python reference implementations (%.6g format).
*/
static String formatKnuthSamplingRate(double rate) {
// Use %.6g format: up to 6 significant digits, no trailing zeros
// This matches Go's strconv.FormatFloat(rate, 'g', 6, 64) and Python's f"{rate:.6g}"
String formatted = String.format("%.6g", rate);
Comment thread
bm1549 marked this conversation as resolved.
Outdated
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should fix the localization issue:

Suggested change
String formatted = String.format("%.6g", rate);
String formatted = String.format(Locale.ROOT, "%.6g", rate);

Note you'll also need to add an import for Locale

Copy link
Copy Markdown
Contributor

@dougqh dougqh Mar 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a pretty hot place to be using String.format. Profiles already show that PropagationTags is a major source of allocation. That said, I'm not going to block this PR, but I do expect that I'll end up reworking this sooner than later.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if we can replace that with a tiny custom formatter that rounds to 6 decimal places and trims trailing zeros? That would avoid Formatter allocations while keeping the output stable and locale-independent.

Copy link
Copy Markdown
Contributor

@dougqh dougqh Mar 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think we'll have to. That String.format call is quite expensive in terms of allocation.
It is auto-boxing a double, then creating an array for the var-args, and then allocating a String.
I started working on a benchmark to check the overhead.

Copy link
Copy Markdown
Contributor

@dougqh dougqh Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apparently, the boxing, var-args, and String were just the tip of the iceberg. After looking into it, format ends up allocating a bunch of streams, etc.

For comparison sake, I'll see what repeatedly using a formatter does, but ultimately, I think we'll just have to roll our own lighter solution.

Copy link
Copy Markdown
Contributor Author

@bm1549 bm1549 Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dougqh just handled this in the latest commit

I've replaced String.format with manual char-array arithmetic. Benchmarked showed ~11ns vs ~320ns (30x improvement). It also eagerly caches the TagValue in updateKnuthSamplingRate() so getKnuthSamplingRateTagValue() on the header-injection hot path is just a volatile read (~1.2ns, zero allocation)

To be safe, I've also added a JMH benchmark (KnuthSamplingRateFormatBenchmark) comparing all three approaches.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I made some suggestions about the benchmark.
I'll also re-run my stress test to see how the new approach compares.

Copy link
Copy Markdown
Contributor

@dougqh dougqh Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The new approach is certainly much better, but there's still a significant increase in allocation.
And unfortunately, that means that this change reverses the progress made in the last couple releases.

Fortunately, I think propagation tags itself can be improved to avoid work most of the time.
But tor now, I would suggest that this be put behind a flag that is off by default -- until the corresponding backend changes are ready.

Concurrently, I'll look into further improvements to PropagationTags to see if the overhead can be deferred. That said, we need to be very careful adding to propagation in general because it is one of the hottest portions of any tracer.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in the latest commit. Added a static cache at the class level

Now updateRateFreshTrace shows ~ 0 B/op structurally, not JIT-dependent. Also updated the benchmark per your suggestions: throughput mode, 8 threads, -prof gc.

// Remove trailing zeros after decimal point (avoid forbidden String#replaceAll)
if (formatted.contains(".")) {
Comment thread
bm1549 marked this conversation as resolved.
Outdated
int end = formatted.length();
while (end > 0 && formatted.charAt(end - 1) == '0') {
end--;
}
if (end > 0 && formatted.charAt(end - 1) == '.') {
end--;
}
formatted = formatted.substring(0, end);
}
return formatted;
}

@Override
public DDSpan setSpanSamplingPriority(double rate, int limit) {
context.setSpanSamplingPriority(rate, limit);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,14 @@ public interface Factory {

public abstract String getDebugPropagation();

/**
* Updates the Knuth sampling rate (_dd.p.ksr) propagated tag. This records the sampling rate that
* was applied when making an agent-based or rule-based sampling decision.
*
* @param rate the formatted sampling rate string (up to 6 significant digits, no trailing zeros)
*/
public abstract void updateKnuthSamplingRate(String rate);
Comment thread
bm1549 marked this conversation as resolved.
Outdated

public HashMap<String, String> createTagMap() {
HashMap<String, String> result = new HashMap<>();
fillTagMap(result);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ abstract class PTagsCodec {
protected static final TagKey TRACE_ID_TAG = TagKey.from("tid");
protected static final TagKey TRACE_SOURCE_TAG = TagKey.from("ts");
protected static final TagKey DEBUG_TAG = TagKey.from("debug");
protected static final TagKey KNUTH_SAMPLING_RATE_TAG = TagKey.from("ksr");
protected static final String PROPAGATION_ERROR_MALFORMED_TID = "malformed_tid ";
protected static final String PROPAGATION_ERROR_INCONSISTENT_TID = "inconsistent_tid ";
protected static final TagKey UPSTREAM_SERVICES_DEPRECATED_TAG = TagKey.from("upstream_services");
Expand Down Expand Up @@ -49,6 +50,11 @@ static String headerValue(PTagsCodec codec, PTags ptags) {
if (ptags.getDebugPropagation() != null) {
size = codec.appendTag(sb, DEBUG_TAG, TagValue.from(ptags.getDebugPropagation()), size);
}
if (ptags.getKnuthSamplingRateTagValue() != null) {
size =
codec.appendTag(
sb, KNUTH_SAMPLING_RATE_TAG, ptags.getKnuthSamplingRateTagValue(), size);
}
Iterator<TagElement> it = ptags.getTagPairs().iterator();
while (it.hasNext() && !codec.isTooLarge(sb, size)) {
TagElement tagKey = it.next();
Expand Down Expand Up @@ -103,6 +109,11 @@ static void fillTagMap(PTags propagationTags, Map<String, String> tagMap) {
tagMap.put(
DEBUG_TAG.forType(Encoding.DATADOG).toString(), propagationTags.getDebugPropagation());
}
if (propagationTags.getKnuthSamplingRateTagValue() != null) {
tagMap.put(
KNUTH_SAMPLING_RATE_TAG.forType(Encoding.DATADOG).toString(),
propagationTags.getKnuthSamplingRateTagValue().forType(Encoding.DATADOG).toString());
}
if (propagationTags.getTraceIdHighOrderBitsHexTagValue() != null) {
tagMap.put(
TRACE_ID_TAG.forType(Encoding.DATADOG).toString(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static datadog.trace.core.propagation.PropagationTags.HeaderType.DATADOG;
import static datadog.trace.core.propagation.PropagationTags.HeaderType.W3C;
import static datadog.trace.core.propagation.ptags.PTagsCodec.DECISION_MAKER_TAG;
import static datadog.trace.core.propagation.ptags.PTagsCodec.KNUTH_SAMPLING_RATE_TAG;
import static datadog.trace.core.propagation.ptags.PTagsCodec.TRACE_ID_TAG;
import static datadog.trace.core.propagation.ptags.PTagsCodec.TRACE_SOURCE_TAG;

Expand Down Expand Up @@ -90,6 +91,9 @@ static class PTags extends PropagationTags {
private volatile int traceSource;
private volatile String debugPropagation;

// extracted Knuth sampling rate tag for easier updates
private volatile TagValue knuthSamplingRateTagValue;
Comment thread
bm1549 marked this conversation as resolved.

// xDatadogTagsSize of the tagPairs, does not include the decision maker tag
private volatile int xDatadogTagsSize = -1;

Expand Down Expand Up @@ -265,6 +269,20 @@ public String getDebugPropagation() {
return debugPropagation;
}

@Override
public void updateKnuthSamplingRate(String rate) {
TagValue newValue = rate == null ? null : TagValue.from(rate);
if (!java.util.Objects.equals(knuthSamplingRateTagValue, newValue)) {
clearCachedHeader(DATADOG);
clearCachedHeader(W3C);
}
knuthSamplingRateTagValue = newValue;
}

TagValue getKnuthSamplingRateTagValue() {
return knuthSamplingRateTagValue;
}

@Override
public int getSamplingPriority() {
return samplingPriority;
Expand Down Expand Up @@ -390,6 +408,9 @@ int getXDatadogTagsSize() {
size = PTagsCodec.calcXDatadogTagsSize(getTagPairs());
size = PTagsCodec.calcXDatadogTagsSize(size, DECISION_MAKER_TAG, decisionMakerTagValue);
size = PTagsCodec.calcXDatadogTagsSize(size, TRACE_ID_TAG, traceIdHighOrderBitsHexTagValue);
size =
PTagsCodec.calcXDatadogTagsSize(
size, KNUTH_SAMPLING_RATE_TAG, knuthSamplingRateTagValue);
int currentProductTraceSource = traceSource;
if (currentProductTraceSource != ProductTraceSource.UNSET) {
size =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ class DDAgentApiTest extends DDCoreSpecification {
[[buildSpan(1L, "service.name", "my-service", PropagationTags.factory().fromHeaderValue(PropagationTags.HeaderType.DATADOG, "_dd.p.usr=123"))]] | [[new TreeMap<>([
"duration" : 10,
"error" : 0,
"meta" : ["thread.name": Thread.currentThread().getName(), "_dd.p.usr": "123", "_dd.p.dm": "-1", "_dd.svc_src" : "m"] +
"meta" : ["thread.name": Thread.currentThread().getName(), "_dd.p.usr": "123", "_dd.p.dm": "-1", "_dd.p.ksr": "1", "_dd.svc_src" : "m"] +
(Config.get().isExperimentalPropagateProcessTagsEnabled() ? ["_dd.tags.process" : ProcessTags.getTagsForSerialization().toString()] : []),
"metrics" : [
(DDSpanContext.PRIORITY_SAMPLING_KEY) : 1,
Expand All @@ -185,7 +185,7 @@ class DDAgentApiTest extends DDCoreSpecification {
[[buildSpan(100L, "resource.name", "my-resource", PropagationTags.factory().fromHeaderValue(PropagationTags.HeaderType.DATADOG, "_dd.p.usr=123"))]] | [[new TreeMap<>([
"duration" : 10,
"error" : 0,
"meta" : ["thread.name": Thread.currentThread().getName(), "_dd.p.usr": "123", "_dd.p.dm": "-1"] +
"meta" : ["thread.name": Thread.currentThread().getName(), "_dd.p.usr": "123", "_dd.p.dm": "-1", "_dd.p.ksr": "1"] +
(Config.get().isExperimentalPropagateProcessTagsEnabled() ? ["_dd.tags.process" : ProcessTags.getTagsForSerialization().toString()] : []),
"metrics" : [
(DDSpanContext.PRIORITY_SAMPLING_KEY) : 1,
Expand Down
Loading
Loading