Skip to content

Commit b55cb17

Browse files
dougqhclaude
andcommitted
Add regression test for queue-full delta loss
Fills CoreMetricCollector's drain past its ArrayBlockingQueue capacity and verifies every counter's delta is reported exactly once across cycles. Fails against the pre-fix code (counters read-and-reset while the queue was full were dropped) and passes with the remainingCapacity() guard. Robust to the shared SpanMetricRegistryImpl singleton: tags its own instrumentations with a unique prefix, counts only those, and drains cycle-by-cycle until the queue is empty. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 50247ce commit b55cb17

1 file changed

Lines changed: 91 additions & 0 deletions

File tree

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package datadog.trace.api.telemetry;
2+
3+
import static datadog.trace.api.telemetry.MetricCollector.RAW_QUEUE_SIZE;
4+
import static org.junit.jupiter.api.Assertions.assertEquals;
5+
import static org.junit.jupiter.api.Assertions.assertTrue;
6+
7+
import datadog.trace.api.metrics.SpanMetricRegistryImpl;
8+
import datadog.trace.api.telemetry.CoreMetricCollector.CoreMetric;
9+
import java.util.Collection;
10+
import org.junit.jupiter.api.Test;
11+
12+
/**
13+
* Regression guard for the queue-full delta-loss bug in {@link
14+
* CoreMetricCollector#prepareMetrics()}.
15+
*
16+
* <p>Before the fix, the drain read a counter with {@code getValueAndReset()} — which clears the
17+
* counter's delta baseline — and only then tried to {@code offer()} it onto a fixed-capacity queue.
18+
* When the queue was already full the offer failed, so that already-reset delta was lost
19+
* permanently rather than being carried to the next cycle. The fix checks {@code
20+
* remainingCapacity()} before reading each counter and stops early, leaving the untouched counters
21+
* for next time.
22+
*/
23+
class CoreMetricCollectorQueueFullTest {
24+
25+
/**
26+
* Fills the drain past its queue capacity and verifies that every counter's delta is reported
27+
* exactly once across as many cycles as it takes to empty — none dropped while the queue is full.
28+
*
29+
* <p>{@link CoreMetricCollector} is a singleton over the global {@link SpanMetricRegistryImpl},
30+
* so this test can't assume a clean registry. It stays robust by tagging its own instrumentations
31+
* with a unique prefix and counting only those, and it drains cycle-by-cycle until the queue
32+
* yields nothing at all (so it never terminates early just because a cycle happened to surface
33+
* only unrelated counters left behind by other tests). With the fix the sum of our own metrics
34+
* equals the number we created; with the bug the ones skipped while the queue was full are lost,
35+
* so the sum comes up short.
36+
*/
37+
@Test
38+
void skippedCountersRetainDeltaAcrossCycles() {
39+
SpanMetricRegistryImpl registry = SpanMetricRegistryImpl.getInstance();
40+
CoreMetricCollector collector = CoreMetricCollector.getInstance();
41+
42+
// Discard anything left in the queue by other tests sharing the singleton.
43+
collector.drain();
44+
45+
// Unique instrumentation-name prefix so we count only the instrumentations this test creates,
46+
// regardless of whatever else has already polluted the shared registry. CoreMetricCollector
47+
// emits these as "integration_name:<name>" tags.
48+
String namePrefix = "queue-full-test-" + System.nanoTime() + "-";
49+
String tagPrefix = "integration_name:" + namePrefix;
50+
51+
// Create more non-zero counters than the queue can hold so at least one collection cycle
52+
// overflows. A single onSpanCreated() per instrumentation leaves exactly one non-zero counter
53+
// (spans_created) each; the rest read as 0 and are skipped.
54+
int created = RAW_QUEUE_SIZE + 200;
55+
for (int i = 0; i < created; i++) {
56+
registry.get(namePrefix + i).onSpanCreated();
57+
}
58+
59+
// Drain cycle-by-cycle until the queue is fully emptied, summing how many of our own metrics
60+
// surface. Guard the loop so a bug that never converges fails loudly instead of hanging.
61+
int seen = 0;
62+
int cycles = 0;
63+
while (true) {
64+
collector.prepareMetrics();
65+
Collection<CoreMetric> drained = collector.drain();
66+
if (drained.isEmpty()) {
67+
break;
68+
}
69+
for (CoreMetric metric : drained) {
70+
if (isOurs(metric, tagPrefix)) {
71+
seen++;
72+
}
73+
}
74+
assertTrue(++cycles < 20, "drain did not converge; possible lost/looping delta");
75+
}
76+
77+
assertEquals(
78+
created,
79+
seen,
80+
"every counter's delta must be reported exactly once; a full queue must not drop any");
81+
}
82+
83+
private static boolean isOurs(CoreMetric metric, String tagPrefix) {
84+
for (String tag : metric.tags) {
85+
if (tag.startsWith(tagPrefix)) {
86+
return true;
87+
}
88+
}
89+
return false;
90+
}
91+
}

0 commit comments

Comments
 (0)