|
| 1 | +package datadog.trace.api.telemetry; |
| 2 | + |
| 3 | +import static datadog.trace.api.telemetry.MetricCollector.RAW_QUEUE_SIZE; |
| 4 | +import static org.junit.jupiter.api.Assertions.assertEquals; |
| 5 | +import static org.junit.jupiter.api.Assertions.assertTrue; |
| 6 | + |
| 7 | +import datadog.trace.api.metrics.SpanMetricRegistryImpl; |
| 8 | +import datadog.trace.api.telemetry.CoreMetricCollector.CoreMetric; |
| 9 | +import java.util.Collection; |
| 10 | +import org.junit.jupiter.api.Test; |
| 11 | + |
| 12 | +/** |
| 13 | + * Regression guard for the queue-full delta-loss bug in {@link |
| 14 | + * CoreMetricCollector#prepareMetrics()}. |
| 15 | + * |
| 16 | + * <p>Before the fix, the drain read a counter with {@code getValueAndReset()} — which clears the |
| 17 | + * counter's delta baseline — and only then tried to {@code offer()} it onto a fixed-capacity queue. |
| 18 | + * When the queue was already full the offer failed, so that already-reset delta was lost |
| 19 | + * permanently rather than being carried to the next cycle. The fix checks {@code |
| 20 | + * remainingCapacity()} before reading each counter and stops early, leaving the untouched counters |
| 21 | + * for next time. |
| 22 | + */ |
| 23 | +class CoreMetricCollectorQueueFullTest { |
| 24 | + |
| 25 | + /** |
| 26 | + * Fills the drain past its queue capacity and verifies that every counter's delta is reported |
| 27 | + * exactly once across as many cycles as it takes to empty — none dropped while the queue is full. |
| 28 | + * |
| 29 | + * <p>{@link CoreMetricCollector} is a singleton over the global {@link SpanMetricRegistryImpl}, |
| 30 | + * so this test can't assume a clean registry. It stays robust by tagging its own instrumentations |
| 31 | + * with a unique prefix and counting only those, and it drains cycle-by-cycle until the queue |
| 32 | + * yields nothing at all (so it never terminates early just because a cycle happened to surface |
| 33 | + * only unrelated counters left behind by other tests). With the fix the sum of our own metrics |
| 34 | + * equals the number we created; with the bug the ones skipped while the queue was full are lost, |
| 35 | + * so the sum comes up short. |
| 36 | + */ |
| 37 | + @Test |
| 38 | + void skippedCountersRetainDeltaAcrossCycles() { |
| 39 | + SpanMetricRegistryImpl registry = SpanMetricRegistryImpl.getInstance(); |
| 40 | + CoreMetricCollector collector = CoreMetricCollector.getInstance(); |
| 41 | + |
| 42 | + // Discard anything left in the queue by other tests sharing the singleton. |
| 43 | + collector.drain(); |
| 44 | + |
| 45 | + // Unique instrumentation-name prefix so we count only the instrumentations this test creates, |
| 46 | + // regardless of whatever else has already polluted the shared registry. CoreMetricCollector |
| 47 | + // emits these as "integration_name:<name>" tags. |
| 48 | + String namePrefix = "queue-full-test-" + System.nanoTime() + "-"; |
| 49 | + String tagPrefix = "integration_name:" + namePrefix; |
| 50 | + |
| 51 | + // Create more non-zero counters than the queue can hold so at least one collection cycle |
| 52 | + // overflows. A single onSpanCreated() per instrumentation leaves exactly one non-zero counter |
| 53 | + // (spans_created) each; the rest read as 0 and are skipped. |
| 54 | + int created = RAW_QUEUE_SIZE + 200; |
| 55 | + for (int i = 0; i < created; i++) { |
| 56 | + registry.get(namePrefix + i).onSpanCreated(); |
| 57 | + } |
| 58 | + |
| 59 | + // Drain cycle-by-cycle until the queue is fully emptied, summing how many of our own metrics |
| 60 | + // surface. Guard the loop so a bug that never converges fails loudly instead of hanging. |
| 61 | + int seen = 0; |
| 62 | + int cycles = 0; |
| 63 | + while (true) { |
| 64 | + collector.prepareMetrics(); |
| 65 | + Collection<CoreMetric> drained = collector.drain(); |
| 66 | + if (drained.isEmpty()) { |
| 67 | + break; |
| 68 | + } |
| 69 | + for (CoreMetric metric : drained) { |
| 70 | + if (isOurs(metric, tagPrefix)) { |
| 71 | + seen++; |
| 72 | + } |
| 73 | + } |
| 74 | + assertTrue(++cycles < 20, "drain did not converge; possible lost/looping delta"); |
| 75 | + } |
| 76 | + |
| 77 | + assertEquals( |
| 78 | + created, |
| 79 | + seen, |
| 80 | + "every counter's delta must be reported exactly once; a full queue must not drop any"); |
| 81 | + } |
| 82 | + |
| 83 | + private static boolean isOurs(CoreMetric metric, String tagPrefix) { |
| 84 | + for (String tag : metric.tags) { |
| 85 | + if (tag.startsWith(tagPrefix)) { |
| 86 | + return true; |
| 87 | + } |
| 88 | + } |
| 89 | + return false; |
| 90 | + } |
| 91 | +} |
0 commit comments