Skip to content

Commit 293ce08

Browse files
committed
Merge branch 'master' into sarahchen6/implement-48h-cooldown-for-gradle-dependencies
2 parents 2708573 + 7e99f00 commit 293ce08

16 files changed

Lines changed: 347 additions & 108 deletions

File tree

.claude/skills/migrate-groovy-to-java/SKILL.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ Migrate test Groovy files to Java using JUnit 5
1212
2. Convert Groovy files to Java using JUnit 5
1313
3. Make sure the tests are still passing after migration and that the test count has not changed
1414
4. Remove Groovy files
15-
5. Add the migrated module path(s) to `.github/g2j-migrated-modules.txt`
1615

1716
When converting Groovy code to Java code, make sure that:
1817
- The Java code generated is compatible with JDK 8

.gitlab-ci.yml

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ workflow:
7777
auto_cancel:
7878
on_new_commit: interruptible
7979
rules:
80+
# skip gitlab pipeline for github merge queue - we are currently using the datadog merge queue
81+
- if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue\//'
82+
when: never
8083
- if: '$CI_COMMIT_BRANCH == "master"'
8184
auto_cancel:
8285
on_new_commit: none
@@ -231,15 +234,9 @@ default:
231234
- export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
232235
- export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
233236
- *normalize_node_index
234-
# for weird reasons, Gradle will always "chmod 700" the .gradle folder
235-
# with Gitlab caching, .gradle is always owned by root and thus Gradle's chmod invocation fails
236-
# This dance is a hack to have .gradle owned by the Gitlab runner user
237-
- gitlab_section_start "gradle-dance" "Fix .gradle directory permissions"
238-
- cp -r .gradle .gradle-copy
239-
- rm -rf .gradle
240-
- mv .gradle-copy .gradle
241-
- ls -la
242-
- gitlab_section_end "gradle-dance"
237+
# GitLab's cache helper restores .gradle as root, but we run as non-root-user (uid 1001),
238+
# and Gradle does `chmod 700 .gradle` on startup which requires user ownership.
239+
- sudo chown -R 1001:1001 .gradle
243240
after_script:
244241
- *cgroup_info
245242
- *container_info

dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/ExtensionFinder.java

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static datadog.opentelemetry.tooling.OtelExtensionHandler.OPENTELEMETRY;
44
import static datadog.trace.agent.tooling.ExtensionHandler.DATADOG;
55

6+
import datadog.trace.api.telemetry.OtelSpiCollector;
67
import de.thetaphi.forbiddenapis.SuppressForbidden;
78
import java.io.File;
89
import java.io.FileNotFoundException;
@@ -26,6 +27,33 @@ public final class ExtensionFinder {
2627

2728
private static final ExtensionHandler[] handlers = {OPENTELEMETRY, DATADOG};
2829

30+
private static final String EXTENSIONS_PATH_SOURCE = "extensions_path";
31+
private static final String SERVICES_PREFIX = "META-INF/services/";
32+
33+
private static final String[] OTEL_SPI_FQNS = {
34+
"io.opentelemetry.context.ContextStorageProvider",
35+
"io.opentelemetry.exporter.internal.compression.CompressorProvider",
36+
"io.opentelemetry.exporter.internal.grpc.GrpcSenderProvider",
37+
"io.opentelemetry.exporter.internal.http.HttpSenderProvider",
38+
"io.opentelemetry.javaagent.extension.AgentListener",
39+
"io.opentelemetry.javaagent.extension.ignore.IgnoredTypesConfigurer",
40+
"io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule",
41+
"io.opentelemetry.javaagent.tooling.BeforeAgentListener",
42+
"io.opentelemetry.javaagent.tooling.LoggingCustomizer",
43+
"io.opentelemetry.javaagent.tooling.bootstrap.BootstrapPackagesConfigurer",
44+
"io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider",
45+
"io.opentelemetry.sdk.autoconfigure.spi.AutoConfigureListener",
46+
"io.opentelemetry.sdk.autoconfigure.spi.ConditionalResourceProvider",
47+
"io.opentelemetry.sdk.autoconfigure.spi.ConfigurablePropagatorProvider",
48+
"io.opentelemetry.sdk.autoconfigure.spi.ResourceProvider",
49+
"io.opentelemetry.sdk.autoconfigure.spi.internal.ComponentProvider",
50+
"io.opentelemetry.sdk.autoconfigure.spi.logs.ConfigurableLogRecordExporterProvider",
51+
"io.opentelemetry.sdk.autoconfigure.spi.metrics.ConfigurableMetricExporterProvider",
52+
"io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSamplerProvider",
53+
"io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider",
54+
"io.opentelemetry.sdk.autoconfigure.spi.traces.SpanExporterCustomizer",
55+
};
56+
2957
/**
3058
* Discovers extensions on the configured path and creates a classloader for each extension.
3159
* Registers the combined classloader with {@link Utils#setExtendedClassLoader(ClassLoader)}.
@@ -40,6 +68,7 @@ public static boolean findExtensions(String extensionsPath, Class<?>... extensio
4068
String[] descriptors = descriptors(extensionTypes);
4169

4270
for (JarFile jar : findExtensionJars(extensionsPath)) {
71+
recordOtelSpiTelemetry(jar);
4372
URL extensionURL = findExtensionURL(jar, descriptors);
4473
if (null != extensionURL) {
4574
log.debug("Found extension jar {}", jar.getName());
@@ -60,6 +89,15 @@ public static boolean findExtensions(String extensionsPath, Class<?>... extensio
6089
return !classLoaders.isEmpty();
6190
}
6291

92+
/** Reports telemetry for any tracked OpenTelemetry SPI service descriptors present in the jar. */
93+
static void recordOtelSpiTelemetry(JarFile jar) {
94+
for (String fqn : OTEL_SPI_FQNS) {
95+
if (null != jar.getJarEntry(SERVICES_PREFIX + fqn)) {
96+
OtelSpiCollector.getInstance().recordSpiDetected(fqn, EXTENSIONS_PATH_SOURCE);
97+
}
98+
}
99+
}
100+
63101
/** Closes jar resources from the extension path which did not contain any extensions. */
64102
private static void close(List<JarFile> unusedJars) {
65103
for (JarFile jar : unusedJars) {
Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
package datadog.trace.agent.tooling;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static org.junit.jupiter.api.Assertions.assertTrue;
5+
6+
import datadog.trace.api.telemetry.OtelSpiCollector;
7+
import java.io.IOException;
8+
import java.io.OutputStream;
9+
import java.nio.file.Files;
10+
import java.nio.file.Path;
11+
import java.util.Collection;
12+
import java.util.HashSet;
13+
import java.util.Set;
14+
import java.util.jar.JarEntry;
15+
import java.util.jar.JarFile;
16+
import java.util.jar.JarOutputStream;
17+
import org.junit.jupiter.api.BeforeEach;
18+
import org.junit.jupiter.api.Test;
19+
import org.junit.jupiter.api.io.TempDir;
20+
21+
public class ExtensionFinderTest {
22+
23+
private static final String AUTOCONFIGURE_PROPAGATOR =
24+
"io.opentelemetry.sdk.autoconfigure.spi.ConfigurablePropagatorProvider";
25+
private static final String AUTOCONFIGURE_RESOURCE =
26+
"io.opentelemetry.sdk.autoconfigure.spi.ResourceProvider";
27+
private static final String AUTOCONFIGURE_SAMPLER =
28+
"io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSamplerProvider";
29+
private static final String AUTOCONFIGURE_EXPORTER =
30+
"io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider";
31+
private static final String JAVAAGENT_INSTRUMENTATION_MODULE =
32+
"io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule";
33+
private static final String JAVAAGENT_AGENT_LISTENER =
34+
"io.opentelemetry.javaagent.extension.AgentListener";
35+
private static final String SHADED_AUTOCONFIGURE_SAMPLER =
36+
"io.opentelemetry.javaagent.shaded.io.opentelemetry.sdk.autoconfigure.spi.ConfigurableSamplerProvider";
37+
38+
private final OtelSpiCollector collector = OtelSpiCollector.getInstance();
39+
40+
@BeforeEach
41+
public void clearCollector() {
42+
collector.drain();
43+
}
44+
45+
@Test
46+
public void singleOtelSpiIsReported(@TempDir Path tempDir) throws IOException {
47+
Path jarPath = buildJar(tempDir, "ext.jar", AUTOCONFIGURE_PROPAGATOR);
48+
49+
try (JarFile jar = new JarFile(jarPath.toFile(), false)) {
50+
ExtensionFinder.recordOtelSpiTelemetry(jar);
51+
}
52+
53+
Collection<OtelSpiCollector.OtelSpiMetric> drained = collector.drain();
54+
assertEquals(1, drained.size());
55+
OtelSpiCollector.OtelSpiMetric metric = drained.iterator().next();
56+
assertEquals("otel.spi.detected", metric.metricName);
57+
assertTrue(metric.tags.contains("spi_class:" + AUTOCONFIGURE_PROPAGATOR));
58+
assertTrue(metric.tags.contains("source:extensions_path"));
59+
}
60+
61+
@Test
62+
public void allFourAutoconfigureSpisAreReported(@TempDir Path tempDir) throws IOException {
63+
Path jarPath =
64+
buildJar(
65+
tempDir,
66+
"ext.jar",
67+
AUTOCONFIGURE_PROPAGATOR,
68+
AUTOCONFIGURE_RESOURCE,
69+
AUTOCONFIGURE_SAMPLER,
70+
AUTOCONFIGURE_EXPORTER);
71+
72+
try (JarFile jar = new JarFile(jarPath.toFile(), false)) {
73+
ExtensionFinder.recordOtelSpiTelemetry(jar);
74+
}
75+
76+
assertEquals(
77+
new HashSet<>(
78+
java.util.Arrays.asList(
79+
AUTOCONFIGURE_PROPAGATOR,
80+
AUTOCONFIGURE_RESOURCE,
81+
AUTOCONFIGURE_SAMPLER,
82+
AUTOCONFIGURE_EXPORTER)),
83+
reportedFqns(collector.drain()));
84+
}
85+
86+
@Test
87+
public void javaagentExtensionSpisAreReported(@TempDir Path tempDir) throws IOException {
88+
Path jarPath =
89+
buildJar(tempDir, "ext.jar", JAVAAGENT_INSTRUMENTATION_MODULE, JAVAAGENT_AGENT_LISTENER);
90+
91+
try (JarFile jar = new JarFile(jarPath.toFile(), false)) {
92+
ExtensionFinder.recordOtelSpiTelemetry(jar);
93+
}
94+
95+
assertEquals(
96+
new HashSet<>(
97+
java.util.Arrays.asList(JAVAAGENT_INSTRUMENTATION_MODULE, JAVAAGENT_AGENT_LISTENER)),
98+
reportedFqns(collector.drain()));
99+
}
100+
101+
@Test
102+
public void nonOtelSpiIsIgnored(@TempDir Path tempDir) throws IOException {
103+
Path jarPath =
104+
buildJar(
105+
tempDir,
106+
"ext.jar",
107+
"com.example.MyService",
108+
"org.springframework.context.ApplicationContextInitializer",
109+
"java.sql.Driver");
110+
111+
try (JarFile jar = new JarFile(jarPath.toFile(), false)) {
112+
ExtensionFinder.recordOtelSpiTelemetry(jar);
113+
}
114+
115+
assertEquals(0, collector.drain().size());
116+
}
117+
118+
@Test
119+
public void jarWithoutAnyServiceDescriptorsEmitsNothing(@TempDir Path tempDir)
120+
throws IOException {
121+
Path jarPath = tempDir.resolve("empty.jar");
122+
try (JarOutputStream jos = new JarOutputStream(Files.newOutputStream(jarPath))) {
123+
jos.putNextEntry(new JarEntry("README.txt"));
124+
jos.write("not an extension".getBytes());
125+
jos.closeEntry();
126+
}
127+
128+
try (JarFile jar = new JarFile(jarPath.toFile(), false)) {
129+
ExtensionFinder.recordOtelSpiTelemetry(jar);
130+
}
131+
132+
assertEquals(0, collector.drain().size());
133+
}
134+
135+
@Test
136+
public void mixedOtelAndNonOtelReportsOnlyOtel(@TempDir Path tempDir) throws IOException {
137+
Path jarPath =
138+
buildJar(
139+
tempDir,
140+
"ext.jar",
141+
AUTOCONFIGURE_PROPAGATOR,
142+
"com.example.MyService",
143+
JAVAAGENT_AGENT_LISTENER,
144+
"java.sql.Driver");
145+
146+
try (JarFile jar = new JarFile(jarPath.toFile(), false)) {
147+
ExtensionFinder.recordOtelSpiTelemetry(jar);
148+
}
149+
150+
assertEquals(
151+
new HashSet<>(java.util.Arrays.asList(AUTOCONFIGURE_PROPAGATOR, JAVAAGENT_AGENT_LISTENER)),
152+
reportedFqns(collector.drain()));
153+
}
154+
155+
private static Set<String> reportedFqns(Collection<OtelSpiCollector.OtelSpiMetric> drained) {
156+
Set<String> fqns = new HashSet<>();
157+
for (OtelSpiCollector.OtelSpiMetric metric : drained) {
158+
for (String tag : metric.tags) {
159+
if (tag.startsWith("spi_class:")) {
160+
fqns.add(tag.substring("spi_class:".length()));
161+
}
162+
}
163+
}
164+
return fqns;
165+
}
166+
167+
/** Builds a jar with empty {@code META-INF/services/<fqn>} entries for each given FQN. */
168+
private static Path buildJar(Path dir, String name, String... serviceFqns) throws IOException {
169+
Path jarPath = dir.resolve(name);
170+
try (OutputStream out = Files.newOutputStream(jarPath);
171+
JarOutputStream jos = new JarOutputStream(out)) {
172+
for (String fqn : serviceFqns) {
173+
jos.putNextEntry(new JarEntry("META-INF/services/" + fqn));
174+
jos.closeEntry();
175+
}
176+
}
177+
return jarPath;
178+
}
179+
}

dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/logs/data/OtelLogRecordProcessor.java

Lines changed: 55 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@
1414
import java.util.Queue;
1515
import java.util.WeakHashMap;
1616
import java.util.concurrent.ArrayBlockingQueue;
17+
import java.util.concurrent.BlockingQueue;
18+
import java.util.concurrent.TimeUnit;
1719
import java.util.function.BiConsumer;
1820

1921
/** Processes log records, grouping them by instrumentation scope. */
2022
public final class OtelLogRecordProcessor {
21-
private static final int MAX_QUEUE_SIZE = Config.get().getLogsOtelQueueSize();
22-
private static final int MAX_BATCH_SIZE = Config.get().getLogsOtelBatchSize();
2323

2424
private static final Comparator<OtlpLogRecord> BY_SCOPE =
2525
Comparator.comparing(o -> o.instrumentationScope);
@@ -29,18 +29,67 @@ public final class OtelLogRecordProcessor {
2929

3030
public static final OtelLogRecordProcessor INSTANCE = new OtelLogRecordProcessor();
3131

32-
private final Queue<OtlpLogRecord> queue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE);
32+
private final int maxQueueSize = Config.get().getLogsOtelQueueSize();
33+
private final int maxBatchSize = Config.get().getLogsOtelBatchSize();
34+
35+
private final Queue<OtlpLogRecord> queue = new ArrayBlockingQueue<>(maxQueueSize);
36+
37+
private final BlockingQueue<Boolean> logsReady = new ArrayBlockingQueue<>(1);
38+
private volatile int logsNeeded = Integer.MAX_VALUE;
3339

3440
public void addLog(OtlpLogRecord logRecord) {
35-
queue.offer(logRecord);
41+
if (queue.offer(logRecord)) {
42+
// report when we have enough logs for the collector's needs
43+
if (queue.size() >= logsNeeded) {
44+
logsReady.offer(true);
45+
}
46+
}
3647
}
3748

38-
public void collectLogs(OtlpLogsVisitor visitor) {
49+
public void waitForLogs(OtlpLogsVisitor visitor, int intervalMillis) {
50+
long nextExportNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(intervalMillis);
51+
List<OtlpLogRecord> batch = new ArrayList<>(maxBatchSize);
52+
int batchSize = 0;
53+
54+
while (true) {
55+
56+
// attempt to collect enough logs to complete the batch
57+
OtlpLogRecord record;
58+
while (batchSize < maxBatchSize && (record = queue.poll()) != null) {
59+
batch.add(record);
60+
batchSize++;
61+
}
62+
63+
// bail out if we have enough logs, or the interval has expired
64+
long waitNanos;
65+
if (batchSize >= maxBatchSize || (waitNanos = nextExportNanos - System.nanoTime()) <= 0) {
66+
break;
67+
}
68+
69+
logsNeeded = maxBatchSize - batchSize; // declare what we need and wait
70+
try {
71+
if (queue.isEmpty()) {
72+
logsReady.poll(waitNanos, TimeUnit.NANOSECONDS);
73+
}
74+
} catch (InterruptedException ignore) {
75+
// don't set interrupt flag as we might then busy-loop, just return batch as-is
76+
break;
77+
} finally {
78+
logsNeeded = Integer.MAX_VALUE;
79+
}
80+
}
81+
82+
visitBatch(visitor, batch); // send what we have for this interval
83+
}
84+
85+
private static void visitBatch(OtlpLogsVisitor visitor, List<OtlpLogRecord> batch) {
86+
batch.sort(BY_SCOPE);
87+
3988
OtlpScopedLogsVisitor scopedVisitor = null;
4089
OtelInstrumentationScope currentScope = null;
4190
BiConsumer<Map<?, ?>, OtlpAttributeVisitor> attributesReader = null;
4291
ClassLoader attributesClassLoader = null;
43-
for (OtlpLogRecord logRecord : batchByScope()) {
92+
for (OtlpLogRecord logRecord : batch) {
4493
if (logRecord.instrumentationScope != currentScope) {
4594
currentScope = logRecord.instrumentationScope;
4695
scopedVisitor = visitor.visitScopedLogs(currentScope);
@@ -70,20 +119,4 @@ public static void registerAttributeReader(
70119
ClassLoader cl, BiConsumer<Map<?, ?>, OtlpAttributeVisitor> reader) {
71120
ATTRIBUTE_READERS.put(cl, reader);
72121
}
73-
74-
private List<OtlpLogRecord> batchByScope() {
75-
// capture expected batch size; records emitted after here go into next batch
76-
int batchSize = Math.min(queue.size(), MAX_BATCH_SIZE);
77-
List<OtlpLogRecord> batch = new ArrayList<>(batchSize);
78-
for (int i = 0; i < batchSize; i++) {
79-
OtlpLogRecord logRecord = queue.poll();
80-
if (logRecord != null) {
81-
batch.add(logRecord);
82-
} else {
83-
break; // should not happen unless another thread is also batching records
84-
}
85-
}
86-
batch.sort(BY_SCOPE);
87-
return batch;
88-
}
89122
}

0 commit comments

Comments
 (0)