diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 00b54848832..29afea7ba67 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -3,7 +3,6 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_STARTUP_LOGS_ENABLED; import static datadog.trace.api.Platform.isJavaVersionAtLeast; import static datadog.trace.api.Platform.isOracleJDK8; -import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; import static datadog.trace.bootstrap.Library.WILDFLY; import static datadog.trace.bootstrap.Library.detectLibraries; import static datadog.trace.util.AgentThreadFactory.AgentThread.JMX_STARTUP; @@ -45,7 +44,6 @@ import datadog.trace.util.AgentTaskScheduler; import datadog.trace.util.AgentThreadFactory.AgentThread; import datadog.trace.util.throwable.FatalAgentMisconfigurationError; -import de.thetaphi.forbiddenapis.SuppressForbidden; import java.lang.instrument.Instrumentation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -291,8 +289,6 @@ public static void start( codeOriginEnabled = isFeatureEnabled(AgentFeature.CODE_ORIGIN); agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); - patchJPSAccess(inst); - if (profilingEnabled) { if (!isOracleJDK8()) { // Profiling agent startup code is written in a way to allow `startProfilingAgent` be called @@ -422,23 +418,6 @@ private static void injectAgentArgsConfig(String agentArgs) { } } - @SuppressForbidden - public static void patchJPSAccess(Instrumentation inst) { - if (Platform.isJavaVersionAtLeast(9)) { - // Unclear if supported for J9, may need to revisit - try { - Class.forName("datadog.trace.util.JPMSJPSAccess") - .getMethod("patchModuleAccess", Instrumentation.class) - .invoke(null, inst); - } catch (Exception e) { - log.debug( - SEND_TELEMETRY, - "Failed to patch module access for jvmstat and Java version " - + Platform.getRuntimeVersion()); - } - } - } - public static void shutdown(final boolean sync) { StaticEventLogger.end("Agent"); StaticEventLogger.stop(); diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 8592f244c5f..e60521abbc4 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -83,6 +83,16 @@ ext.generalShadowJarConfig = { exclude '**/com/kenai/jffi/Init.class' relocate('com.kenai.jffi.Init', 'com.kenai.jffi.PatchInit') + // Minimize and relocate the airlift compressor dependency for ZSTD + exclude '**/io/airlift/compress/bzip2/**' + exclude '**/io/airlift/compress/deflate/**' + exclude '**/io/airlift/compress/gzip/**' + exclude '**/io/airlift/compress/hadoop/**' + exclude '**/io/airlift/compress/lz4/**' + exclude '**/io/airlift/compress/lzo/**' + exclude '**/io/airlift/compress/snappy/**' + relocate 'io.airlift', 'datadog.io.airlift' + final String projectName = "${project.name}" // Prevents conflict with other instances, but doesn't relocate instrumentation diff --git a/dd-smoke-tests/profiling-integration-tests/build.gradle b/dd-smoke-tests/profiling-integration-tests/build.gradle index 553a139aae1..c01e430292f 100644 --- a/dd-smoke-tests/profiling-integration-tests/build.gradle +++ b/dd-smoke-tests/profiling-integration-tests/build.gradle @@ -20,7 +20,8 @@ dependencies { implementation project(':dd-trace-api') api project(':dd-trace-ot') implementation 'org.apache.commons:commons-math3:3.6.1' - implementation 'org.lz4:lz4-java:1.8.0' + implementation libs.lz4 + implementation libs.aircompressor implementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation project(':dd-smoke-tests') @@ -28,6 +29,7 @@ dependencies { testImplementation libs.bundles.junit5 testImplementation libs.bundles.mockito testImplementation libs.bundles.jmc + testImplementation libs.aircompressor testImplementation(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.10') } diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java index 6adf84ddacc..4c26b072d48 100644 --- a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java @@ -17,9 +17,11 @@ import datadog.trace.api.Platform; import datadog.trace.api.config.ProfilingConfig; import delight.fileupload.FileUpload; +import io.airlift.compress.zstd.ZstdInputStream; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.InetAddress; import java.nio.file.Files; import java.nio.file.Path; @@ -150,7 +152,7 @@ void teardown() throws Exception { } @Test - @DisplayName("Test continuous recording - no jmx delay, no jmethodid cache") + @DisplayName("Test continuous recording - no jmx delay, default compression") public void testContinuousRecording_no_jmx_delay(final TestInfo testInfo) throws Exception { testWithRetry( () -> @@ -161,7 +163,7 @@ public void testContinuousRecording_no_jmx_delay(final TestInfo testInfo) throws } @Test - @DisplayName("Test continuous recording - no jmx delay, jmethodid cache") + @DisplayName("Test continuous recording - no jmx delay, zstd compression") public void testContinuousRecording_no_jmx_delay_jmethodid_cache(final TestInfo testInfo) throws Exception { testWithRetry( @@ -173,7 +175,7 @@ public void testContinuousRecording_no_jmx_delay_jmethodid_cache(final TestInfo } @Test - @DisplayName("Test continuous recording - 1 sec jmx delay, no jmethodid cache") + @DisplayName("Test continuous recording - 1 sec jmx delay, default compression") public void testContinuousRecording(final TestInfo testInfo) throws Exception { testWithRetry( () -> @@ -184,7 +186,7 @@ public void testContinuousRecording(final TestInfo testInfo) throws Exception { } @Test - @DisplayName("Test continuous recording - 1 sec jmx delay, jmethodid cache") + @DisplayName("Test continuous recording - 1 sec jmx delay, zstd compression") public void testContinuousRecording_jmethodid_cache(final TestInfo testInfo) throws Exception { testWithRetry( () -> @@ -198,7 +200,7 @@ private void testContinuousRecording( final int jmxFetchDelay, final boolean endpointCollectionEnabled, final boolean asyncProfilerEnabled, - final boolean jmethodIdCacheEnabled) + final boolean withZstd) throws Exception { final ObjectMapper mapper = new ObjectMapper(); try { @@ -207,7 +209,7 @@ private void testContinuousRecording( jmxFetchDelay, endpointCollectionEnabled, asyncProfilerEnabled, - jmethodIdCacheEnabled, + withZstd, logFilePath) .start(); @@ -261,7 +263,11 @@ private void testContinuousRecording( assertEquals(InetAddress.getLocalHost().getHostName(), requestTags.get("host")); assertFalse(logHasErrors(logFilePath)); - IItemCollection events = JfrLoaderToolkit.loadEvents(new ByteArrayInputStream(rawJfr.get())); + InputStream eventStream = new ByteArrayInputStream(rawJfr.get()); + if (withZstd) { + eventStream = new ZstdInputStream(eventStream); + } + IItemCollection events = JfrLoaderToolkit.loadEvents(eventStream); assertTrue(events.hasItems()); Pair rangeStartAndEnd = getRangeStartAndEnd(events); // This nano-second compensates for the added nano second in @@ -308,7 +314,11 @@ private void testContinuousRecording( period > 0 && period <= upperLimit, () -> "Upload period = " + period + "ms, expected (0, " + upperLimit + "]ms"); - events = JfrLoaderToolkit.loadEvents(new ByteArrayInputStream(rawJfr.get())); + eventStream = new ByteArrayInputStream(rawJfr.get()); + if (withZstd) { + eventStream = new ZstdInputStream(eventStream); + } + events = JfrLoaderToolkit.loadEvents(eventStream); assertTrue(events.hasItems()); verifyDatadogEventsNotCorrupt(events); rangeStartAndEnd = getRangeStartAndEnd(events); @@ -689,7 +699,7 @@ private ProcessBuilder createDefaultProcessBuilder( final int jmxFetchDelay, final boolean endpointCollectionEnabled, final boolean asyncProfilerEnabled, - final boolean jmethodIdCacheEnabled, + final boolean withZstd, final Path logFilePath) { return createProcessBuilder( VALID_API_KEY, @@ -698,7 +708,7 @@ private ProcessBuilder createDefaultProcessBuilder( PROFILING_UPLOAD_PERIOD_SECONDS, endpointCollectionEnabled, asyncProfilerEnabled, - jmethodIdCacheEnabled, + withZstd, 0, logFilePath); } @@ -710,7 +720,7 @@ private ProcessBuilder createProcessBuilder( final int profilingUploadPeriodSecs, final boolean endpointCollectionEnabled, final boolean asyncProfilerEnabled, - final boolean jmethodIdCacheEnabled, + final boolean withZstd, final int exitDelay, final Path logFilePath) { return createProcessBuilder( @@ -722,7 +732,7 @@ private ProcessBuilder createProcessBuilder( profilingUploadPeriodSecs, endpointCollectionEnabled, asyncProfilerEnabled, - jmethodIdCacheEnabled, + withZstd, exitDelay, logFilePath); } @@ -736,7 +746,7 @@ private static ProcessBuilder createProcessBuilder( final int profilingUploadPeriodSecs, final boolean endpointCollectionEnabled, final boolean asyncProfilerEnabled, - final boolean jmethodIdCacheEnabled, + final boolean withZstd, final int exitDelay, final Path logFilePath) { final String templateOverride = @@ -770,7 +780,7 @@ private static ProcessBuilder createProcessBuilder( "-Ddd.profiling.debug.dump_path=/tmp/dd-profiler", "-Ddd.profiling.queueing.time.enabled=true", "-Ddd.profiling.queueing.time.threshold.millis=0", - "-Ddd.profiling.experimental.jmethodid_cache.enabled=" + jmethodIdCacheEnabled, + "-Ddd.profiling.debug.upload.compression=" + (withZstd ? "zstd" : "on"), "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", "-Ddd.profiling.context.attributes=foo,bar", "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug", diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index cb46e079d9a..87058d15809 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -36,6 +36,7 @@ final class CachedData { exclude(dependency('com.datadoghq.okio:okio')) exclude(dependency('com.squareup.okio:okio')) exclude(dependency('org.lz4:lz4-java')) + exclude(dependency('io.airlift:aircompressor')) // dogstatsd and its transitives exclude(dependency('com.datadoghq:java-dogstatsd-client')) @@ -67,7 +68,8 @@ CachedData.deps.shared = [ libs.jnr.unixsocket, libs.moshi, libs.jctools, - libs.lz4 + libs.lz4, + libs.aircompressor ] ext { diff --git a/internal-api/internal-api-9/src/main/java/datadog/trace/util/JPMSJPSAccess.java b/internal-api/internal-api-9/src/main/java/datadog/trace/util/JPMSJPSAccess.java deleted file mode 100644 index e02e0c253ff..00000000000 --- a/internal-api/internal-api-9/src/main/java/datadog/trace/util/JPMSJPSAccess.java +++ /dev/null @@ -1,40 +0,0 @@ -package datadog.trace.util; - -import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; - -import datadog.trace.api.Platform; -import java.lang.instrument.Instrumentation; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class JPMSJPSAccess { - private static Logger log = LoggerFactory.getLogger(JPMSJPSAccess.class); - - public static void patchModuleAccess(Instrumentation inst) { - Module unnamedModule = ClassLoader.getSystemClassLoader().getUnnamedModule(); - Module jvmstatModule = ModuleLayer.boot().findModule("jdk.internal.jvmstat").orElse(null); - - if (jvmstatModule != null) { - Map> extraOpens = Map.of("sun.jvmstat.monitor", Set.of(unnamedModule)); - - // Redefine the module - inst.redefineModule( - jvmstatModule, - Collections.emptySet(), - extraOpens, - extraOpens, - Collections.emptySet(), - Collections.emptyMap()); - } else { - log.debug( - SEND_TELEMETRY, - "Failed to find the jdk.internal.jvmstat module, skipping patching of module access on " - + Platform.getRuntimeVersion() - + " " - + Platform.getRuntimeVendor()); - } - } -} diff --git a/internal-api/internal-api-9/src/test/groovy/datadog/trace/util/PidHelperTest.groovy b/internal-api/internal-api-9/src/test/groovy/datadog/trace/util/PidHelperTest.groovy deleted file mode 100644 index ac271ac29e1..00000000000 --- a/internal-api/internal-api-9/src/test/groovy/datadog/trace/util/PidHelperTest.groovy +++ /dev/null @@ -1,21 +0,0 @@ -package datadog.trace.util - -import datadog.trace.test.util.DDSpecification -import net.bytebuddy.agent.ByteBuddyAgent - -class PidHelperTest extends DDSpecification { - - def "PID is available everywhere we test"() { - expect: - !PidHelper.getPid().isEmpty() - } - - def "JPS via jvmstat is used when possible"() { - when: - def inst = ByteBuddyAgent.install() - JPMSJPSAccess.patchModuleAccess(inst) - - then: - JPSUtils.VMPids != null - } -} diff --git a/internal-api/src/main/java/datadog/trace/util/JPSUtils.java b/internal-api/src/main/java/datadog/trace/util/JPSUtils.java deleted file mode 100644 index 24b6f176beb..00000000000 --- a/internal-api/src/main/java/datadog/trace/util/JPSUtils.java +++ /dev/null @@ -1,25 +0,0 @@ -package datadog.trace.util; - -import de.thetaphi.forbiddenapis.SuppressForbidden; -import java.lang.reflect.Method; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public final class JPSUtils { - private static final Logger log = LoggerFactory.getLogger(JPSUtils.class); - - @SuppressForbidden - public static Set getVMPids() { - try { - Class monitoredHostClass = Class.forName("sun.jvmstat.monitor.MonitoredHost"); - Method getMonitoredHostMethod = - monitoredHostClass.getDeclaredMethod("getMonitoredHost", String.class); - Object vmHost = getMonitoredHostMethod.invoke(null, "localhost"); - return (Set) monitoredHostClass.getDeclaredMethod("activeVms").invoke(vmHost); - } catch (Exception e) { - log.debug("Failed to invoke jvmstat with exception ", e); - return null; - } - } -} diff --git a/internal-api/src/main/java/datadog/trace/util/PidHelper.java b/internal-api/src/main/java/datadog/trace/util/PidHelper.java index 94060a487a8..6ba90686f47 100644 --- a/internal-api/src/main/java/datadog/trace/util/PidHelper.java +++ b/internal-api/src/main/java/datadog/trace/util/PidHelper.java @@ -1,20 +1,14 @@ package datadog.trace.util; import datadog.trace.api.Platform; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.context.TraceScope; import de.thetaphi.forbiddenapis.SuppressForbidden; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.lang.management.ManagementFactory; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -39,6 +33,7 @@ public static long getPidAsLong() { return PID_AS_LONG; } + @SuppressWarnings("unchecked") private static String findPid() { String pid = ""; if (Platform.isJavaVersionAtLeast(9)) { @@ -123,14 +118,6 @@ private static Path getJavaProcessesDir() { } public static Set getJavaPids() { - // Attempt to use jvmstat directly, fall through to jps process fork strategy - Set directlyObtainedPids = JPSUtils.getVMPids(); - if (directlyObtainedPids != null) { - return directlyObtainedPids; - } - - // Some JDKs don't have jvmstat available as a module, attempt to read from the hsperfdata - // directory instead try (Stream stream = Files.list(getJavaProcessesDir())) { return stream .map(Path::getFileName) @@ -145,56 +132,22 @@ public static Set getJavaPids() { if (name.isEmpty()) { return false; } - for (int i = 0; i < name.length(); i++) { - if (!Character.isDigit(name.charAt(i))) { - return false; - } + char c = name.charAt(0); + if (c < '0' || c > '9') { + // Short-circuit - let's not parse as long something that is definitely not a long + // number + return false; + } + long pid = -1; + try { + pid = Long.parseLong(name); + } catch (NumberFormatException ignored) { } - return true; + return pid != -1; }) .collect(Collectors.toSet()); } catch (IOException e) { - log.debug("Unable to obtain Java PIDs via hsperfdata", e); - } - - // there is no supported Java API to achieve this - // one could use sun.jvmstat.monitor.MonitoredHost but it is an internal API and can go away at - // any time - - // also, no guarantee it will work with all JVMs - ProcessBuilder pb = new ProcessBuilder("jps"); - try (TraceScope ignored = AgentTracer.get().muteTracing()) { - Process p = pb.start(); - // start draining the subcommand's pipes asynchronously to avoid flooding them - CompletableFuture> collecting = - CompletableFuture.supplyAsync( - () -> { - try (BufferedReader br = - new BufferedReader(new InputStreamReader(p.getInputStream()))) { - return br.lines() - .filter(l -> !l.contains("jps")) - .map( - l -> { - int idx = l.indexOf(' '); - return l.substring(0, idx); - }) - .collect(java.util.stream.Collectors.toSet()); - } catch (IOException e) { - log.debug("Unable to list java processes via 'jps'", e); - return Collections.emptySet(); - } - }); - if (p.waitFor(1200, TimeUnit.MILLISECONDS)) { - if (p.exitValue() == 0) { - return collecting.get(); - } else { - log.debug("Execution of 'jps' failed with exit code {}", p.exitValue()); - } - } else { - p.destroyForcibly(); - log.debug("Execution of 'jps' timed out"); - } - } catch (Exception e) { - log.debug("Unable to list java processes via 'jps'", e); + log.debug("Unable to obtain Java PIDs", e); } return Collections.emptySet(); }