fix(crashtracking): Properly handle J9 crash report (#10976)

fix(crashtracking): Properly handle J9 crash report

The J9 crashtracking initializer only worked when the JVM arguments
spelled out everything explicitly. It missed two OpenJ9 behaviors:

- `-Xdump:java:file=...` is optional, and without it the javacore is
written to the JVM working directory
- the `-Xdump:tool:exec=` value may be passed as `... %pid`, not only
the escaped `...\ %pid` form

As a result the crash uploader could be initialized with the wrong
javacore location or fail to recover the configured uploader script
path.

Now it defaults the javacore location to `user.dir` when OpenJ9 uses
its implicit output path, and accept both `exec=<path> %pid` and
`exec=<path>\ %pid` when extracting the crash uploader script path.

Co-authored-by: devflow.devflow-routing-intake <devflow.devflow-routing-intake@kubernetes.us1.ddbuild.io>

Author: bric3

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/Initializer.java

@@ -8,6 +8,7 @@
 import com.sun.management.HotSpotDiagnosticMXBean;
 import datadog.environment.JavaVirtualMachine;
 import datadog.environment.OperatingSystem;
+import datadog.environment.SystemProperties;
 import datadog.libs.ddprof.DdprofLibraryLoader;
 import datadog.trace.api.Platform;
 import datadog.trace.util.TempLocationManager;
@@ -127,22 +128,31 @@ public static boolean initialize(boolean forceJmx) {
    */
   private static boolean initializeJ9() {
     try {
-      String scriptPath = getJ9CrashUploaderScriptPath();
-
       // Check if -Xdump:tool is already configured via JVM arguments
       boolean xdumpConfigured = isXdumpToolConfigured();
       // Get custom javacore path if configured
       String javacorePath = getJ9JavacorePath();
+      if (javacorePath == null || javacorePath.isEmpty()) {
+        // OpenJ9 defaults javacore output to the JVM working directory. Persist that location in
+        // the uploader config so the crash script does not need to guess from its own cwd.
+        javacorePath = SystemProperties.get("user.dir");
+      }
 
       if (xdumpConfigured) {
         LOG.debug("J9 crash tracking: -Xdump:tool already configured, crash uploads enabled");
+        // Use the path from the -Xdump:tool arg when available (allows callers to specify a known
+        // path via -Xdump:tool:events=gpf+abort,exec=<path>\ %pid), falling back to the default
+        // TempLocationManager path when the path cannot be extracted.
+        String extractedPath = extractJ9ScriptPathFromXdumpArg();
+        String scriptPath = extractedPath != null ? extractedPath : getJ9CrashUploaderScriptPath();
         // Initialize the crash uploader script and config manager
         CrashUploaderScriptInitializer.initialize(scriptPath, null, javacorePath);
         // Also set up OOME notifier script
         String oomeScript = getScript("dd_oome_notifier");
         OOMENotifierScriptInitializer.initialize(oomeScript);
         return true;
       } else {
+        String scriptPath = getJ9CrashUploaderScriptPath();
         // Log instructions for manual configuration
         LOG.info("J9 JVM detected. To enable crash tracking, add this JVM argument at startup:");
         LOG.info("  -Xdump:tool:events=gpf+abort,exec={}\\ %pid", scriptPath);
@@ -158,6 +168,40 @@ private static boolean initializeJ9() {
     return false;
   }
 
+  /**
+   * Extract the crash uploader script path from the {@code -Xdump:tool} JVM argument.
+   *
+   * <p>Looks for a JVM argument of the form {@code
+   * -Xdump:tool:events=...,exec=/path/to/dd_crash_uploader.sh\ %pid} and returns the script path
+   * portion (before the {@code \ %pid} argument separator).
+   *
+   * @return the script path, or {@code null} if not found or not extractable
+   */
+  private static String extractJ9ScriptPathFromXdumpArg() {
+    List<String> vmArgs = JavaVirtualMachine.getVmOptions();
+    for (String arg : vmArgs) {
+      if (arg.startsWith("-Xdump:tool") && arg.contains("dd_crash_uploader")) {
+        int execIdx = arg.indexOf("exec=");
+        if (execIdx >= 0) {
+          String execVal = arg.substring(execIdx + 5);
+          // Separator between command and args: plain space, or "\ " (backslash + space) as
+          // suggested by the Initializer's log hint. Check plain space first since that is the
+          // form that actually works when the shell splits the exec string into tokens.
+          int spaceIdx = execVal.indexOf(' ');
+          if (spaceIdx >= 0) {
+            String candidate = execVal.substring(0, spaceIdx);
+            // Strip a trailing backslash left over from the "\ %pid" notation
+            return candidate.endsWith("\\")
+                ? candidate.substring(0, candidate.length() - 1)
+                : candidate;
+          }
+          return execVal;
+        }
+      }
+    }
+    return null;
+  }
+
   /**
    * Get the custom javacore file path from -Xdump:java:file=... JVM argument.
    *

dd-smoke-tests/crashtracking/build.gradle

@@ -42,4 +42,3 @@ tasks.withType(Test).configureEach {
     showStandardStreams = true
   }
 }
-

dd-smoke-tests/crashtracking/src/main/java/datadog/smoketest/crashtracking/OpenJ9CrashtrackingTestApplication.java

@@ -0,0 +1,56 @@
+package datadog.smoketest.crashtracking;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test application for OpenJ9 crash tracking smoke tests.
+ *
+ * <p>Waits for the agent to write the crash-uploader script, then crashes the JVM via a null
+ * pointer write using {@code sun.misc.Unsafe.putAddress(0L, 0L)} (accessed via reflection so the
+ * class compiles against any Java version). This triggers a GPF (general protection fault) on
+ * OpenJ9, which the {@code -Xdump:tool:events=gpf+abort,...} handler detects.
+ *
+ * <p>Note: {@code sun.misc.Unsafe.getLong(0L)} is converted to a Java-level {@link
+ * NullPointerException} on Semeru/OpenJ9 25, so it does not exercise crash tracking. {@code
+ * sun.misc.Unsafe.putAddress(0L, 0L)} goes directly to {@code unsafePut64} in the JVM native
+ * library and produces a native SIGSEGV at address 0.
+ *
+ * <p>System properties consumed:
+ *
+ * <ul>
+ *   <li>{@code dd.test.crash_script} — path of the crash-uploader script; the application waits for
+ *       the agent to write it before crashing, ensuring the agent is fully initialized
+ * </ul>
+ */
+public class OpenJ9CrashtrackingTestApplication {
+  public static void main(String[] args) throws Exception {
+    // Wait for the agent to write the crash-uploader script (proves initialization is done)
+    String scriptPath = System.getProperty("dd.test.crash_script");
+    if (scriptPath != null) {
+      long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
+      while (!Files.exists(Paths.get(scriptPath)) && System.nanoTime() < deadline) {
+        Thread.sleep(200);
+      }
+      if (!Files.exists(Paths.get(scriptPath))) {
+        System.err.println("Timeout: crash script not created at " + scriptPath);
+        System.exit(-1);
+      }
+    }
+
+    System.out.println("===> Crash script ready, crashing JVM via Unsafe.putAddress(0L, 0L)...");
+    System.out.flush();
+
+    // Write to address 0 via sun.misc.Unsafe to trigger a SIGSEGV (GPF event).
+    // Unsafe.getLong(0L) was not enough on OpenJ9 here; it threw a NullPointerException instead.
+    Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
+    Field f = unsafeClass.getDeclaredField("theUnsafe");
+    f.setAccessible(true);
+    Object theUnsafe = f.get(null);
+    Method putAddress = unsafeClass.getDeclaredMethod("putAddress", long.class, long.class);
+    putAddress.invoke(theUnsafe, 0L, 0L);
+  }
+}

dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/AbstractCrashtrackingSmokeTest.java

@@ -0,0 +1,120 @@
+package datadog.smoketest;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.squareup.moshi.Moshi;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import okhttp3.mockwebserver.Dispatcher;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+
+abstract class AbstractCrashtrackingSmokeTest {
+  static final OutputThreads OUTPUT = new OutputThreads();
+  static final Path LOG_FILE_DIR =
+      Paths.get(System.getProperty("datadog.smoketest.builddir"), "reports");
+
+  MockWebServer tracingServer;
+  final BlockingQueue<CrashTelemetryData> crashEvents = new LinkedBlockingQueue<>();
+  final Moshi moshi = new Moshi.Builder().build();
+  Path tempDir;
+
+  @BeforeEach
+  void setUpTracingServer() throws Exception {
+    tempDir = Files.createTempDirectory("dd-smoketest-");
+    crashEvents.clear();
+    tracingServer = new MockWebServer();
+    tracingServer.setDispatcher(
+        new Dispatcher() {
+          @Override
+          public MockResponse dispatch(RecordedRequest request) {
+            String data = request.getBody().readString(StandardCharsets.UTF_8);
+            System.out.println("URL ====== " + request.getPath());
+            if ("/telemetry/proxy/api/v2/apmtelemetry".equals(request.getPath())) {
+              try {
+                MinimalTelemetryData minimal =
+                    moshi.adapter(MinimalTelemetryData.class).fromJson(data);
+                if ("logs".equals(minimal.request_type)) {
+                  crashEvents.add(moshi.adapter(CrashTelemetryData.class).fromJson(data));
+                }
+              } catch (IOException e) {
+                System.out.println("Unable to parse: " + e);
+              }
+            }
+            System.out.println(data);
+            return new MockResponse().setResponseCode(200);
+          }
+        });
+    OUTPUT.clearMessages();
+  }
+
+  @AfterEach
+  void tearDownTracingServer() throws Exception {
+    tracingServer.shutdown();
+    try (Stream<Path> files = Files.walk(tempDir)) {
+      files.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+    }
+    Files.deleteIfExists(tempDir);
+  }
+
+  @AfterAll
+  static void shutdownOutputThreads() {
+    OUTPUT.close();
+  }
+
+  protected long crashDataTimeoutMs() {
+    return 10 * 1000;
+  }
+
+  protected String assertCrashPing() throws InterruptedException, IOException {
+    CrashTelemetryData crashData = crashEvents.poll(crashDataTimeoutMs(), TimeUnit.MILLISECONDS);
+    assertNotNull(crashData, "Crash ping not sent");
+    assertTrue(crashData.payload.get(0).tags.contains("is_crash_ping:true"), "Not a crash ping");
+    final Object uuid =
+        moshi.adapter(Map.class).fromJson(crashData.payload.get(0).message).get("crash_uuid");
+    assertNotNull(uuid, "crash uuid not found");
+    return uuid.toString();
+  }
+
+  protected CrashTelemetryData assertCrashData(String uuid)
+      throws InterruptedException, IOException {
+    CrashTelemetryData crashData = crashEvents.poll(crashDataTimeoutMs(), TimeUnit.MILLISECONDS);
+    assertNotNull(crashData, "Crash data not uploaded");
+    assertTrue(
+        crashData.payload.get(0).tags.contains("severity:crash"), "Expected severity:crash tag");
+    final Object receivedUuid =
+        moshi.adapter(Map.class).fromJson(crashData.payload.get(0).message).get("uuid");
+    assertEquals(uuid, receivedUuid, "crash uuid should match the one sent with the ping");
+    return crashData;
+  }
+
+  static String javaPath() {
+    String sep = FileSystems.getDefault().getSeparator();
+    return System.getProperty("java.home") + sep + "bin" + sep + "java";
+  }
+
+  static String appShadowJar() {
+    return System.getProperty("datadog.smoketest.app.shadowJar.path");
+  }
+
+  static String agentShadowJar() {
+    return System.getProperty("datadog.smoketest.agent.shadowJar.path");
+  }
+}