Unset inherited JVM env vars in crashtracking scripts

The OOME notifier and crash uploader scripts now unset
JDK_JAVA_OPTIONS, JAVA_TOOL_OPTIONS, and _JAVA_OPTIONS before
spawning a child JVM. This prevents port conflicts, memory
contention, and lost diagnostics when the parent environment
contains flags like JMX remote ports.

Fixes #10766

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: jbachorik

dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.bat

@@ -34,6 +34,12 @@ echo Tags: %tags%
 echo JAVA_HOME: %java_home%
 echo PID: %PID%
 
+:: Clear environment variables that the parent JVM may have set so the child JVM
+:: starts with a minimal configuration (avoids port conflicts, memory contention, etc.)
+set JDK_JAVA_OPTIONS=
+set JAVA_TOOL_OPTIONS=
+set _JAVA_OPTIONS=
+
 :: Execute the Java command with the loaded values
 "%java_home%\bin\java" -Ddd.dogstatsd.start-delay=0 -jar "%agent%" sendOomeEvent "%tags%"
 set RC=%ERRORLEVEL%

dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.sh

@@ -49,6 +49,12 @@ echo "Tags: $config_tags"
 echo "JAVA_HOME: $config_java_home"
 echo "PID: $PID"
 
+# Clear environment variables that the parent JVM may have set so the child JVM
+# starts with a minimal configuration (avoids port conflicts, memory contention, etc.)
+unset JDK_JAVA_OPTIONS
+unset JAVA_TOOL_OPTIONS
+unset _JAVA_OPTIONS
+
 # Execute the Java command with the loaded values
 "$config_java_home/bin/java" -Ddd.dogstatsd.start-delay=0 -jar "$config_agent" sendOomeEvent "$config_tags"
 RC=$?

dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.bat

@@ -4,6 +4,11 @@ setlocal enabledelayedexpansion
 :: Check if PID is provided
 if "%1"=="" (
     echo "Error: No PID provided. Running in legacy mode."
+    :: Clear environment variables that the parent JVM may have set
+    set JDK_JAVA_OPTIONS=
+    set JAVA_TOOL_OPTIONS=
+    set _JAVA_OPTIONS=
+
     "!JAVA_HOME!\bin\java" -jar "!AGENT_JAR!" uploadCrash "!JAVA_ERROR_FILE!"
     if %ERRORLEVEL% EQU 0 (
         echo "Uploaded error file \"!JAVA_ERROR_FILE!\""
@@ -128,6 +133,12 @@ echo Error Log: %config_hs_err%
 echo JAVA_HOME: %config_java_home%
 echo PID: %PID%
 
+:: Clear environment variables that the parent JVM may have set so the child JVM
+:: starts with a minimal configuration (avoids port conflicts, memory contention, etc.)
+set JDK_JAVA_OPTIONS=
+set JAVA_TOOL_OPTIONS=
+set _JAVA_OPTIONS=
+
 :: Execute the Java command with the loaded values
 "%config_java_home%\bin\java" -jar "%config_agent%" uploadCrash -c "%configFile%" "%config_hs_err%"
 set RC=%ERRORLEVEL%

dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.sh

@@ -7,6 +7,11 @@ set +e
 if [ -z "$1" ]; then
   echo "Warn: No PID provided. Running in legacy mode."
 
+  # Clear environment variables that the parent JVM may have set
+  unset JDK_JAVA_OPTIONS
+  unset JAVA_TOOL_OPTIONS
+  unset _JAVA_OPTIONS
+
   "!JAVA_HOME!/bin/java" -jar "!AGENT_JAR!" uploadCrash "!JAVA_ERROR_FILE!"
   if [ $? -eq 0 ]; then
     echo "Error file !JAVA_ERROR_FILE! was uploaded successfully"
@@ -119,6 +124,12 @@ echo "Error Log: $config_hs_err"
 echo "JAVA_HOME: $config_java_home"
 echo "PID: $PID"
 
+# Clear environment variables that the parent JVM may have set so the child JVM
+# starts with a minimal configuration (avoids port conflicts, memory contention, etc.)
+unset JDK_JAVA_OPTIONS
+unset JAVA_TOOL_OPTIONS
+unset _JAVA_OPTIONS
+
 # Execute the Java command with the loaded values
 "$config_java_home/bin/java" -jar "$config_agent" uploadCrash -c "$configFile" "$config_hs_err"
 RC=$?

dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java

@@ -13,6 +13,7 @@
 import datadog.environment.OperatingSystem;
 import java.io.File;
 import java.io.IOException;
+import java.net.ServerSocket;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
@@ -319,6 +320,132 @@ void testCombineTracking() throws Exception {
     assertOOMEvent();
   }
 
+  /**
+   * Verifies that the OOME notifier script correctly unsets inherited JVM environment variables.
+   * Without the fix, the child JVM spawned by the script would inherit JDK_JAVA_OPTIONS containing
+   * JMX port-binding flags, causing a BindException and losing the OOME event.
+   *
+   * @see <a href="https://github.com/DataDog/dd-trace-java/issues/10766">#10766</a>
+   */
+  @Test
+  void testOomeTrackingWithInheritedEnvVars() throws Exception {
+    int jmxPort = findFreePort();
+
+    Path script = tempDir.resolve("dd_oome_notifier." + getExtension());
+    String onErrorValue = script + " %p";
+    String errorFile = tempDir.resolve("hs_err_pid%p.log").toString();
+
+    String onOOMEArg =
+        !Platform.isLinux()
+            ? "-XX:OnOutOfMemoryError=" + onErrorValue
+            : "-Ddd.crashtracking.debug.autoconfig.enable=true";
+
+    List<String> processArgs = new ArrayList<>();
+    processArgs.add(javaPath());
+    processArgs.add("-javaagent:" + agentShadowJar());
+    processArgs.add("-Xmx96m");
+    processArgs.add("-Xms96m");
+    if (!onOOMEArg.isEmpty()) {
+      processArgs.add(onOOMEArg);
+    }
+    processArgs.add("-XX:ErrorFile=" + errorFile);
+    processArgs.add("-XX:+CrashOnOutOfMemoryError");
+    processArgs.add("-Ddd.dogstatsd.start-delay=0");
+    processArgs.add("-Ddd.trace.enabled=false");
+    processArgs.add("-jar");
+    processArgs.add(appShadowJar());
+
+    ProcessBuilder pb = new ProcessBuilder(processArgs);
+    pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort()));
+    // Simulate admission controller injecting JMX flags via JDK_JAVA_OPTIONS
+    pb.environment()
+        .put(
+            "JDK_JAVA_OPTIONS",
+            "-Dcom.sun.management.jmxremote"
+                + " -Dcom.sun.management.jmxremote.port="
+                + jmxPort
+                + " -Dcom.sun.management.jmxremote.rmi.port="
+                + jmxPort
+                + " -Dcom.sun.management.jmxremote.authenticate=false"
+                + " -Dcom.sun.management.jmxremote.ssl=false");
+
+    System.out.println("==> Process args: " + pb.command());
+    System.out.println("==> JMX port: " + jmxPort);
+
+    Process p = pb.start();
+    OUTPUT.captureOutput(
+        p, LOG_FILE_DIR.resolve("testProcess.testOomeTrackingWithInheritedEnvVars.log").toFile());
+
+    assertExpectedCrash(p);
+    assertOOMEvent();
+  }
+
+  /**
+   * Verifies that the crash uploader script correctly unsets inherited JVM environment variables.
+   * Without the fix, the child JVM spawned by the script would inherit JDK_JAVA_OPTIONS containing
+   * JMX port-binding flags, causing a BindException and losing the crash data.
+   *
+   * @see <a href="https://github.com/DataDog/dd-trace-java/issues/10766">#10766</a>
+   */
+  @Test
+  void testCrashTrackingWithInheritedEnvVars() throws Exception {
+    int jmxPort = findFreePort();
+
+    Path script = tempDir.resolve("dd_crash_uploader." + getExtension());
+    String onErrorValue = script + " %p";
+    String errorFile = tempDir.resolve("hs_err.log").toString();
+
+    String onErrorArg =
+        !Platform.isLinux()
+            ? "-XX:OnError=" + onErrorValue
+            : "-Ddd.crashtracking.debug.autoconfig.enable=true";
+
+    List<String> processArgs = new ArrayList<>();
+    processArgs.add(javaPath());
+    processArgs.add("-javaagent:" + agentShadowJar());
+    processArgs.add("-Xmx96m");
+    processArgs.add("-Xms96m");
+    if (!onErrorArg.isEmpty()) {
+      processArgs.add(onErrorArg);
+    }
+    processArgs.add("-XX:ErrorFile=" + errorFile);
+    processArgs.add("-XX:+CrashOnOutOfMemoryError");
+    processArgs.add("-Ddd.dogstatsd.start-delay=0");
+    processArgs.add("-Ddd.trace.enabled=false");
+    processArgs.add("-jar");
+    processArgs.add(appShadowJar());
+
+    ProcessBuilder pb = new ProcessBuilder(processArgs);
+    pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
+    // Simulate admission controller injecting JMX flags via JDK_JAVA_OPTIONS
+    pb.environment()
+        .put(
+            "JDK_JAVA_OPTIONS",
+            "-Dcom.sun.management.jmxremote"
+                + " -Dcom.sun.management.jmxremote.port="
+                + jmxPort
+                + " -Dcom.sun.management.jmxremote.rmi.port="
+                + jmxPort
+                + " -Dcom.sun.management.jmxremote.authenticate=false"
+                + " -Dcom.sun.management.jmxremote.ssl=false");
+
+    System.out.println("==> Process args: " + pb.command());
+    System.out.println("==> JMX port: " + jmxPort);
+
+    Process p = pb.start();
+    OUTPUT.captureOutput(
+        p, LOG_FILE_DIR.resolve("testProcess.testCrashTrackingWithInheritedEnvVars.log").toFile());
+
+    assertExpectedCrash(p);
+    assertCrashData(assertCrashPing());
+  }
+
+  private static int findFreePort() throws IOException {
+    try (ServerSocket socket = new ServerSocket(0)) {
+      return socket.getLocalPort();
+    }
+  }
+
   private static void assertExpectedCrash(Process p) throws InterruptedException {
     // exit code -1 means the test application exited prematurely
     // exit code > 0 means the test application crashed, as expected