ci: raise ASan test JVM heap to fix flaky nightly OOMs (#640)

Author: jbachorik

build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt

@@ -222,9 +222,13 @@ object ConfigurationPresets {
                     // HeapBaseMinAddress is not accepted by JDK <= 11 (constraint violation);
                     // those JDKs rely on the vm.mmap_rnd_bits=8 CI-level mitigation instead.
                     if (PlatformUtils.testJvmMajorVersion() >= 12) {
+                        // HeapBaseMinAddress=64MB leaves ~1.9GB of address space below the
+                        // shadow region (0x7fff7000); 1024m keeps heap+CompressedClassSpace
+                        // well within that margin while giving forkEvery-restarted JVMs
+                        // enough headroom to avoid "Java heap space" OOMs seen in nightly CI.
                         config.testJvmArgs.addAll(listOf(
                             "-XX:HeapBaseMinAddress=0x4000000",
-                            "-Xmx512m",
+                            "-Xmx1024m",
                             "-XX:CompressedClassSpaceSize=256m"
                         ))
                     }

build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt

@@ -321,7 +321,7 @@ class ProfilerTestPlugin : Plugin<Project> {
                         // https://github.com/eclipse-openj9/openj9/issues/23514
                         !PlatformUtils.isTestJvmJ9()
                     }
-                    // The ASAN test JVM is pinned to -Xmx512m (see ConfigurationPresets.kt)
+                    // The ASAN test JVM is pinned to -Xmx1024m (see ConfigurationPresets.kt)
                     // to keep the heap below ASan's shadow-memory region. Without forking,
                     // Gradle reuses one JVM for the whole suite, and per-test allocations
                     // (JFR recordings, JMC object models) accumulate until it OOMs late in

ddprof-test/src/test/java/com/datadoghq/profiler/JVMAccessTest.java

@@ -3,7 +3,9 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
+import java.lang.management.ManagementFactory;
 import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -77,10 +79,40 @@ void testGetFlag() {
         // The test relies on the gradle test task setting the JVM flags to expected values
         assertEquals("build/hs_err_pid%p.log", flags.getStringFlag("ErrorFile")); // set to 'build/hs_err_pid%p.log' in the test task
         assertTrue(flags.getBooleanFlag("ResizeTLAB")); // set to 'true' in the test task
-        assertEquals(512 * 1024 * 1024, flags.getIntFlag("MaxHeapSize")); // set to 512m in the test task
+        // Derive the expected heap from this JVM's own -Xmx argument rather than hardcoding a
+        // value, so the test stays valid regardless of what the build task passes per config.
+        assertEquals(configuredMaxHeapBytes(), flags.getIntFlag("MaxHeapSize"));
         assertNotNull(flags.getStringFlag("OnError"));
     }
 
+    private static long configuredMaxHeapBytes() {
+        // HotSpot honors the last -Xmx flag on the command line when duplicates are
+        // present (e.g. ASan configs append a config-specific -Xmx after the
+        // standard one), so scan from the end to find the effective value.
+        List<String> jvmArgs = ManagementFactory.getRuntimeMXBean().getInputArguments();
+        for (int i = jvmArgs.size() - 1; i >= 0; i--) {
+            String arg = jvmArgs.get(i);
+            if (arg.startsWith("-Xmx")) {
+                return parseMemorySize(arg.substring("-Xmx".length()));
+            }
+        }
+        throw new IllegalStateException("Test JVM was not launched with -Xmx: " + jvmArgs);
+    }
+
+    private static long parseMemorySize(String value) {
+        try {
+            char unit = Character.toLowerCase(value.charAt(value.length() - 1));
+            switch (unit) {
+                case 'k': return Long.parseLong(value.substring(0, value.length() - 1)) * 1024L;
+                case 'm': return Long.parseLong(value.substring(0, value.length() - 1)) * 1024L * 1024L;
+                case 'g': return Long.parseLong(value.substring(0, value.length() - 1)) * 1024L * 1024L * 1024L;
+                default: return Long.parseLong(value);
+            }
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("Invalid -Xmx memory size: " + value, e);
+        }
+    }
+
     @Test
     void testGetFlagMismatch() {
         JVMAccess.Flags flags = JVMAccess.getInstance().flags();