Add critical watermark + GC nudge to chaos MemoryGovernor

pace() now brakes harder (50ms vs 5ms) and update() fires a one-shot
System.gc() above 0.93 cgroup / 0.90 heap, since plain throttling wasn't
enough to stop a heap OOME in job 1861234807.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: jbachorik

ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java

@@ -37,7 +37,11 @@
  * treated as permanently low rather than guessed at.
  *
  * <p>Antagonists poll {@link #pace()} once per loop iteration — a volatile
- * read plus, only while throttled, a short sleep.
+ * read plus, only while throttled, a short sleep. Above a second, higher
+ * critical watermark, {@code pace()} sleeps much longer and the sampler
+ * fires a one-off {@link System#gc()} to force reclaim before OOME hits —
+ * plain throttling wasn't always enough to stop a heap OOME once several
+ * antagonists kept allocating past the regular watermark (job 1861234807).
  */
 final class MemoryGovernor {
 
@@ -47,8 +51,13 @@ final class MemoryGovernor {
     // shed off-heap first), so back off earlier and require more slack.
     private static final double HEAP_HIGH_WATERMARK = 0.80;
     private static final double HEAP_LOW_WATERMARK = 0.65;
+    // Critical tier: same signals, higher bar. Crossing this calls for a
+    // harder brake than pace()'s regular 5ms sleep, plus a one-time GC nudge.
+    private static final double CGROUP_CRITICAL_WATERMARK = 0.93;
+    private static final double HEAP_CRITICAL_WATERMARK = 0.90;
     private static final long SAMPLE_INTERVAL_MS = 500;
     private static final long THROTTLE_SLEEP_MS = 5;
+    private static final long CRITICAL_THROTTLE_SLEEP_MS = 50;
 
     private static final Path[] USAGE_PATHS = {
             Paths.get("/sys/fs/cgroup/memory.current"),
@@ -60,6 +69,7 @@ final class MemoryGovernor {
     };
 
     private static volatile boolean throttled;
+    private static volatile boolean critical;
 
     private MemoryGovernor() {
     }
@@ -81,7 +91,13 @@ static void start() {
 
     /** Called from an antagonist's hot allocation loop. No-op unless throttled. */
     static void pace() {
-        if (throttled) {
+        if (critical) {
+            try {
+                Thread.sleep(CRITICAL_THROTTLE_SLEEP_MS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        } else if (throttled) {
             try {
                 Thread.sleep(THROTTLE_SLEEP_MS);
             } catch (InterruptedException e) {
@@ -119,18 +135,31 @@ private static void update(Path usagePath, long limitBytes) throws IOException {
 
         boolean high = cgroupFraction >= CGROUP_HIGH_WATERMARK || heapFraction >= HEAP_HIGH_WATERMARK;
         boolean low = cgroupFraction <= CGROUP_LOW_WATERMARK && heapFraction <= HEAP_LOW_WATERMARK;
+        boolean crit = cgroupFraction >= CGROUP_CRITICAL_WATERMARK || heapFraction >= HEAP_CRITICAL_WATERMARK;
         if (high && !throttled) {
             throttled = true;
-            log(cgroupFraction, heapFraction, true);
+            log(cgroupFraction, heapFraction, "throttling");
         } else if (low && throttled) {
             throttled = false;
-            log(cgroupFraction, heapFraction, false);
+            log(cgroupFraction, heapFraction, "released");
         }
         // else: in the dead zone between watermarks — keep current state.
+
+        if (crit && !critical) {
+            critical = true;
+            log(cgroupFraction, heapFraction, "critical");
+            // One-shot nudge on the crossing, not every sample tick, so a
+            // sustained critical state doesn't turn into a GC storm on top
+            // of the memory pressure it's meant to relieve.
+            System.gc();
+        } else if (low && critical) {
+            critical = false;
+            log(cgroupFraction, heapFraction, "critical-released");
+        }
     }
 
-    private static void log(double cgroupFraction, double heapFraction, boolean nowThrottled) {
-        System.out.println("[chaos] memory-governor " + (nowThrottled ? "throttling" : "released")
+    private static void log(double cgroupFraction, double heapFraction, String state) {
+        System.out.println("[chaos] memory-governor " + state
                 + " cgroup=" + String.format("%.2f", cgroupFraction)
                 + " heap=" + String.format("%.2f", heapFraction));
     }