Reference chains for surviving live-heap samples (PROF-15341)

.github/workflows/test_workflow.yml

@@ -387,10 +387,14 @@ jobs:
       - name: Setup OS
         if: steps.set_enabled.outputs.enabled == 'true'
         run: |
+          # ports.ubuntu.com's IPv6 route is intermittently unreachable from GH-hosted
+          # aarch64 runners; force IPv4 so apt doesn't stall/fail on a dead IPv6 path,
+          # and retry on top in case of other transient connection blips.
+          printf 'Acquire::ForceIPv4 "true";\nAcquire::Retries "5";\n' | sudo tee /etc/apt/apt.conf.d/99force-ipv4 >/dev/null
           sudo apt update -y
           sudo apt remove -y g++
           sudo apt autoremove -y
-          sudo apt install -y curl zip unzip clang make build-essential binutils gdb
+          sudo apt install -y curl zip unzip clang make build-essential binutils gdb libgtest-dev libgmock-dev
           # Install debug symbols for system libraries
           sudo apt install -y libc6-dbg
           if [[ ${{ matrix.java_version }} =~ "-zing" ]]; then

.gitlab/reliability/chaos_check.sh

@@ -1,178 +1,17 @@
 #!/usr/bin/env bash
-
-set +e # Disable exit on error
+#
+# CI entry point for the chaos harness. Thin wrapper around
+# utils/run-chaos-harness.sh — this file only supplies the CI-specific
+# caching paths (so repeated scheduled runs on the same runner reuse the
+# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's
+# `artifacts:` block expects at the repo root. All the actual build/run logic
+# lives in the standalone script, which is also runnable by hand.
 
 HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )"
 
-RUNTIME=${1}
-CONFIG=${2:-profiler+tracer}
-ALLOCATOR=${3:-gmalloc}
-
-echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}"
-
-CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}"
-# CHAOS_JDK uses sdkman notation (<version>-<dist>); extract major for Adoptium API.
-JDK_MAJOR="${CHAOS_JDK%%.*}"
-JDK_ARCH=$(uname -m | sed 's/x86_64/x64/')
-JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}"
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  TMP=$(mktemp -d)
-  DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse"
-  echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..."
-  if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then
-    echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2
-    rm -rf "${TMP}"
-    exit 1
-  fi
-  mkdir -p "${JDK_INSTALL_DIR}"
-  tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1
-  rm -rf "${TMP}"
-fi
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2
-  exit 1
-fi
-export JAVA_HOME="${JDK_INSTALL_DIR}"
-export PATH="${JAVA_HOME}/bin:${PATH}"
-ACTIVE_JDK=$(java -version 2>&1 | head -1)
-# Check major version only — patch may differ from the Adoptium latest GA.
-if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then
-  echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2
-  exit 1
-fi
-
-# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot.
-# Running mvn from /tmp avoids the empty pom.xml at the repo root.
-DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1)
-if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then
-  DDPROF_JAR="${DDPROF_JAR_LOCAL}"
-  echo "Using local ddprof jar: ${DDPROF_JAR}"
-else
-  if [ -z "${CURRENT_VERSION:-}" ]; then
-    echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2
-    exit 1
-  fi
-  echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots"
-  (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \
-      -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \
-      -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION})
-  DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar"
-fi
-
-if [ ! -f "${DDPROF_JAR}" ]; then
-  echo "FAIL:ddprof jar unavailable" >&2
-  exit 1
-fi
-
-mkdir -p /var/lib/datadog
-wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer'
-
-# chaos.jar is produced once per pipeline by the chaos:build job (stresstest
-# stage) and pulled here as an artifact. Fall back to an inline build if the
-# artifact is absent (e.g. local repro outside CI).
-CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar"
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "chaos.jar artifact not present — building inline"
-  ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q )
-fi
-
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "FAIL:chaos.jar unavailable" >&2
-  exit 1
-fi
-
-# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's
-# (relocated) profiler classes match the version under test.
-DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \
-DDPROF_JAR=${DDPROF_JAR} \
-OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \
-"${ROOT}/utils/patch-dd-java-agent.sh"
-
-if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then
-  echo "FAIL:dd-java-agent patching failed" >&2
-  exit 1
-fi
-
-PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar
-
-case $CONFIG in
-  profiler)
-    echo "Running with profiler only"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false"
-    # @Trace is a no-op without the tracer, so trace-context is excluded here.
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  profiler+tracer)
-    echo "Running with profiler and tracer"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true"
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  *)
-    echo "Unknown configuration: $CONFIG"
-    exit 1
-    ;;
-esac
-
-case $ALLOCATOR in
-  gmalloc)
-    echo "Running with gmalloc"
-    ;;
-  tcmalloc)
-    echo "Running with tcmalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4')
-    # thread-churn/dump-storm antagonists cycle many short-lived threads;
-    # tcmalloc's defaults are slow to return their per-thread caches to the
-    # OS, which was inflating container RSS past the OOM limit on aarch64.
-    export TCMALLOC_RELEASE_RATE=10
-    export TCMALLOC_AGGRESSIVE_DECOMMIT=1
-    ;;
-  jemalloc)
-    echo "Running with jemalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so')
-    ;;
-  *)
-    echo "Unknown allocator: $ALLOCATOR"
-    echo "Valid values are: gmalloc, tcmalloc, jemalloc"
-    exit 1
-    ;;
-esac
-
-echo "LD_PRELOAD=$LD_PRELOAD"
-
-timeout "$((RUNTIME + 300))" \
-java -javaagent:${PATCHED_AGENT} \
-     ${ENABLEMENT} \
-     -Ddd.profiling.upload.period=10 \
-     -Ddd.profiling.start-force-first=true \
-     -Ddd.profiling.ddprof.liveheap.enabled=true \
-     -Ddd.profiling.ddprof.alloc.enabled=true \
-     -Ddd.profiling.ddprof.wall.enabled=true \
-     -Ddd.profiling.ddprof.nativemem.enabled=true \
-     -Ddd.env=java-profiler-stability \
-     -Ddd.service=java-profiler-chaos \
-     -Xmx2g -Xms2g \
-     -XX:MaxMetaspaceSize=384m \
-     -XX:ErrorFile=${HERE}/../../hs_err.log \
-     -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \
-     -jar ${CHAOS_JAR} \
-     --duration ${RUNTIME}s \
-     --antagonists ${ANTAGONISTS}
-
-RC=$?
-echo "RC=$RC"
+export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}"
+export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}"
+export CHAOS_ERROR_FILE="${ROOT}/hs_err.log"
 
-if [ $RC -ne 0 ]; then
-  CRASH_MSG="Chaos harness crashed (RC=${RC})"
-  HS_ERR="${HERE}/../../hs_err.log"
-  if [ -f "${HS_ERR}" ]; then
-    SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120)
-    FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120)
-    [ -n "${SIG}" ]   && CRASH_MSG="${CRASH_MSG};${SIG}"
-    [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}"
-  fi
-  echo "FAIL:${CRASH_MSG}" >&2
-  exit 1
-fi
+exec "${ROOT}/utils/run-chaos-harness.sh" "$@"

AGENTS.md

@@ -437,6 +437,19 @@ table: see [doc/build/JdkUpgrades.md](doc/build/JdkUpgrades.md).
 - All code needs to strive to be lean in terms of resources consumption and easy to follow -
     do not shy away from factoring out self containing code to shorter functions with explicit name
 
+## CI / Automation Script Changes
+- Before rewriting or replacing a script invoked from `.gitlab-ci.yml`, `.gitlab/**/*.yml`, or any CI job,
+  run `git log -p -- <script>` (or diff against the version being replaced) and enumerate every behavior
+  branch it has — fallbacks, env-var-driven paths, error exits. Preserve all of them unless explicitly
+  asked to drop one; do not silently narrow behavior while refactoring.
+- Grep the calling `.gitlab-ci.yml` / `.gitlab/**/*.yml` jobs for how the script is invoked (env vars set,
+  exit codes expected) and confirm the rewrite still satisfies every caller, not just the common/local case.
+- When removing an item from a default list or config (an antagonist, a test tag, a feature flag), cite
+  the exact file/line proving it is inert or unused in that context before removing it — never remove
+  based on assumption alone.
+- After changing a CI-invoked script, run a syntax check (e.g. `bash -n <script>`) and trace through each
+  CI job that calls it before considering the change complete.
+
 ### C/C++ Code Style
 - **Indentation**: Match the exact indentation style of the surrounding code in each file. Do not introduce inconsistent indentation — reviewers will flag it.
 - **Minimal complexity**: Do not split inline logic into separate helper functions unless the helpers are reused or the original is genuinely hard to follow. Unnecessary splits add indirection without value.

build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt

@@ -3,6 +3,7 @@ package com.datadoghq.profiler
 
 import com.datadoghq.native.NativeBuildExtension
 import com.datadoghq.native.model.BuildConfiguration
+import com.datadoghq.native.model.Platform
 import com.datadoghq.native.util.PlatformUtils
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
@@ -204,6 +205,18 @@ class ProfilerTestPlugin : Plugin<Project> {
 
         // Environment variables (explicit for consistency across both paths)
         val envVars = buildMap<String, String> {
+            // Turn silent glibc heap corruption (e.g. a use-after-free write into a
+            // freed chunk) into an immediate, attributable SIGABRT instead of a crash
+            // much later in an unrelated allocation. Only meaningful against glibc's
+            // own malloc: skip on musl (no MALLOC_CHECK_ support), and skip whenever a
+            // sanitizer/allocator already replaces malloc via LD_PRELOAD.
+            if (PlatformUtils.currentPlatform == Platform.LINUX &&
+                !PlatformUtils.isMusl() &&
+                !testEnv.containsKey("LD_PRELOAD")
+            ) {
+                put("MALLOC_CHECK_", "3")
+                put("MALLOC_PERTURB_", (1..255).random().toString())
+            }
             putAll(testEnv)
             put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1")
             put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString())

ddprof-lib/src/main/cpp/arguments.cpp

@@ -18,6 +18,7 @@
 #include "arguments.h"
 #include "vmEntry.h"
 
+#include <algorithm>
 #include <errno.h>
 #include <limits.h>
 #include <stdio.h>
@@ -81,6 +82,24 @@ static const Multiplier UNIVERSAL[] = {
 //                          and keep the liveness track of 10% of the allocation
 //                          samples
 //     generations        - track surviving generations
+//     referencechains[=BOOL[:hops=N][:budget=N][:ttl=N][:framecap=N][:pausetarget=N][:painbudget=N][:firstpassbudget=N]]
+//                        - (PROF-15341, off by default) tag/BFS-walk live-heap
+//                          samples' referrer chains back toward a GC root.
+//                          pausetarget=N (ms) is the pause-time-SLO ceiling
+//                          ReferenceChainTracker::updatePacing() adapts the
+//                          effective budget/cadence toward (pause-time pacing
+//                          controller). painbudget=N (percent) bounds how much
+//                          wall-clock time a *restarted* search (one begun
+//                          after a prior search already completed/abandoned)
+//                          may spend on average - see PainBudget (painBudget.h).
+//                          firstpassbudget=N overrides just the search's
+//                          one-shot, root-seeded first pass's edge budget
+//                          (default 0 - falls back to budget=N) since that
+//                          pass alone decides which GC roots ever enter the
+//                          frontier at all, unlike every later pass's cheap,
+//                          incremental per-node expansion.
+//                          Sub-options are placeholders pending future tuning;
+//                          see doc/architecture/LiveHeapReferenceChains*.md
 //     lightweight[=BOOL] - enable lightweight profiling - events without
 //     stacktraces (default: true)
 //     remotesym[=BOOL]   - enable remote symbolication for native frames
@@ -428,6 +447,83 @@ Error Arguments::parse(const char *args) {
         _nativesocket = true;
       }
 
+      CASE("referencechains")
+      {
+        // Sub-options are colon-delimited key=value pairs after the boolean,
+        // e.g. "referencechains=true:hops=64:budget=2000". Parsed manually
+        // (not via strtok) because the outer arg loop above is itself mid
+        // strtok(..., ",") over the same buffer - a nested strtok call would
+        // clobber its saved state.
+        char *config = value ? strchr(value, ':') : nullptr;
+        if (config) {
+          *(config++) = 0;
+        }
+        if (value != NULL) {
+          switch (value[0]) {
+          case 'n': // no
+          case 'f': // false
+          case '0': // 0
+            _reference_chains = false;
+            break;
+          default:
+            _reference_chains = true;
+          }
+        } else {
+          _reference_chains = true;
+        }
+        char *cursor = config;
+        while (cursor != NULL) {
+          char *next = strchr(cursor, ':');
+          if (next) {
+            *(next++) = 0;
+          }
+          char *eq = strchr(cursor, '=');
+          if (eq) {
+            *(eq++) = 0;
+            // Floor every sub-option at the parse boundary rather than
+            // trusting a downstream cast/clamp to make an operator-supplied
+            // negative value safe: a negative hops value in particular gets
+            // compared as `depth >= (u32)ctx->hop_cap` (referenceChains.cpp),
+            // so an unclamped negative wraps to ~4e9 and silently disables
+            // the hop cap entirely - the opposite of the flag's intent, and
+            // it removes the one guard that otherwise bounds how long a
+            // single reference chain (and therefore its
+            // datadog.ReferenceChain JFR event) can grow. A negative budget
+            // similarly collapses ReferenceChainTracker::_effective_budget
+            // to 0 (updatePacing()'s own PID-clamp logic), which truncates
+            // every pass immediately and leaves the search RUNNING
+            // (re-walking the whole graph each cadence) until TTL instead of
+            // making progress. A negative framecap is handed straight to
+            // FrontierTable's constructor, which floors it to a
+            // zero-capacity table (that class's own std::max(max_cap, 0)),
+            // silently disabling tracking rather than erroring. ttl/
+            // pausetarget/painbudget already have incidental downstream
+            // clamps (runPass()'s `_ttl_ms > 0` gate, this class's own
+            // PidController/PainBudget std::max(..., 0) calls) but are
+            // floored here too so every sub-option's validation lives at one
+            // boundary instead of being split between here and several
+            // unrelated call sites.
+            if (strcasecmp(cursor, "hops") == 0) {
+              _reference_chains_hop_cap = std::max(atoi(eq), 1);
+            } else if (strcasecmp(cursor, "budget") == 0) {
+              _reference_chains_budget = std::max(atoi(eq), 1);
+            } else if (strcasecmp(cursor, "ttl") == 0) {
+              _reference_chains_ttl_ms = std::max(atol(eq), 0L);
+            } else if (strcasecmp(cursor, "framecap") == 0) {
+              _reference_chains_frontier_cap = std::max(atoi(eq), 1);
+            } else if (strcasecmp(cursor, "pausetarget") == 0) {
+              _reference_chains_pause_target_ms = std::max(atol(eq), 0L);
+            } else if (strcasecmp(cursor, "painbudget") == 0) {
+              _reference_chains_pain_budget_percent =
+                  std::min(std::max(atoi(eq), 0), 100);
+            } else if (strcasecmp(cursor, "firstpassbudget") == 0) {
+              _reference_chains_first_pass_budget = std::max(atoi(eq), 0);
+            }
+          }
+          cursor = next;
+        }
+      }
+
       DEFAULT()
       if (_unknown_arg == NULL)
         _unknown_arg = arg;