Refactored dumping hanged tests by collecting child processes too.

Author: AlexeyKuznetsov-DD

.gitlab-ci.yml

@@ -31,7 +31,7 @@ variables:
   GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/"
   BUILDER_IMAGE_VERSION_PREFIX: "v26.01-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "v25.05-")
   REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
-  DEFAULT_TEST_JVMS: /^(8|11|17|21|25|stable)$/ # the latest "stable" version is 26
+  DEFAULT_TEST_JVMS: /^(8|21|ibm8)$/ # the latest "stable" version is 26
   PROFILE_TESTS:
     description: "Enable profiling of tests"
     value: "false"
@@ -468,17 +468,17 @@ check_build_src:
   variables:
     GRADLE_TARGET: ":buildSrc:build"
 
-check_base:
-  extends: .check_job
-  variables:
-    GRADLE_TARGET: ":baseCheck"
-
-check_inst:
-  extends: .check_job
-  parallel: 4
-  variables:
-    GRADLE_TARGET: ":instrumentationCheck"
-    CACHE_TYPE: "inst"
+# check_base:
+#   extends: .check_job
+#   variables:
+#     GRADLE_TARGET: ":baseCheck"
+#
+# check_inst:
+#   extends: .check_job
+#   parallel: 4
+#   variables:
+#     GRADLE_TARGET: ":instrumentationCheck"
+#     CACHE_TYPE: "inst"
 
 check_smoke:
   extends: .check_job
@@ -670,36 +670,36 @@ agent_integration_tests:
         DD_HOSTNAME: "local-agent"
         DD_API_KEY: "invalid_key_but_this_is_fine"
 
-test_base:
-  extends: .test_job
-  variables:
-    GRADLE_TARGET: ":baseTest"
-    CACHE_TYPE: "base"
-  parallel:
-    matrix: *test_matrix_4
-  script:
-    - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi
-    - !reference [.test_job, script]
-
-test_inst:
-  extends: .test_job_with_test_agent
-  variables:
-    GRADLE_TARGET: ":instrumentationTest"
-    CACHE_TYPE: "inst"
-  parallel:
-    matrix: *test_matrix_8
-
-test_inst_latest:
-  extends: .test_job_with_test_agent
-  variables:
-    GRADLE_TARGET: ":instrumentationLatestDepTest"
-    CACHE_TYPE: "latestdep"
-  parallel:
-    matrix:
-      - testJvm: ["8", "17", "21", "25"] # the latest "stable" version is LTS v25
-        # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
-        # This emulates "parallel" by including it in the matrix
-        CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]
+# test_base:
+#   extends: .test_job
+#   variables:
+#     GRADLE_TARGET: ":baseTest"
+#     CACHE_TYPE: "base"
+#   parallel:
+#     matrix: *test_matrix_4
+#   script:
+#     - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi
+#     - !reference [.test_job, script]
+
+# test_inst:
+#   extends: .test_job_with_test_agent
+#   variables:
+#     GRADLE_TARGET: ":instrumentationTest"
+#     CACHE_TYPE: "inst"
+#   parallel:
+#     matrix: *test_matrix_8
+#
+# test_inst_latest:
+#   extends: .test_job_with_test_agent
+#   variables:
+#     GRADLE_TARGET: ":instrumentationLatestDepTest"
+#     CACHE_TYPE: "latestdep"
+#   parallel:
+#     matrix:
+#       - testJvm: ["8", "17", "21", "25"] # the latest "stable" version is LTS v25
+#         # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
+#         # This emulates "parallel" by including it in the matrix
+#         CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]
 
 test_flaky:
   extends: .test_job_with_test_agent

buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt

@@ -120,43 +120,27 @@ class DumpHangedTestPlugin : Plugin<Project> {
 
       dumpsDir.mkdirs()
 
-      fun file(name: String, ext: String = "log") =
-        File(dumpsDir, "$name-${System.currentTimeMillis()}.$ext")
+      val allProcessesFile = file(dumpsDir, "all-processes")
+      runCmd(Redirect.to(allProcessesFile), "ps", "-ef")
 
-      // Collect all JVMs pids.
-      val allJavaProcessesFile = file("all-java-processes")
-      runCmd(Redirect.to(allJavaProcessesFile), "jcmd", "-l")
+      val allProcesses = extractProcesses(allProcessesFile)
 
-      // On IBM JDK thread dump can be collected by signaling the matching `Gradle Test Executor` process with `kill -3`.
-      // It will be writen into `/tmp/javacore.YYYYMMDD.HHMMSS.PID.SEQ.txt
-      if (isIbm8(allJavaProcessesFile)) {
-        val allProcessesFile = file("all-processes")
-        runCmd(Redirect.to(allProcessesFile), "ps", "-ef")
-        extractPidsIbm8(allProcessesFile).forEach { ibm8Pid ->
-          runCmd(Redirect.INHERIT, "kill", "-3", ibm8Pid)
-        }
-      } else {
-        val pids = extractPids(allJavaProcessesFile)
+      val gradleTestExecutors = allProcesses.filter { it.command.contains("Gradle Test Executor") }
+      val childProcesses = collectChildProcesses(allProcesses, gradleTestExecutors)
 
-        pids.forEach { pid ->
-          // Collect heap dump by pid.
-          val heapDumpPath = file("${pid}-heap-dump", "hprof").absolutePath
-          runCmd(Redirect.INHERIT, "jcmd", pid, "GC.heap_dump", heapDumpPath)
-
-          // Collect thread dump by pid.
-          val threadDumpFile = file("${pid}-thread-dump")
-          runCmd(Redirect.to(threadDumpFile), "jcmd", pid, "Thread.print", "-l")
-        }
+      (gradleTestExecutors + childProcesses).forEach { process -> collectDump(dumpsDir, process) }
 
-        // Just in case collect all thread dumps by using special PID `0`.
-        val allThreadsFile = file("all-thread-dumps")
-        runCmd(Redirect.to(allThreadsFile), "jcmd", "0", "Thread.print", "-l")
-      }
+      // Just in case collect all thread dumps by using special PID `0`.
+      val allThreadsFile = file(dumpsDir, "all-thread-dumps")
+      runCmd(Redirect.to(allThreadsFile), "jcmd", "0", "Thread.print", "-l")
     } catch (e: Throwable) {
       t.logger.warn("Taking dumps failed with error: ${e.message ?: e.javaClass.name}, for ${t.path}")
     }
   }
 
+  private fun file(baseDir: File, name: String, ext: String = "log") =
+    File(baseDir, "$name-${System.currentTimeMillis()}.$ext")
+
   private fun cleanup(t: Task) {
     val future = t.extra
       .takeIf { it.has(DUMP_FUTURE_KEY) }
@@ -183,23 +167,55 @@ class DumpHangedTestPlugin : Plugin<Project> {
     }
   }
 
-  private fun isIbm8(file: File): Boolean =
-    file.readLines().any { it.contains("-PtestJvm=ibm8") }
+  private data class ProcessInfo(
+    val pid: String,
+    val ppid: String,
+    val command: String,
+    val isIbm: Boolean
+  )
 
-  private fun extractPids(file: File): List<String> =
-    file.readLines()
-      .filter { it.contains("Gradle Test Executor") }
-      .map { it.substringBefore(' ') }
+  private val whitespaceRegex = Regex("\\s+")
 
-  private fun extractPidsIbm8(file: File): List<String> =
+  // ps -ef format produce output like: `UID PID PPID C STIME TTY TIME CMD`
+  private fun extractProcesses(file: File): List<ProcessInfo> =
     file.readLines()
-      .filter { it.contains("Gradle Test Executor") }
-      .filter { it.contains("ibm", ignoreCase = true) }
-      .mapNotNull(::extractPid)
+      .filter { it.contains("/bin/java") }
+      .map {
+        val parts = it.trimStart().split(whitespaceRegex, limit = 8)
+        val command = parts.getOrNull(7) ?: ""
+
+        ProcessInfo(
+          pid = parts[1],
+          ppid = parts[2],
+          command = command,
+          isIbm = command.contains("/ibm8")
+        )
+      }
 
-  private val whitespaceRegex = Regex("\\s+")
+  private fun collectChildProcesses(
+    allProcesses: List<ProcessInfo>,
+    gradleTestExecutors: List<ProcessInfo>
+  ): List<ProcessInfo> {
+    val parentPids = gradleTestExecutors.map { it.pid }.toSet()
+    return allProcesses.filter { parentPids.contains(it.ppid) }
+  }
 
-  // ps -ef format produce output like: UID PID PPID ...
-  private fun extractPid(line: String): String? =
-    line.trimStart().split(whitespaceRegex, limit = 3).getOrNull(1)
+  private fun collectDump(
+    baseDir: File,
+    process: ProcessInfo
+  ) {
+    if (process.isIbm) {
+      // On IBM JDK thread dump can be collected by signaling process with `kill -3`.
+      // It will be writen into `/tmp/javacore.YYYYMMDD.HHMMSS.PID.SEQ.txt
+      runCmd(Redirect.INHERIT, "kill", "-3", process.pid)
+    } else {
+      // Collect heap dump by pid.
+      val heapDumpPath = file(baseDir, "${process.pid}-heap-dump", "hprof").absolutePath
+      runCmd(Redirect.INHERIT, "jcmd", process.pid, "GC.heap_dump", heapDumpPath)
+
+      // Collect thread dump by pid.
+      val threadDumpFile = file(baseDir, "${process.pid}-thread-dump", "log")
+      runCmd(Redirect.to(threadDumpFile), "jcmd", process.pid, "Thread.print", "-l")
+    }
+  }
 }

dd-smoke-tests/concurrent/java-21/src/test/groovy/datadog/smoketest/concurrent/AbstractConcurrentTest.groovy

@@ -65,7 +65,7 @@ abstract class AbstractConcurrentTest extends AbstractSmokeTest {
   }
 
   protected void receivedCorrectTrace() {
-    waitForTrace(defaultPoll, checkTrace())
+    waitForTrace(freezePoll, checkTrace())
     assert traceCount.get() == 1
     assert testedProcess.waitFor(TIMEOUT_SECS, SECONDS)
     assert testedProcess.exitValue() == 0

dd-smoke-tests/osgi/src/main/java/datadog/smoketest/osgi/app/OSGiApplication.java

@@ -70,6 +70,8 @@ public static void main(final String[] args) throws Exception {
 
     framework.waitForStop(1_000);
 
+    Thread.sleep(1000 * 60 * 21); // TODO emulating hanging task on CI
+
     // XXX: Knopflerfish will leave some dangling non-daemon thread and prevent shutdown here.
     System.exit(0);
   }

dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy

@@ -68,6 +68,9 @@ abstract class AbstractSmokeTest extends ProcessManager {
   @Shared
   protected final PollingConditions defaultPoll = new PollingConditions(timeout: 30, initialDelay: 0, delay: 1, factor: 1)
 
+  @Shared
+  protected final PollingConditions freezePoll = new PollingConditions(timeout: 1260, initialDelay: 0, delay: 5, factor: 2)
+
   @Shared
   @AutoCleanup
   protected TestHttpServer server = httpServer {