Skip to content

Unified timeout of 15 minutes. Improved logic of taking thread and heap dumps. #9552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AlexeyKuznetsov-DD merged 2 commits into from
Sep 17, 2025
Merged

Unified timeout of 15 minutes. Improved logic of taking thread and heap dumps. #9552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
25e0e43
Unified timeout of 15 minutes. Improved logic of taking thread and he…
AlexeyKuznetsov-DD Sep 16, 2025
b0ff556
Minor
AlexeyKuznetsov-DD Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions dd-java-agent/agent-ci-visibility/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import java.time.Duration
import java.time.temporal.ChronoUnit

buildscript {
repositories {
mavenLocal()
Expand Down Expand Up @@ -67,7 +64,3 @@ shadowJar {
jar {
archiveClassifier = 'unbundled'
}

tasks.withType(Test).configureEach {
timeout = Duration.of(12, ChronoUnit.MINUTES)
}
12 changes: 0 additions & 12 deletions dd-java-agent/instrumentation/aws-java/aws-java-sqs-1.0/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import java.time.Duration
import java.time.temporal.ChronoUnit

muzzle {
pass {
group = "com.amazonaws"
Expand All @@ -18,15 +15,6 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test')

apply from: "$rootDir/gradle/configure_tests.gradle"

// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
forkedTest {
timeout = Duration.of(15, ChronoUnit.MINUTES)
}

latestDepForkedTest {
timeout = Duration.of(15, ChronoUnit.MINUTES)
}

dependencies {
compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-sqs', version: '1.11.0'
compileOnly group: 'com.amazonaws', name: 'amazon-sqs-java-messaging-lib', version: '1.0.0'
Expand Down
13 changes: 0 additions & 13 deletions dd-java-agent/instrumentation/aws-java/aws-java-sqs-2.0/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import java.time.Duration
import java.time.temporal.ChronoUnit

muzzle {
pass {
group = "software.amazon.awssdk"
Expand All @@ -19,16 +16,6 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test')

apply from: "$rootDir/gradle/configure_tests.gradle"

// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
forkedTest {
timeout = Duration.of(15, ChronoUnit.MINUTES)
}

latestDepForkedTest {
timeout = Duration.of(15, ChronoUnit.MINUTES)
}


dependencies {
compileOnly group: 'software.amazon.awssdk', name: 'sqs', version: '2.2.0'
compileOnly group: 'com.amazonaws', name: 'amazon-sqs-java-messaging-lib', version: '2.0.0'
Expand Down
8 changes: 0 additions & 8 deletions dd-java-agent/instrumentation/kafka/kafka-clients-0.11/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import java.time.Duration
import java.time.temporal.ChronoUnit

muzzle {
pass {
name = "since-0.11"
Expand All @@ -25,11 +22,6 @@ addTestSuite('iastLatestDepTest3')

apply from: "$rootDir/gradle/configure_tests.gradle"

// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
forkedTest {
timeout = Duration.of(15, ChronoUnit.MINUTES)
}

dependencies {
compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: '0.11.0.0'
implementation project(':dd-java-agent:instrumentation:kafka:kafka-common')
Expand Down
12 changes: 0 additions & 12 deletions dd-java-agent/instrumentation/vertx/vertx-rx-3.5/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import java.time.Duration
import java.time.temporal.ChronoUnit

apply from: "$rootDir/gradle/java.gradle"

addTestSuiteForDir('latestDepTest', 'test')
Expand All @@ -12,15 +9,6 @@ latestDepTest {
finalizedBy 'latestDepForkedTest'
}

// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
forkedTest {
timeout = Duration.of(15, ChronoUnit.MINUTES)
}

latestDepForkedTest {
timeout = Duration.of(15, ChronoUnit.MINUTES)
}

dependencies {
testImplementation project(':dd-java-agent:instrumentation:netty:netty-4.1')
testImplementation project(':dd-java-agent:instrumentation:trace-annotation')
Expand Down
3 changes: 1 addition & 2 deletions dd-smoke-tests/gradle/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ test {
environment "MAVEN_REPOSITORY_PROXY", project.property("mavenRepositoryProxy")
}

// overriding the default timeout of 9 minutes set in configure_tests.gradle,
// as Gradle smoke tests might run for a longer duration
// Overriding the default timeout as Gradle smoke tests might run for a longer duration.
timeout = Duration.of(20, ChronoUnit.MINUTES)
}

3 changes: 1 addition & 2 deletions dd-smoke-tests/maven/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ tasks.withType(Test).configureEach {
dependsOn "shadowJar"
jvmArgs "-Ddatadog.smoketest.maven.jar.path=${tasks.shadowJar.archiveFile.get()}"

// overriding the default timeout set in configure_tests.gradle, as Maven smoke
// tests might run for a longer duration
// Overriding the default timeout as Maven smoke tests might run for a longer duration.
timeout = Duration.of(25, ChronoUnit.MINUTES)

if (project.hasProperty("mavenRepositoryProxy")) {
Expand Down
10 changes: 2 additions & 8 deletions gradle/configure_tests.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ def forkedTestLimit = gradle.sharedServices.registerIfAbsent("forkedTestLimit",
maxParallelUsages = 3
}

// Force timeout after 9 minutes (The timeout is configurable per job, default job timeout is 1h)
def testTimeoutDuration = Duration.of(9, ChronoUnit.MINUTES)

testing {
suites.configureEach {
// Use JUnit 5 to run tests
Expand All @@ -39,10 +36,6 @@ testing {

// Go through the Test tasks and configure them
tasks.withType(Test).configureEach {
// All tests must complete within 15 minutes.
// This value is quite big because with lower values (3 mins) we were experiencing large number of false positives
timeout = Duration.ofMinutes(15)

// Disable all tests if skipTests property was specified
onlyIf { !project.rootProject.hasProperty("skipTests") }

Expand Down Expand Up @@ -82,7 +75,8 @@ tasks.withType(Test).configureEach {
exclude("**/*ForkedTest*")
}

timeout = testTimeoutDuration
// Set test timeout for 15 minutes. Default job timeout is 1h (configured on CI level).
timeout = Duration.of(15, ChronoUnit.MINUTES)

check.dependsOn(it)
}
Expand Down
32 changes: 20 additions & 12 deletions gradle/dump_hanging_test.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@ tasks.withType(Test).configureEach { testTask ->
t
})

// Calculate delay for taking dumps as test timeout minus 2 minutes, but no less than 1 minute.
def delayMinutes = Math.max(1L, timeout.get().minusMinutes(2).toMinutes())
// Calculate delay for taking dumps as test timeout minus 1 minutes, but no less than 1 minute.
def delayMinutes = Math.max(1L, timeout.get().minusMinutes(1).toMinutes())

def future = scheduler.schedule({
logger.warn("Taking dumps for: ${testTask.getPath()} after ${delayMinutes} minutes.")

try {
// Use Gradle's build dir and adjust for CI artifacts collection if needed.
def dumpDir = layout.buildDirectory.dir('dumps').map {
def dumpsDir = layout.buildDirectory.dir('dumps').map {
if (providers.environmentVariable("CI").isPresent()) {
// Move reports into the folder collected by the collect_reports.sh script.
new File(it.getAsFile().absolutePath.replace('dd-trace-java/dd-java-agent', 'dd-trace-java/workspace/dd-java-agent'))
Expand All @@ -25,7 +27,15 @@ tasks.withType(Test).configureEach { testTask ->
}
}.get()

dumpDir.mkdirs()
dumpsDir.mkdirs()

// For simplicity, use `0` as the PID, which collects all thread dumps across JVMs.
// Single file can be useful for quick search.
def threadDumpsFile = new File(dumpsDir, "all-thread-dumps-${System.currentTimeMillis()}.log")
new ProcessBuilder("jcmd", "0", "Thread.print", "-l")
Copy link
Copy Markdown
Contributor

@bric3 bric3 Sep 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Today I learned that using 0 would send the command to all JVM processes.

.redirectErrorStream(true)
.redirectOutput(threadDumpsFile)
.start().waitFor()

// Collect PIDs of all Java processes.
def jvmProcesses = 'jcmd -l'.execute().text.readLines()
Expand All @@ -36,20 +46,18 @@ tasks.withType(Test).configureEach { testTask ->
.collect({ it.substring(0, it.indexOf(' ')) })

pids.each { pid ->
logger.warn("Taking dumps for: ${testTask.getPath()}")
// Collect heap dump by pid.
def heapDumpFile = new File(dumpsDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath
def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}"
cmd.execute().waitFor()

// Collect thread dump.
def threadDumpFile = new File(dumpDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log")
// Collect thread dump by pid.
def threadDumpFile = new File(dumpsDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log")
new ProcessBuilder('jcmd', pid, 'Thread.print', '-l')
.redirectErrorStream(true)
.redirectOutput(threadDumpFile)
.start()
.waitFor()

// Collect heap dump.
def heapDumpFile = new File(dumpDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath
def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}"
cmd.execute().waitFor()
}
} catch (Throwable e) {
logger.warn("Dumping failed: ${e.message}")
Expand Down