diff --git a/build.gradle.kts b/build.gradle.kts index 396b3453db9..f76a199988a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,14 +3,13 @@ import com.diffplug.gradle.spotless.SpotlessExtension plugins { id("datadog.gradle-debug") id("datadog.dependency-locking") + id("datadog.tracer-version") + id("datadog.dump-hanged-test") id("com.diffplug.spotless") version "6.13.0" id("com.github.spotbugs") version "5.0.14" id("de.thetaphi.forbiddenapis") version "3.8" - - id("tracer-version") id("io.github.gradle-nexus.publish-plugin") version "2.0.0" - id("com.gradleup.shadow") version "8.3.6" apply false id("me.champeau.jmh") version "0.7.3" apply false id("org.gradle.playframework") version "0.13" apply false diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 9ec83bbb1be..d80c0c65132 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -27,9 +27,13 @@ gradlePlugin { implementationClass = "datadog.gradle.plugin.CallSiteInstrumentationPlugin" } create("tracer-version-plugin") { - id = "tracer-version" + id = "datadog.tracer-version" implementationClass = "datadog.gradle.plugin.version.TracerVersionPlugin" } + create("dump-hanged-test-plugin") { + id = "datadog.dump-hanged-test" + implementationClass = "datadog.gradle.plugin.dump.DumpHangedTestPlugin" + } create("supported-config-generation") { id = "supported-config-generator" implementationClass = "datadog.gradle.plugin.config.SupportedConfigPlugin" diff --git a/buildSrc/src/integTest/kotlin/datadog/gradle/plugin/version/DumpHangedTestIntegrationTest.kt b/buildSrc/src/integTest/kotlin/datadog/gradle/plugin/version/DumpHangedTestIntegrationTest.kt new file mode 100644 index 00000000000..0b1fc6bbad0 --- /dev/null +++ b/buildSrc/src/integTest/kotlin/datadog/gradle/plugin/version/DumpHangedTestIntegrationTest.kt @@ -0,0 +1,123 @@ +package datadog.gradle.plugin.version + +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.UnexpectedBuildFailure +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertNotNull +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.nio.file.Paths + +class DumpHangedTestIntegrationTest { + @Test + fun `should not take dumps`(@TempDir projectDir: File) { + val output = runGradleTest(projectDir, testSleep = 1000) + + // Assert Gradle output has no evidence of taking dumps. + assertFalse(output.contains("Taking dumps after 15 seconds delay for :test")) + assertFalse(output.contains("Requesting stop of task ':test' as it has exceeded its configured timeout of 20s.")) + + assertTrue(file(projectDir, "build").exists()) // Assert build happened. + assertFalse(file(projectDir, "build", "dumps").exists()) // Assert no dumps created. + } + + @Test + fun `should take dumps`(@TempDir projectDir: File) { + val output = runGradleTest(projectDir, testSleep = 25_0000) + + // Assert Gradle output has evidence of taking dumps. + assertTrue(output.contains("Taking dumps after 15 seconds delay for :test")) + assertTrue(output.contains("Requesting stop of task ':test' as it has exceeded its configured timeout of 20s.")) + + assertTrue(file(projectDir, "build").exists()) // Assert build happened. + + val dumps = file(projectDir, "build", "dumps") + assertTrue(dumps.exists()) // Assert dumps created. + + // Assert actual dumps created. + val dumpFiles = dumps.list() + assertNotNull(dumpFiles.find { it.endsWith(".hprof") }) + assertNotNull(dumpFiles.find { it.startsWith("all-thread-dumps") }) + } + + private fun runGradleTest(projectDir: File, testSleep: Long): List { + file(projectDir, "settings.gradle.kts").writeText( + """ + rootProject.name = "test-project" + """.trimIndent() + ) + + file(projectDir, "build.gradle.kts").writeText( + """ + import java.time.Duration + + plugins { + id("java") + id("datadog.dump-hanged-test") + } + + group = "datadog.dump.test" + + repositories { + mavenCentral() + } + + dependencies { + testImplementation(platform("org.junit:junit-bom:5.10.0")) + testImplementation("org.junit.jupiter:junit-jupiter") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") + } + + dumpHangedTest { + // Set the dump offset for 5 seconds to trigger taking dumps after 15 seconds. + dumpOffset.set(5) + } + + tasks.withType().configureEach { + // Set test timeout after 20 seconds. + timeout.set(Duration.ofSeconds(20)) + + useJUnitPlatform() + } + """.trimIndent() + ) + + file(projectDir, "src", "test", "java", "SimpleTest.java", makeDirectory = true).writeText( + """ + import org.junit.jupiter.api.Test; + + public class SimpleTest { + @Test + public void test() throws InterruptedException { + Thread.sleep($testSleep); + } + } + """.trimIndent() + ) + + try { + val buildResult = GradleRunner.create() + .forwardOutput() + .withPluginClasspath() + .withArguments("test") + .withProjectDir(projectDir) + .build() + + return buildResult.output.lines() + } catch (e: UnexpectedBuildFailure) { + return e.buildResult.output.lines() + } + } + + private fun file(projectDir: File, vararg parts: String, makeDirectory: Boolean = false): File { + val f = Paths.get(projectDir.absolutePath, *parts).toFile() + + if (makeDirectory) { + f.parentFile.mkdirs() + } + + return f + } +} diff --git a/buildSrc/src/integTest/kotlin/datadog/gradle/plugin/version/TracerVersionIntegrationTest.kt b/buildSrc/src/integTest/kotlin/datadog/gradle/plugin/version/TracerVersionIntegrationTest.kt index 779fe2f22f0..15fa42968c4 100644 --- a/buildSrc/src/integTest/kotlin/datadog/gradle/plugin/version/TracerVersionIntegrationTest.kt +++ b/buildSrc/src/integTest/kotlin/datadog/gradle/plugin/version/TracerVersionIntegrationTest.kt @@ -265,7 +265,7 @@ class TracerVersionIntegrationTest { File(projectDir, "build.gradle.kts").writeText( """ plugins { - id("tracer-version") + id("datadog.tracer-version") } tasks.register("printVersion") { diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt new file mode 100644 index 00000000000..a8297d4dfe6 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt @@ -0,0 +1,177 @@ +package datadog.gradle.plugin.dump + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.Property +import org.gradle.api.provider.Provider +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters +import org.gradle.api.tasks.testing.Test +import org.gradle.kotlin.dsl.extra +import org.gradle.kotlin.dsl.withType +import java.io.File +import java.io.IOException +import java.lang.ProcessBuilder.Redirect +import java.time.Duration +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +/** + * Plugin to collect thread and heap dumps for hanged tests. + */ +class DumpHangedTestPlugin : Plugin { + companion object { + private const val DUMP_FUTURE_KEY = "dumping_future" + } + + /** Plugin properties */ + abstract class DumpHangedTestProperties @Inject constructor(objects: ObjectFactory) { + // Time offset (in seconds) before a test reaches its timeout at which dumps should be started. + // Defaults to 60 seconds. + val dumpOffset: Property = objects.property(Long::class.java) + } + + /** Executor wrapped with proper Gradle lifecycle. */ + abstract class DumpSchedulerService : BuildService, AutoCloseable { + private val executor: ScheduledExecutorService = + Executors.newSingleThreadScheduledExecutor { r -> Thread(r, "hanged-test-dump").apply { isDaemon = true } } + + fun schedule(task: () -> Unit, delay: Duration): ScheduledFuture<*> = + executor.schedule(task, delay.toMillis(), TimeUnit.MILLISECONDS) + + override fun close() { + executor.shutdownNow() + } + } + + override fun apply(project: Project) { + if (project.rootProject != project) { + return + } + + val scheduler = project.gradle.sharedServices + .registerIfAbsent("dumpHangedTestScheduler", DumpSchedulerService::class.java) + + // Create plugin properties. + val props = project.extensions.create("dumpHangedTest", DumpHangedTestProperties::class.java) + + fun configure(p: Project) { + p.tasks.withType().configureEach { + doFirst { schedule(this, scheduler, props) } + doLast { cleanup(this) } + } + } + + configure(project) + + project.subprojects(::configure) + } + + private fun schedule(t: Task, scheduler: Provider, props: DumpHangedTestProperties) { + val taskName = t.path + + if (t.extra.has(DUMP_FUTURE_KEY)) { + t.logger.info("Taking dumps already scheduled for $taskName") + return + } + + val dumpOffset = props.dumpOffset.getOrElse(60) + val delay = t.timeout.map { it.minusSeconds(dumpOffset) }.orNull + + if (delay == null || delay.seconds < 0) { + t.logger.info("Taking dumps has invalid timeout configured for $taskName") + return + } + + val future = scheduler.get().schedule({ + t.logger.quiet("Taking dumps after ${delay.seconds} seconds delay for $taskName") + + takeDump(t) + }, delay) + + t.extra.set(DUMP_FUTURE_KEY, future) + } + + private fun takeDump(t: Task) { + try { + // Use Gradle's build dir and adjust for CI artifacts collection if needed. + val dumpsDir: File = t.project.layout.buildDirectory + .dir("dumps") + .map { dir -> + if (t.project.providers.environmentVariable("CI").isPresent) { + // Move reports into the folder collected by the collect_reports.sh script. + File( + dir.asFile.absolutePath.replace( + "dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent" + ) + ) + } else { + dir.asFile + } + } + .get() + + dumpsDir.mkdirs() + + fun file(name: String, ext: String = "log") = + File(dumpsDir, "$name-${System.currentTimeMillis()}.$ext") + + // For simplicity, use `0` as the PID, which collects all thread dumps across JVMs. + val allThreadsFile = file("all-thread-dumps") + runCmd(Redirect.to(allThreadsFile), "jcmd", "0", "Thread.print", "-l") + + // Collect all JVMs pids. + val allJavaProcessesFile = file("all-java-processes") + runCmd(Redirect.to(allJavaProcessesFile), "jcmd", "-l") + + // Collect pids for 'Gradle Test Executor'. + val pids = allJavaProcessesFile.readLines() + .filter { it.contains("Gradle Test Executor") } + .map { it.substringBefore(' ') } + + pids.forEach { pid -> + // Collect heap dump by pid. + val heapDumpPath = file("${pid}-heap-dump", "hprof").absolutePath + runCmd(Redirect.INHERIT, "jcmd", pid, "GC.heap_dump", heapDumpPath) + + // Collect thread dump by pid. + val threadDumpFile = file("${pid}-thread-dump") + runCmd(Redirect.to(threadDumpFile), "jcmd", pid, "Thread.print", "-l") + } + } catch (e: Throwable) { + t.logger.warn("Taking dumps failed with error: ${e.message}, for ${t.path}") + } + } + + private fun cleanup(t: Task) { + val future = t.extra + .takeIf { it.has(DUMP_FUTURE_KEY) } + ?.get(DUMP_FUTURE_KEY) as? ScheduledFuture<*> + + if (future != null && !future.isDone) { + t.logger.info("Taking dump canceled with remaining delay of ${future.getDelay(TimeUnit.SECONDS)} seconds for ${t.path}") + future.cancel(false) + } + } + + private fun runCmd( + redirectTo: Redirect, + vararg args: String + ) { + val exitCode = ProcessBuilder(*args) + .redirectErrorStream(true) + .redirectOutput(redirectTo) + .start() + .waitFor() + + if (exitCode != 0) { + throw IOException("Process failed: ${args.joinToString(" ")}, exit code: $exitCode") + } + } +} diff --git a/gradle/configure_tests.gradle b/gradle/configure_tests.gradle index 2875fb0b2b8..2afd783cc8e 100644 --- a/gradle/configure_tests.gradle +++ b/gradle/configure_tests.gradle @@ -1,8 +1,6 @@ import java.time.Duration import java.time.temporal.ChronoUnit -apply from: "$rootDir/gradle/dump_hanging_test.gradle" - def isTestingInstrumentation(Project project) { return [ "junit-4.10", diff --git a/gradle/dump_hanging_test.gradle b/gradle/dump_hanging_test.gradle deleted file mode 100644 index d10ea461b44..00000000000 --- a/gradle/dump_hanging_test.gradle +++ /dev/null @@ -1,83 +0,0 @@ -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit - -// Schedule thread and heap dumps collection near test timeout. -tasks.withType(Test).configureEach { testTask -> - doFirst { - def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> - Thread t = new Thread(r, 'dump-scheduler') - t.daemon = true - t - }) - - // Calculate delay for taking dumps as test timeout minus 1 minutes, but no less than 1 minute. - def delayMinutes = Math.max(1L, timeout.get().minusMinutes(1).toMinutes()) - - def future = scheduler.schedule({ - logger.warn("Taking dumps for: ${testTask.getPath()} after ${delayMinutes} minutes.") - - try { - // Use Gradle's build dir and adjust for CI artifacts collection if needed. - def dumpsDir = layout.buildDirectory.dir('dumps').map { - if (providers.environmentVariable("CI").isPresent()) { - // Move reports into the folder collected by the collect_reports.sh script. - new File(it.getAsFile().absolutePath.replace('dd-trace-java/dd-java-agent', 'dd-trace-java/workspace/dd-java-agent')) - } else { - it.asFile - } - }.get() - - dumpsDir.mkdirs() - - // For simplicity, use `0` as the PID, which collects all thread dumps across JVMs. - // Single file can be useful for quick search. - def threadDumpsFile = new File(dumpsDir, "all-thread-dumps-${System.currentTimeMillis()}.log") - new ProcessBuilder("jcmd", "0", "Thread.print", "-l") - .redirectErrorStream(true) - .redirectOutput(threadDumpsFile) - .start().waitFor() - - // Collect PIDs of all Java processes. - def jvmProcesses = 'jcmd -l'.execute().text.readLines() - - // Collect pids for 'Gradle test executors'. - def pids = jvmProcesses - .findAll({ it.contains('Gradle Test Executor') }) - .collect({ it.substring(0, it.indexOf(' ')) }) - - pids.each { pid -> - // Collect heap dump by pid. - def heapDumpFile = new File(dumpsDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath - def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}" - cmd.execute().waitFor() - - // Collect thread dump by pid. - def threadDumpFile = new File(dumpsDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log") - new ProcessBuilder('jcmd', pid, 'Thread.print', '-l') - .redirectErrorStream(true) - .redirectOutput(threadDumpFile) - .start() - .waitFor() - } - } catch (Throwable e) { - logger.warn("Dumping failed: ${e.message}") - } - finally { - scheduler.shutdown() - } - }, delayMinutes, TimeUnit.MINUTES) - - // Store handles for cancellation in doLast. - testTask.ext.dumpFuture = future - testTask.ext.dumpScheduler = scheduler - } - - doLast { - // Cancel if the task finished before the scheduled dump. - try { - testTask.ext.dumpFuture?.cancel(false) - } finally { - testTask.ext.dumpScheduler?.shutdownNow() - } - } -}