Fix the gap in the build time instrumentation plugin to allow processing additional folder (#10735)

fix(build): TODO non-deterministic OTel class embedding in otel-bootstrap shadow jar

The problem was that `./gradlew shadowJar --rerun-tasks` execution was
non-deterministic depending on the state of the build. This resulted in
`otel-bootstrap` jar sometimes embedded the full set of OTel semconv
classes (~189), and sometimes only the Java-compiled ones (~4).

### Root cause

`unpackJars` and `compileJava` shared the same output directory
(`compileJava.destinationDirectory` = `build/classes/java/main/`).

After a first successful build, `main/` contained **instrumented** OTel
classes (modified in-place by the ByteBuddy `doLast` action).
But on the next execution **with** `--rerun-tasks`:

1. `unpackJars` re-ran and wrote **original** (un-instrumented) OTel
   classes to `main/`, replacing the previously-instrumented versions.
2. The Java compiler detected that files in its output directory had
   changed (instrumented → original) and switched to
   **non-incremental mode**, in the info log:

   Not incremental: removing prior outputs"

   thus deleting **all** prior outputs from `main/` — including the
   OTel classes just placed by `unpackJars`.
3. In the end only the project's compiled and processed classes files
   remained.

The bug was not manifested on a clean state because `main/` had no prior
files for the Java compiler to compare against.

---

This commit fixes the situation by filling a gap in the build time
instrumentation plugin:

- `unpackJars`'s output is now a dedicated `build/otel-unpacked/`
  directory to avoid overlap with `compileJava.destinationDirectory`.
- Build time instrumentation plugin allow to declare a
  `includeClassDirectories` used as task input, its classes will be
  included for post processing.

Because `build/otel-unpacked/` is now only written to by `unpackJars`,
Gradle's stale-output cleanup for `compileJava` never touches it.

chore: Rename unpackJars to unpackOtelJars

fix(build): Properly wire the unpackJars task

The `unpackJars` task dependency do not have to be declared as a
dependency of the compile task, however it should be declared as
an included input of the build time instrumentation.

Also, the `unpackJars` is now a `Sync` task to deletes files that
are no longer in the source JARs.

style: sourcesDir to classesDir

Merge branch 'master' into bdu/allow-build-time-instrumentation-to-process-additional-classes

Merge branch 'master' into bdu/allow-build-time-instrumentation-to-process-additional-classes

Co-authored-by: brice.dutheil <brice.dutheil@datadoghq.com>

Author: bric3

buildSrc/src/main/groovy/datadog/gradle/plugin/instrument/BuildTimeInstrumentationExtension.groovy

@@ -1,9 +1,11 @@
 package datadog.gradle.plugin.instrument
 
+import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.DirectoryProperty
 import org.gradle.api.provider.ListProperty
 
 abstract class BuildTimeInstrumentationExtension {
   abstract ListProperty<String> getPlugins()
   abstract ListProperty<DirectoryProperty> getAdditionalClasspath()
+  abstract ConfigurableFileCollection getIncludeClassDirectories()
 }

buildSrc/src/main/groovy/datadog/gradle/plugin/instrument/BuildTimeInstrumentationPlugin.groovy

@@ -126,6 +126,7 @@ class BuildTimeInstrumentationPlugin implements Plugin<Project> {
           it.inputs.property("javaVersion", javaVersion)
           it.inputs.property("plugins", extension.plugins)
           it.inputs.files(extension.additionalClasspath)
+          it.inputs.files(extension.includeClassDirectories)
 
           // Temporary location for raw (un-instrumented) classes
           DirectoryProperty tmpUninstrumentedClasses = project.objects.directoryProperty().value(
@@ -149,7 +150,8 @@ class BuildTimeInstrumentationPlugin implements Plugin<Project> {
               extension.plugins,
               instrumentingClassPath,
               it.destinationDirectory,
-              tmpUninstrumentedClasses
+              tmpUninstrumentedClasses,
+              extension.includeClassDirectories
             )
           )
           logger.info("[BuildTimeInstrumentationPlugin] Configured post-compile instrumentation for $compileTaskName for source-set $sourceSetName")

buildSrc/src/main/groovy/datadog/gradle/plugin/instrument/InstrumentAction.groovy

@@ -44,6 +44,15 @@ abstract class InstrumentAction implements WorkAction<InstrumentWorkParameters>
       from(classesDirectory)
       into(tmpUninstrumentedDir)
     }
+    // Merge any additional class directories (e.g. unpacked dependency JARs) to be processed
+    parameters.includeClassDirectories.files.each { classesDir ->
+      if (classesDir.exists()) {
+        fileSystemOperations.copy {
+          from(classesDir)
+          into(tmpUninstrumentedDir)
+        }
+      }
+    }
     fileSystemOperations.delete {
       delete(objects.fileTree().from(classesDirectory))
     }

buildSrc/src/main/groovy/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.groovy

@@ -35,30 +35,33 @@ abstract class InstrumentPostProcessingAction implements Action<AbstractCompile>
   final FileCollection instrumentingClassPath
   final DirectoryProperty compilerOutputDirectory
   final DirectoryProperty tmpDirectory
+  final FileCollection includeClassDirectories
 
   @Inject
   InstrumentPostProcessingAction(
       String javaVersion,
       ListProperty<String> plugins,
       FileCollection instrumentingClassPath,
       DirectoryProperty compilerOutputDirectory,
-      DirectoryProperty tmpDirectory
+      DirectoryProperty tmpDirectory,
+      FileCollection includeClassDirectories
   ) {
     this.javaVersion = javaVersion == BuildTimeInstrumentationPlugin.DEFAULT_JAVA_VERSION ? JavaLanguageVersion.current() : JavaLanguageVersion.of(javaVersion)
     this.plugins = plugins
     this.instrumentingClassPath = instrumentingClassPath
     this.compilerOutputDirectory = compilerOutputDirectory
     this.tmpDirectory = tmpDirectory
+    this.includeClassDirectories = includeClassDirectories
   }
 
   @Override
   void execute(AbstractCompile task) {
     logger.info(
         """
-        [InstrumentPostProcessingAction] About to instrument classes 
-          javaVersion=${javaVersion}, 
-          plugins=${plugins.get()}, 
-          instrumentingClassPath=${instrumentingClassPath.files}, 
+        [InstrumentPostProcessingAction] About to instrument classes
+          javaVersion=${javaVersion},
+          plugins=${plugins.get()},
+          instrumentingClassPath=${instrumentingClassPath.files},
           rawClassesDirectory=${compilerOutputDirectory.get().asFile}
         """.stripIndent()
     )
@@ -72,6 +75,7 @@ abstract class InstrumentPostProcessingAction implements Action<AbstractCompile>
       parameters.instrumentingClassPath.setFrom(postCompileAction.instrumentingClassPath)
       parameters.compilerOutputDirectory.set(postCompileAction.compilerOutputDirectory)
       parameters.tmpDirectory.set(postCompileAction.tmpDirectory)
+      parameters.includeClassDirectories.setFrom(postCompileAction.includeClassDirectories)
     })
   }
 

buildSrc/src/main/groovy/datadog/gradle/plugin/instrument/InstrumentWorkParameters.groovy

@@ -14,4 +14,5 @@ interface InstrumentWorkParameters extends WorkParameters {
   ConfigurableFileCollection getInstrumentingClassPath()
   DirectoryProperty getCompilerOutputDirectory()
   DirectoryProperty getTmpDirectory()
+  ConfigurableFileCollection getIncludeClassDirectories()
 }

dd-java-agent/agent-otel/otel-bootstrap/build.gradle

@@ -28,8 +28,26 @@ configurations {
   }
 }
 
+// Unpack the embeddedClasspath jars into otelUnpackedDir so that OtelShimGradlePlugin can
+// instrument them (e.g. inject the OTel shim).
+def unpackOtelJars = tasks.register('unpackOtelJars', Sync) {
+  exclude 'META-INF/'
+  exclude '**/module-info.class'
+  exclude '**/package-info.class'
+
+  from(configurations.named('embeddedClasspath').map {
+    it.collect {
+      zipTree(it)
+    }
+  })
+  // Note: they are intentionally kept in a separate directory from compileJava's output so that
+  // Gradle's stale-output cleanup on a full rebuild does not delete them before instrumentation.
+  into layout.buildDirectory.dir("otel-unpacked")
+}
+
 buildTimeInstrumentation {
   plugins.addAll('datadog.opentelemetry.tooling.shim.OtelShimGradlePlugin')
+  includeClassDirectories.from(unpackOtelJars)
 }
 
 minimumInstructionCoverage = 0.0
@@ -56,27 +74,6 @@ dependencies {
 
   compileOnly project(':dd-java-agent:agent-bootstrap')
   implementation project(':dd-java-agent:agent-otel:otel-shim')
-
-  // buildTimeInstrumentationPlugin project(path: ':dd-java-agent:agent-otel:otel-tooling', configuration: 'buildTimeInstrumentationToolingPlugins')
-}
-
-// unpack embeddedClasspath to same path as compiled classes so it can get instrumented
-tasks.register('unpackJars', Copy) {
-  dependsOn configurations.named('embeddedClasspath')
-  exclude 'META-INF/'
-  exclude '**/module-info.class'
-  exclude '**/package-info.class'
-
-  from(configurations.named('embeddedClasspath').map {
-    it.collect {
-      zipTree(it)
-    }
-  })
-  into compileJava.destinationDirectory
-}
-
-tasks.named('compileJava') {
-  dependsOn 'unpackJars'
 }
 
 tasks.named("shadowJar", ShadowJar) {