Prevent instrumenting debugger agent classes (#10803)

Prevent instrumenting debugger agent classes

Skip all classes from  debugger agent to avoid LinkageError
attempted duplicate class definition error

use array instead of list

Co-authored-by: jean-philippe.bempel <jean-philippe.bempel@datadoghq.com>

Author: jpbempel

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerTransformer.java

@@ -95,8 +95,19 @@ public class DebuggerTransformer implements ClassFileTransformer {
           SpanProbe.class);
   private static final String JAVA_IO_TMPDIR = "java.io.tmpdir";
   private static final boolean JAVA_AT_LEAST_19 = JavaVirtualMachine.isJavaVersionAtLeast(19);
-
   public static Path DUMP_PATH = Paths.get(SystemProperties.get(JAVA_IO_TMPDIR), "debugger");
+  private static final String[] SKIPPED_PACKAGES =
+      new String[] {
+        "com/datadog/debugger/agent/",
+        "com/datadog/debugger/codeorigin/",
+        "com/datadog/debugger/exception/",
+        "com/datadog/debugger/instrumentation/",
+        "com/datadog/debugger/probe/",
+        "com/datadog/debugger/sink/",
+        "com/datadog/debugger/symbol/",
+        "com/datadog/debugger/uploader/",
+        "com/datadog/debugger/util/"
+      };
 
   private final Config config;
   private final TransformerDefinitionMatcher definitionMatcher;
@@ -331,6 +342,16 @@ private boolean skipInstrumentation(ClassLoader loader, String classFilePath) {
       // in case of anonymous classes
       return true;
     }
+    if (classFilePath.startsWith("com/datadog/debugger/")) {
+      // skip classes/packages that are part of debugger agent to avoid
+      // LinkageError: attempted duplicate class definition
+      // while retransforming a class used by instrumentation
+      for (int i = 0; i < SKIPPED_PACKAGES.length; i++) {
+        if (classFilePath.startsWith(SKIPPED_PACKAGES[i])) {
+          return true;
+        }
+      }
+    }
     return false;
   }
 

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/CapturedSnapshotTest.java

@@ -24,6 +24,7 @@
 import static utils.InstrumentationTestHelper.compileAndLoadClass;
 import static utils.InstrumentationTestHelper.getLineForLineProbe;
 import static utils.InstrumentationTestHelper.loadClass;
+import static utils.TestClassFileHelper.getClassFileBytes;
 import static utils.TestHelper.getFixtureContent;
 import static utils.TestHelper.setFieldInConfig;
 
@@ -32,6 +33,8 @@
 import com.datadog.debugger.el.ValueScript;
 import com.datadog.debugger.el.values.StringValue;
 import com.datadog.debugger.instrumentation.InstrumentationResult;
+import com.datadog.debugger.instrumentation.Types;
+import com.datadog.debugger.probe.CodeOriginProbe;
 import com.datadog.debugger.probe.LogProbe;
 import com.datadog.debugger.probe.MetricProbe;
 import com.datadog.debugger.probe.SpanDecorationProbe;
@@ -3091,6 +3094,44 @@ public void methodParametersAttributeRecord() throws IOException, URISyntaxExcep
     }
   }
 
+  /*
+   * Regression test for: DatadogClassLoader attempted duplicate class definition for
+   * com.datadog.debugger.instrumentation.Types (LinkageError).
+   *
+   * When a CodeOriginProbe matches the agent's Types class (by FQN or simple name), and Types
+   * is being loaded for the first time by DatadogClassLoader, DebuggerTransformer.transform() is
+   * invoked as a ClassFileTransformer. Inside performInstrumentation(), CodeOriginInstrumenter
+   * accesses the static field Types.DEBUGGER_CONTEXT_TYPE, triggering a re-entrant loadClass()
+   * call for Types on the same thread. Because Java's synchronized is reentrant and Types is not
+   * yet registered in the JVM (defineClass hasn't completed), findLoadedClass() returns null and
+   * defineClass is called a second time, producing the LinkageError.
+   */
+  @Test
+  public void noInstrumentationForAgentClasses() throws Exception {
+    // Install a CodeOriginProbe targeting the agent's Types class by FQN.
+    // This simulates a probe accidentally matching an agent class (e.g. via simple-name fallback
+    // in TransformerDefinitionMatcher when a user class is also named "Types").
+    CodeOriginProbe probe =
+        new CodeOriginProbe(
+            PROBE_ID,
+            true,
+            Where.of("com.datadog.debugger.instrumentation.Types", "descriptorToSignature", null));
+    installProbes(probe);
+    byte[] typeBytes = getClassFileBytes(Types.class);
+    // transform() proceeds to performInstrumentation(), which calls
+    // CodeOriginInstrumenter.codeOriginCall() → accesses Types.DEBUGGER_CONTEXT_TYPE.
+    // In production (when Types is not yet loaded), this re-enters DatadogClassLoader.loadClass()
+    // and triggers LinkageError: duplicate class definition for Types.
+    byte[] result =
+        currentTransformer.transform(
+            Types.class.getClassLoader(),
+            "com/datadog/debugger/instrumentation/Types",
+            null,
+            null,
+            typeBytes);
+    assertNull(result);
+  }
+
   private TestSnapshotListener setupInstrumentTheWorldTransformer(
       String excludeFileName, String includeFileName) {
     Config config = mock(Config.class);

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/DebuggerTransformerTest.java

@@ -306,8 +306,8 @@ public void classBeingRedefinedNull() {
       disabledReason = "Issue with J9: Flaky")
   public void classGenerationFailed() {
     Config config = createConfig();
-    final String CLASS_NAME = DebuggerAgent.class.getTypeName();
-    final String METHOD_NAME = "run";
+    final String CLASS_NAME = ArrayList.class.getTypeName();
+    final String METHOD_NAME = "add";
     MockProbe mockProbe = MockProbe.builder(PROBE_ID).where(CLASS_NAME, METHOD_NAME).build();
     LogProbe logProbe1 =
         LogProbe.builder().probeId("logprobe1", 0).where(CLASS_NAME, METHOD_NAME).build();
@@ -332,10 +332,10 @@ public void classGenerationFailed() {
     byte[] newClassBuffer =
         debuggerTransformer.transform(
             ClassLoader.getSystemClassLoader(),
-            "com/datadog/debugger/agent/DebuggerAgent",
+            "java/util/ArrayList",
             null,
             null,
-            getClassFileBytes(DebuggerAgent.class));
+            getClassFileBytes(ArrayList.class));
     assertNull(newClassBuffer);
     ArgumentCaptor<String> strCaptor = ArgumentCaptor.forClass(String.class);
     ArgumentCaptor<ProbeId> probeIdCaptor = ArgumentCaptor.forClass(ProbeId.class);