Load JMXFetch substitutions reflectively, increase native-image heap

Using .class literals causes eager class loading, making @TargetClass
annotations visible to GraalVM's annotation processor even when we
conditionally skip adding them to the result list.

By loading the classes reflectively with Class.forName(), we prevent
them from being discovered when JMXFetch is not present.

Also increased native-image heap from 4GB to 8GB to avoid OOM during
compilation with profiler enabled.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: jbachorik

dd-java-agent/instrumentation/graal/graal-native-image-20.0/src/main/java/datadog/trace/instrumentation/graal/nativeimage/AnnotationSubstitutionProcessorInstrumentation.java

@@ -34,9 +34,58 @@ public static class FindTargetClassesAdvice {
     public static void onExit(@Advice.Return(readOnly = false) List<Class<?>> result) {
       result.add(Target_com_datadog_profiling_agent_ProcessContext.class);
       result.add(Target_datadog_jctools_util_UnsafeRefArrayAccess.class);
-      result.add(Target_org_datadog_jmxfetch_App.class);
-      result.add(Target_org_datadog_jmxfetch_Status.class);
-      result.add(Target_org_datadog_jmxfetch_reporter_JsonReporter.class);
+
+      // Only register JMXFetch substitutions if JMXFetch is actually present on the classpath.
+      //
+      // NOTE: It's unclear why these substitutions get triggered during native-image compilation
+      // when JMXFetch classes are not available. In theory, either:
+      // 1) The substitutions should not be triggered at all (JMXFetch not in use), OR
+      // 2) JMXFetch classes should be available (if JMXFetch is in use)
+      //
+      // However, in practice, adding profiling-scrubber to the agent triggers native-image to
+      // discover these substitutions even when building applications that don't use JMXFetch,
+      // causing "Substitution target not loaded" errors.
+      //
+      // This runtime check works around the issue for GraalVM 20.0 (which lacks the `onlyWith`
+      // field in @TargetClass). For GraalVM 21+, the proper fix would be adding:
+      //   @TargetClass(className = "org.datadog.jmxfetch.App", onlyWith = JmxFetchPresent.class)
+      //
+      // IMPORTANT: We must load these classes reflectively (not using .class literals) to prevent
+      // them from being discovered by GraalVM's annotation processor when JMXFetch is not present.
+      // Using .class literals causes eager loading, making @TargetClass annotations visible even
+      // if we don't add them to the result list.
+      if (isJmxFetchPresent()) {
+        try {
+          ClassLoader cl = FindTargetClassesAdvice.class.getClassLoader();
+          result.add(
+              Class.forName(
+                  "datadog.trace.instrumentation.graal.nativeimage.Target_org_datadog_jmxfetch_App",
+                  false,
+                  cl));
+          result.add(
+              Class.forName(
+                  "datadog.trace.instrumentation.graal.nativeimage.Target_org_datadog_jmxfetch_Status",
+                  false,
+                  cl));
+          result.add(
+              Class.forName(
+                  "datadog.trace.instrumentation.graal.nativeimage.Target_org_datadog_jmxfetch_reporter_JsonReporter",
+                  false,
+                  cl));
+        } catch (ClassNotFoundException e) {
+          // Substitution classes not available, skip them
+        }
+      }
+    }
+
+    private static boolean isJmxFetchPresent() {
+      try {
+        Class.forName(
+            "org.datadog.jmxfetch.App", false, FindTargetClassesAdvice.class.getClassLoader());
+        return true;
+      } catch (ClassNotFoundException e) {
+        return false;
+      }
     }
   }
 }

dd-smoke-tests/spring-boot-3.0-native/application/build.gradle

@@ -38,7 +38,7 @@ if (hasProperty('agentPath')) {
         if (withProfiler && property('profiler') == 'true') {
           buildArgs.add("-J-Ddd.profiling.enabled=true")
         }
-        jvmArgs.add("-Xmx4096M")
+        jvmArgs.add("-Xmx8192M")
       }
     }
   }