diff --git a/dd-java-agent/src/test/groovy/datadog/trace/agent/InstrumenterUnloadTest.groovy b/dd-java-agent/src/test/groovy/datadog/trace/agent/InstrumenterUnloadTest.groovy
index 3e469c62b53..09097926279 100644
--- a/dd-java-agent/src/test/groovy/datadog/trace/agent/InstrumenterUnloadTest.groovy
+++ b/dd-java-agent/src/test/groovy/datadog/trace/agent/InstrumenterUnloadTest.groovy
@@ -26,16 +26,26 @@ class InstrumenterUnloadTest extends Specification {
       , ["DD_API_KEY": API_KEY]
       , new PrintStream(testOutput))
 
-    int unloadCount = 0
+    boolean canaryUnloaded = false
+    int unloadedInstrumentationCount = 0
     new ByteArrayInputStream((testOutput.toByteArray())).eachLine {
       System.out.println(it)
+      if (it =~ /(?i)unload.*Canary/) {
+        canaryUnloaded = true
+      }
       if (it =~ /(?i)unload.* datadog.trace.instrumentation./) {
-        unloadCount++
+        unloadedInstrumentationCount++
       }
     }
 
+    if (!canaryUnloaded) {
+      System.out.println("WARNING: Canary class was not unloaded!")
+    }
+
     then:
     returnCode == 0
-    unloadCount > 0
+    // skip check if we couldn't even unload our Canary class, as that
+    // indicates full GC didn't happen enough to trigger any unloading
+    !canaryUnloaded || unloadedInstrumentationCount > 0
   }
 }
diff --git a/dd-java-agent/src/test/java/jvmbootstraptest/UnloadingChecker.java b/dd-java-agent/src/test/java/jvmbootstraptest/UnloadingChecker.java
index 28137824b81..930735a20df 100644
--- a/dd-java-agent/src/test/java/jvmbootstraptest/UnloadingChecker.java
+++ b/dd-java-agent/src/test/java/jvmbootstraptest/UnloadingChecker.java
@@ -1,13 +1,32 @@
 package jvmbootstraptest;
 
+import static java.util.concurrent.TimeUnit.MINUTES;
+
 import datadog.trace.test.util.GCUtils;
+import java.lang.management.ClassLoadingMXBean;
+import java.lang.management.ManagementFactory;
 
 public class UnloadingChecker {
-  public static void main(final String[] args) {
-    try {
-      GCUtils.awaitGC();
-    } catch (InterruptedException e) {
-      e.printStackTrace();
+  static class Canary {}
+
+  public static void main(final String[] args) throws Exception {
+    ClassLoadingMXBean classLoadingMXBean = ManagementFactory.getClassLoadingMXBean();
+    long initialUnloadCount = classLoadingMXBean.getUnloadedClassCount();
+
+    // load an isolated class which we know can be unloaded after a full GC
+    new IsolatingClassLoader().loadClass("jvmbootstraptest.UnloadingChecker$Canary");
+
+    long waitNanos = MINUTES.toNanos(2);
+    long startNanos = System.nanoTime();
+
+    while (System.nanoTime() - startNanos < waitNanos) {
+      try {
+        GCUtils.awaitGC();
+      } catch (Throwable ignore) {
+      }
+      if (initialUnloadCount < classLoadingMXBean.getUnloadedClassCount()) {
+        break; // some class unloading has taken place, stop and check results
+      }
     }
   }
 }