Add an error sampler for Log probes (#10638)

Add an error sampler for Log probes

log template only probe are rate limited to a higher rate. if condition
is set and returns an error, we don't want to return a snapshot
at the same rate just for reporting the condition evaluation failure.
So we are introducing a specific sampler at 1/s rate to report error
in that case.

Co-authored-by: jean-philippe.bempel <jean-philippe.bempel@datadoghq.com>

Author: jpbempel

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java

@@ -325,6 +325,7 @@ public String toString() {
   protected transient Map<DDTraceId, AtomicInteger> budget =
       Collections.synchronizedMap(new WeakIdentityHashMap<>());
   protected transient Sampler sampler;
+  protected transient Sampler errorSampler;
 
   // no-arg constructor is required by Moshi to avoid creating instance with unsafe and by-passing
   // constructors, including field initializers.
@@ -461,6 +462,7 @@ public void initSamplers() {
                 ? ProbeRateLimiter.DEFAULT_SNAPSHOT_RATE
                 : ProbeRateLimiter.DEFAULT_LOG_RATE);
     sampler = ProbeRateLimiter.createSampler(rate);
+    errorSampler = ProbeRateLimiter.createSampler(1.0); // errors are always sampled at 1/s rate
   }
 
   public List<CaptureExpression> getCaptureExpressions() {
@@ -565,9 +567,13 @@ private void sample(LogStatus logStatus, MethodLocation methodLocation) {
     if (!MethodLocation.isSame(methodLocation, evaluateAt)) {
       return;
     }
+    // if condition has error and no capture Snapshot, the error is reported using errorSampler
+    // at 1/s rate instead of the log template one
+    Sampler localSampler =
+        logStatus.hasConditionErrors && !isCaptureSnapshot() ? errorSampler : sampler;
     boolean sampled =
         !logStatus.getDebugSessionStatus().isDisabled()
-            && ProbeRateLimiter.tryProbe(sampler, isCaptureSnapshot());
+            && ProbeRateLimiter.tryProbe(localSampler, isCaptureSnapshot());
     logStatus.setSampled(sampled);
     if (!sampled) {
       DebuggerAgent.getSink()

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/CapturedSnapshotTest.java

@@ -75,6 +75,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.DoubleConsumer;
 import java.util.stream.Collectors;
 import org.jetbrains.kotlin.com.intellij.util.lang.JavaVersion;
 import org.joor.Reflect;
@@ -1299,6 +1300,35 @@ public void nullCondition() throws IOException, URISyntaxException {
     assertEquals("Cannot dereference field: fld", snapshot.getMessage());
   }
 
+  @Test
+  public void nullConditionTemplateOnly() throws IOException, URISyntaxException {
+    final String CLASS_NAME = "CapturedSnapshot08";
+    LogProbe logProbes =
+        createProbeBuilder(PROBE_ID, CLASS_NAME, "doit", "int (java.lang.String)")
+            .when(
+                new ProbeCondition(
+                    DSL.when(
+                        DSL.eq(
+                            DSL.getMember(
+                                DSL.getMember(DSL.getMember(DSL.ref("nullTyped"), "fld"), "fld"),
+                                "msg"),
+                            DSL.value("hello"))),
+                    "nullTyped.fld.fld.msg == 'hello'"))
+            .captureSnapshot(false)
+            .template("plain log", Collections.emptyList())
+            .build();
+    TestSnapshotListener listener = installProbes(logProbes);
+    Class<?> testClass = compileAndLoadClass(CLASS_NAME);
+    int result = Reflect.onClass(testClass).call("main", "1").get();
+    assertEquals(3, result);
+    Snapshot snapshot = assertOneSnapshot(listener);
+    assertEquals("Cannot dereference field: fld", snapshot.getMessage());
+    List<EvaluationError> evaluationErrors = snapshot.getEvaluationErrors();
+    assertEquals(1, evaluationErrors.size());
+    assertEquals("nullTyped.fld.fld", evaluationErrors.get(0).getExpr());
+    assertEquals("Cannot dereference field: fld", evaluationErrors.get(0).getMessage());
+  }
+
   @Test
   public void shortCircuitingCondition() throws IOException, URISyntaxException {
     final String CLASS_NAME = "CapturedSnapshot08";
@@ -2679,19 +2709,47 @@ public void ensureCallingSamplingLineProbeCondition() throws IOException, URISyn
     doSamplingTest(this::lineProbeCondition, 1, 1);
   }
 
+  @Test
+  public void ensureCallingSamplingLogTemplateOnlyConditionError()
+      throws IOException, URISyntaxException {
+    doSamplingTest(this::nullConditionTemplateOnly, ProbeRateLimiter::setGlobalLogRate, 1, 0, 1);
+  }
+
   private void doSamplingTest(TestMethod testRun, int expectedGlobalCount, int expectedProbeCount)
       throws IOException, URISyntaxException {
+    doSamplingTest(
+        testRun,
+        ProbeRateLimiter::setGlobalSnapshotRate,
+        expectedGlobalCount,
+        expectedProbeCount,
+        0);
+  }
+
+  private void doSamplingTest(
+      TestMethod testRun,
+      DoubleConsumer globalRateSetter,
+      int expectedGlobalCount,
+      int expectedProbeCount,
+      int expectedErrorCount)
+      throws IOException, URISyntaxException {
     MockSampler probeSampler = new MockSampler();
+    MockSampler errorSampler = new MockSampler();
     MockSampler globalSampler = new MockSampler();
-    ProbeRateLimiter.setSamplerSupplier(rate -> rate < 101 ? probeSampler : globalSampler);
-    ProbeRateLimiter.setGlobalSnapshotRate(1000);
+    ProbeRateLimiter.setSamplerSupplier(
+        rate -> {
+          if (rate < 2) return errorSampler;
+          if (rate < 101) return probeSampler;
+          return globalSampler;
+        });
+    globalRateSetter.accept(1000);
     try {
       testRun.run();
     } finally {
       ProbeRateLimiter.setSamplerSupplier(null);
     }
     assertEquals(expectedGlobalCount, globalSampler.getCallCount());
     assertEquals(expectedProbeCount, probeSampler.getCallCount());
+    assertEquals(expectedErrorCount, errorSampler.getCallCount());
   }
 
   @Test