Fix race condition for Failed Test Replay (#10679)

fix: flakiness in junit console FTR test

chore: update comment

Merge branch 'master' into daniel.mohedano/junit-console-flaky-fix

fix: also update flush interval

fix: update approach to requeue

feat: reduce catch scope to assertionerror

Co-authored-by: daniel.mohedano <daniel.mohedano@datadoghq.com>

Author: daniel-mohedano

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/DebuggerSink.java

@@ -92,10 +92,10 @@ public void start() {
   }
 
   public void stop() {
-    lowRateFlush(this);
-    snapshotSink.highRateFlush(null);
     cancelSchedule(this.flushIntervalScheduled);
     cancelSchedule(this.lowRateScheduled);
+    lowRateFlush(this);
+    snapshotSink.highRateFlush(null);
     probeStatusSink.stop();
     symbolSink.stop();
     snapshotSink.stop();

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SnapshotSink.java

@@ -186,6 +186,11 @@ private List<String> getSerializedSnapshots(BlockingQueue<Snapshot> queue, int l
         if (!largeBatch) {
           LOGGER.debug("Sending snapshot for probe: {}", snapshot.getProbe().getId());
         }
+      } catch (AssertionError e) {
+        // Re-queue on AssertionError from Moshi when thread is interrupted so the shutdown hook
+        // flush can retry with a non-interrupted thread
+        ExceptionHelper.logException(LOGGER, e, "Error during snapshot serialization:");
+        queue.offer(snapshot);
       } catch (Exception e) {
         ExceptionHelper.logException(LOGGER, e, "Error during snapshot serialization:");
       }

dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy

@@ -433,7 +433,7 @@ class MockBackend implements AutoCloseable {
   }
 
   List<Map<String, Object>> waitForLogs(int expectedCount) {
-    def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1)
+    def traceReceiveConditions = new PollingConditions(timeout: 30, initialDelay: 1, delay: 0.5, factor: 1)
     traceReceiveConditions.eventually {
       assert receivedLogs.size() == expectedCount
     }

dd-smoke-tests/junit-console/src/test/groovy/datadog/smoketest/JUnitConsoleSmokeTest.groovy

@@ -2,6 +2,7 @@ package datadog.smoketest
 
 
 import datadog.trace.api.config.CiVisibilityConfig
+import datadog.trace.api.config.DebuggerConfig
 import datadog.trace.api.config.GeneralConfig
 import datadog.trace.civisibility.CiVisibilitySmokeTest
 import java.nio.file.FileVisitResult
@@ -52,7 +53,8 @@ class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest {
 
     def exitCode = whenRunningJUnitConsole([
       (CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_COUNT): "3",
-      (GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL): mockBackend.intakeUrl
+      (GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL): mockBackend.intakeUrl,
+      (DebuggerConfig.DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL): "999999" // avoid possible race conditions on shutdown
     ],
     [:])
     assert exitCode == 1