loadtest-controller: add more details to the report for retries

Author: ivchicano

e2e-tests/README.md

@@ -117,7 +117,7 @@ All smoke tests should:
 - "Sessions Created"
 - "Total Participants"
 - "User Connections" (mandatory)
-- Table columns: User, Session, Join Date, Disconnect Date, Retries
+ - Table columns: User, Session, Join Date, Retries, Retry Details
 - Two user rows (User1 and User2) present
 
 If validation fails, the result files are kept in the `results/` directory for debugging.

e2e-tests/scripts/validate-default.sh

@@ -96,16 +96,16 @@ if [ -f "$RESULTS_DIR/results.txt" ]; then
 
         if grep -q "User Connections" "$HTML_FILE"; then
             echo "✓ HTML contains 'User Connections'"
-            if grep -q "Join Date" "$HTML_FILE" && grep -q "Disconnect Date" "$HTML_FILE" && grep -q "Retries" "$HTML_FILE"; then
-                echo "✓ HTML contains new table columns"
+            if grep -q "Join Date" "$HTML_FILE" && grep -q "Retries" "$HTML_FILE" && grep -q "Retry Details" "$HTML_FILE"; then
+                echo "✓ HTML contains expected table columns"
                 if grep -q "User1" "$HTML_FILE" && grep -q "User2" "$HTML_FILE"; then
                     echo "✓ HTML contains two user rows"
                 else
                     echo "✗ HTML missing user rows (expected User1 and User2)"
                     HTML_VALIDATION_PASSED=false
                 fi
             else
-                echo "✗ HTML missing new table columns"
+                echo "✗ HTML missing expected table columns (Join Date, Retries, Retry Details)"
                 HTML_VALIDATION_PASSED=false
             fi
         else

loadtest-controller/src/main/java/io/openvidu/loadtest/models/testcase/ResultReport.java

@@ -7,10 +7,13 @@
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
+import java.util.LinkedHashMap;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import io.openvidu.loadtest.services.BrowserEmulatorClient.RetryAttempt;
+
 public class ResultReport {
 
     private static Logger log = LoggerFactory.getLogger(ResultReport.class);
@@ -42,7 +45,6 @@ public class ResultReport {
     private double retrySuccessRate = 0.0;
     private double avgRetriesPerParticipant = 0.0;
     private int maxRetriesInSingleParticipant = 0;
-    private Map<String, Integer> errorCounts = new TreeMap<>();
     private Map<String, Double> workerCpuAvg = new TreeMap<>();
     private Map<String, Double> workerCpuMax = new TreeMap<>();
     private Map<String, Integer> workerStreams = new TreeMap<>();
@@ -51,6 +53,7 @@ public class ResultReport {
     private Map<String, Calendar> userSuccessTimestamps = new TreeMap<>(); // key: "session-user"
     private Map<String, Integer> userRetryCounts = new TreeMap<>(); // key: "session-user"
     private Map<String, Calendar> userDisconnectTimestamps = new TreeMap<>(); // key: "session-user"
+    private Map<String, List<RetryAttempt>> userRetryAttempts = new LinkedHashMap<>(); // key: "session-user"
 
     public ResultReport() {
     }
@@ -63,9 +66,9 @@ public ResultReport build() {
                 this.kibanaUrl, this.s3BucketName, this.timePerWorker, this.timePerRecordingWorker,
                 this.userStartTimes, this.participantResponses, this.totalRetries, this.successfulRetries,
                 this.retrySuccessRate, this.avgRetriesPerParticipant, this.maxRetriesInSingleParticipant,
-                this.errorCounts, this.workerCpuAvg, this.workerCpuMax, this.workerStreams, this.workerParticipants,
+                this.workerCpuAvg, this.workerCpuMax, this.workerStreams, this.workerParticipants,
                 this.userStartDelaysPercentiles, this.userDisconnectTimestamps,
-                this.userSuccessTimestamps, this.userRetryCounts);
+                this.userSuccessTimestamps, this.userRetryCounts, this.userRetryAttempts);
     }
 
     public ResultReport setManualParticipantAllocation(boolean isManualParticipantAllocation) {
@@ -188,11 +191,6 @@ public ResultReport setMaxRetriesInSingleParticipant(int max) {
         return this;
     }
 
-    public ResultReport setErrorCounts(Map<String, Integer> errorCounts) {
-        this.errorCounts = errorCounts;
-        return this;
-    }
-
     public ResultReport setWorkerCpuStats(Map<String, Double> avg, Map<String, Double> max) {
         this.workerCpuAvg = avg;
         this.workerCpuMax = max;
@@ -230,14 +228,12 @@ public ResultReport setUserDisconnectTimestamps(Map<String, Calendar> userDiscon
         return this;
     }
 
+    public ResultReport setUserRetryAttempts(Map<String, List<RetryAttempt>> userRetryAttempts) {
+        this.userRetryAttempts = userRetryAttempts;
+        return this;
+    }
+
     private void computeAggregates() {
-        // Compute error counts from participantResponses
-        errorCounts.clear();
-        for (CreateParticipantResponse resp : participantResponses) {
-            if (!resp.isResponseOk() && resp.getStopReason() != null) {
-                errorCounts.merge(resp.getStopReason(), 1, Integer::sum);
-            }
-        }
         // Compute per-worker and global CPU stats
         Map<String, List<Double>> cpuPerWorker = new TreeMap<>();
         double cpuSum = 0;
@@ -299,11 +295,6 @@ private double getPercentile(List<Long> sorted, double percentile) {
         return sorted.get(index);
     }
 
-    // Getters for new fields
-    public Map<String, Integer> getErrorCounts() {
-        return errorCounts;
-    }
-
     public Map<String, Double> getWorkerCpuAvg() {
         return workerCpuAvg;
     }
@@ -328,6 +319,10 @@ public Map<String, Calendar> getUserDisconnectTimestamps() {
         return userDisconnectTimestamps;
     }
 
+    public Map<String, List<RetryAttempt>> getUserRetryAttempts() {
+        return userRetryAttempts;
+    }
+
     public int getTotalRetries() {
         return totalRetries;
     }
@@ -360,10 +355,11 @@ private ResultReport(int totalParticipants, int numSessionsCompleted, int numSes
             List<Long> timePerRecordingWorker, Map<Calendar, List<String>> userStartTimes,
             List<CreateParticipantResponse> participantResponses, int totalRetries, int successfulRetries,
             double retrySuccessRate, double avgRetriesPerParticipant, int maxRetriesInSingleParticipant,
-            Map<String, Integer> errorCounts, Map<String, Double> workerCpuAvg, Map<String, Double> workerCpuMax,
+            Map<String, Double> workerCpuAvg, Map<String, Double> workerCpuMax,
             Map<String, Integer> workerStreams, Map<String, Integer> workerParticipants,
             double[] userStartDelaysPercentiles, Map<String, Calendar> userDisconnectTimestamps,
-            Map<String, Calendar> userSuccessTimestamps, Map<String, Integer> userRetryCounts) {
+            Map<String, Calendar> userSuccessTimestamps, Map<String, Integer> userRetryCounts,
+            Map<String, List<RetryAttempt>> userRetryAttempts) {
         this.totalParticipants = totalParticipants;
         this.numSessionsCompleted = numSessionsCompleted;
         this.numSessionsCreated = numSessionsCreated;
@@ -389,7 +385,6 @@ private ResultReport(int totalParticipants, int numSessionsCompleted, int numSes
         this.retrySuccessRate = retrySuccessRate;
         this.avgRetriesPerParticipant = avgRetriesPerParticipant;
         this.maxRetriesInSingleParticipant = maxRetriesInSingleParticipant;
-        this.errorCounts = errorCounts;
         this.workerCpuAvg = workerCpuAvg;
         this.workerCpuMax = workerCpuMax;
         this.workerStreams = workerStreams;
@@ -398,6 +393,7 @@ private ResultReport(int totalParticipants, int numSessionsCompleted, int numSes
         this.userDisconnectTimestamps = userDisconnectTimestamps;
         this.userSuccessTimestamps = userSuccessTimestamps;
         this.userRetryCounts = userRetryCounts;
+        this.userRetryAttempts = userRetryAttempts;
     }
 
     private String getDuration() {

loadtest-controller/src/main/java/io/openvidu/loadtest/services/BrowserEmulatorClient.java

@@ -5,6 +5,7 @@
 import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Calendar;
 import java.util.Map;
@@ -66,6 +67,7 @@ public class BrowserEmulatorClient {
     private WorkerUrlResolver workerUrlResolver;
 
     private ConcurrentHashMap<String, ConcurrentHashMap<String, AtomicInteger>> clientFailures = new ConcurrentHashMap<>();
+    private ConcurrentHashMap<String, ConcurrentHashMap<String, List<RetryAttempt>>> clientRetryAttempts = new ConcurrentHashMap<>();
     private ConcurrentHashMap<String, ConcurrentHashMap<String, Role>> clientRoles = new ConcurrentHashMap<>();
     private ConcurrentHashMap<String, TestCase> participantTestCases = new ConcurrentHashMap<>();
     private ConcurrentHashMap<String, AtomicBoolean> participantConnecting = new ConcurrentHashMap<>();
@@ -81,6 +83,33 @@ public class BrowserEmulatorClient {
 
     private String httpProtocolPrefix;
 
+    public static class RetryAttempt {
+        private final int attemptNumber;
+        private final Calendar errorTimestamp;
+        private Calendar reconnectTimestamp;
+
+        public RetryAttempt(int attemptNumber, Calendar errorTimestamp) {
+            this.attemptNumber = attemptNumber;
+            this.errorTimestamp = errorTimestamp;
+        }
+
+        public void setReconnectTimestamp(Calendar reconnectTimestamp) {
+            this.reconnectTimestamp = reconnectTimestamp;
+        }
+
+        public int getAttemptNumber() {
+            return attemptNumber;
+        }
+
+        public Calendar getErrorTimestamp() {
+            return errorTimestamp;
+        }
+
+        public Calendar getReconnectTimestamp() {
+            return reconnectTimestamp;
+        }
+    }
+
     public BrowserEmulatorClient(LoadTestConfig loadTestConfig, CustomHttpClient httpClient, JsonUtils jsonUtils,
             Sleeper sleeper, WorkerUrlResolver workerUrlResolver) {
         this.loadTestConfig = loadTestConfig;
@@ -94,6 +123,7 @@ public BrowserEmulatorClient(LoadTestConfig loadTestConfig, CustomHttpClient htt
     public void clean() {
         this.isClean.set(true);
         this.clientFailures.clear();
+        this.clientRetryAttempts.clear();
         this.clientRoles.clear();
         this.participantTestCases.clear();
         this.participantConnecting.clear();
@@ -227,8 +257,7 @@ public void addClientFailure(String workerUrl, String participant, String sessio
                 log.debug("Stop reconnecting participant {} in session {}", participant, session);
                 this.lastErrorReconnectingResponse = new CreateParticipantResponse()
                         .setResponseOk(false)
-                        .setStopReason("Participant " + participant + "-" + session + " failed after "
-                                + newFailures + " retries");
+                        .setStopReason(this.buildParticipantFailureReason(participant, session, newFailures, true));
             }
         }
     }
@@ -266,23 +295,27 @@ private void afterDisconnect(String workerUrl, String participant, String sessio
         try {
             ConcurrentHashMap<String, Role> workerRoles = this.clientRoles.get(workerUrl);
             if (workerRoles == null) {
-                log.debug("Worker roles is null for {} in session {} in {}. Waiting ...", participant, session, workerUrl);
+                log.debug("Worker roles is null for {} in session {} in {}. Waiting ...", participant, session,
+                        workerUrl);
                 sleeper.sleep(WAIT_S, null);
                 this.afterDisconnect(workerUrl, participant, session);
                 return;
             }
             Role role = workerRoles.get(user);
             if (role == null) {
-                log.warn("Role is null for {} in session {} in {}. This worker may not have this participant.", participant, session, workerUrl);
+                log.warn("Role is null for {} in session {} in {}. This worker may not have this participant.",
+                        participant, session, workerUrl);
                 return;
             }
             int userNumber = Integer.parseInt(participant.replace(loadTestConfig.getUserNamePrefix(), ""));
             int sessionNumber = Integer.parseInt(session.replace(loadTestConfig.getSessionNamePrefix(), ""));
             CreateParticipantResponse response = null;
             if (role.equals(Role.PUBLISHER)) {
-                response = this.createPublisher(workerUrl, userNumber, sessionNumber, this.participantTestCases.get(user));
+                response = this.createPublisher(workerUrl, userNumber, sessionNumber,
+                        this.participantTestCases.get(user));
             } else {
-                response = this.createSubscriber(workerUrl, userNumber, sessionNumber, this.participantTestCases.get(user));
+                response = this.createSubscriber(workerUrl, userNumber, sessionNumber,
+                        this.participantTestCases.get(user));
             }
             if (response.isResponseOk()) {
                 this.participantReconnecting.remove(user);
@@ -295,7 +328,7 @@ private void afterDisconnect(String workerUrl, String participant, String sessio
         }
     }
 
-    private HttpResponse<String> disconnectUser(String workerUrl, String participant, String session) {
+    private HttpResponse<String> disconnectUser(String workerUrl, String participant, String session) {
         try {
             log.info("Deleting participant {} from worker {}", participant, workerUrl);
             Map<String, String> headers = new HashMap<>();
@@ -464,22 +497,31 @@ private CreateParticipantResponse createParticipant(String workerUrl, int userNu
                         key -> new ConcurrentHashMap<>());
                 AtomicInteger userFailures = failuresMap.computeIfAbsent(user, key -> new AtomicInteger(0));
                 int failures = userFailures.incrementAndGet();
+
+                ConcurrentHashMap<String, List<RetryAttempt>> retryAttemptsMap = this.clientRetryAttempts
+                        .computeIfAbsent(workerUrl, key -> new ConcurrentHashMap<>());
+                List<RetryAttempt> userAttempts = retryAttemptsMap.computeIfAbsent(user, key -> new ArrayList<>());
+                Calendar errorTime = Calendar.getInstance();
+                userAttempts.add(new RetryAttempt(failures, errorTime));
                 log.error("Participant {} in session {} failed {} times", userId, sessionId, failures);
                 sleeper.sleep(WAIT_S, null);
                 if (!loadTestConfig.isRetryMode() || isResponseLimitReached(failures) || endOfTest.get()) {
-                    String reason = "Participant " + userId + "-" + sessionId + " failed after "
-                            + failures + " retries";
-                    // Set lastErrorReconnectingResponse to trigger test termination
+                    boolean isReconnecting = this.participantReconnecting.contains(user);
+                    String reason = this.buildParticipantFailureReason(userId, sessionId, failures, isReconnecting);
                     this.lastErrorReconnectingResponse = new CreateParticipantResponse()
                             .setResponseOk(false)
                             .setStopReason(reason);
-                    // Also set stopReason on the returned response to avoid race condition
-                    // where getLastResponse() may return this object before lastErrorReconnectingResponse is visible
                     return cpr.setResponseOk(false).setStopReason(reason);
                 }
                 log.warn("Retrying");
                 return this.createParticipant(workerUrl, userNumber, sessionNumber, testCase, role);
             } else {
+                ConcurrentHashMap<String, List<RetryAttempt>> retryAttemptsMap = this.clientRetryAttempts
+                        .computeIfAbsent(workerUrl, key -> new ConcurrentHashMap<>());
+                List<RetryAttempt> userAttempts = retryAttemptsMap.computeIfAbsent(user, key -> new ArrayList<>());
+                if (!userAttempts.isEmpty()) {
+                    userAttempts.get(userAttempts.size() - 1).setReconnectTimestamp(Calendar.getInstance());
+                }
                 this.participantConnecting.get(user).set(false);
                 this.saveParticipantData(workerUrl, testCase.isTeaching() ? Role.PUBLISHER : role);
             }
@@ -580,6 +622,15 @@ private boolean isResponseLimitReached(int failures) {
         return failures == loadTestConfig.getRetryTimes();
     }
 
+    private String buildParticipantFailureReason(String participant, String session, int attempts,
+            boolean reconnecting) {
+        if (reconnecting) {
+            return "Participant " + participant + "-" + session + " failed to reconnect after " + attempts
+                    + " attempts";
+        }
+        return "Participant " + participant + "-" + session + " failed after " + attempts + " retries";
+    }
+
     private CreateUserRequestBody generateRequestBody(int userNumber, String sessionNumber, Role role,
             TestCase testCase) {
         boolean video = (testCase.isTeaching() && role.equals(Role.PUBLISHER)) || !testCase.isTeaching();
@@ -721,6 +772,22 @@ public Map<String, Integer> getPerUserRetryCounts() {
         return counts;
     }
 
+    public Map<String, List<RetryAttempt>> getPerUserRetryAttempts() {
+        Map<String, List<RetryAttempt>> result = new LinkedHashMap<>();
+        for (ConcurrentHashMap<String, List<RetryAttempt>> userAttempts : clientRetryAttempts.values()) {
+            for (Map.Entry<String, List<RetryAttempt>> entry : userAttempts.entrySet()) {
+                String userSession = entry.getKey();
+                List<RetryAttempt> attempts = entry.getValue();
+                result.merge(userSession, attempts, (existing, incoming) -> {
+                    existing.addAll(incoming);
+                    existing.sort((a, b) -> Integer.compare(a.getAttemptNumber(), b.getAttemptNumber()));
+                    return existing;
+                });
+            }
+        }
+        return result;
+    }
+
     public void shutdownWorkers(List<String> workerUrls, boolean waitForResponse) {
         if (workerUrls == null || workerUrls.isEmpty()) {
             return;

loadtest-controller/src/main/java/io/openvidu/loadtest/services/core/LoadTestService.java

@@ -359,10 +359,11 @@ private void saveResultReport(TestCase testCase, String participantsBySession, C
                 .setTimePerWorker(shutdownOrchestrator.getWorkerTimes())
                 .setTimePerRecordingWorker(shutdownOrchestrator.getRecordingWorkerTimes())
                 .setUserStartTimes(participantOrchestrator.getUserStartTimes())
-                .setUserSuccessTimestamps(userSuccessTimestamps)
-                .setParticipantResponses(participantOrchestrator.getAllParticipantResponses())
-                .setUserRetryCounts(browserEmulatorClient.getPerUserRetryCounts())
-                .build();
+                .setUserSuccessTimestamps(userSuccessTimestamps)
+                .setParticipantResponses(participantOrchestrator.getAllParticipantResponses())
+                .setUserRetryCounts(browserEmulatorClient.getPerUserRetryCounts())
+                .setUserRetryAttempts(browserEmulatorClient.getPerUserRetryAttempts())
+                .build();
 
         io.exportResults(rr);