HDDS-15582 : Further speed up TestUnhealthyContainersDerbyPerformance

Author: umesh

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerHealthSchemaManager.java

@@ -53,7 +53,7 @@
 public class ContainerHealthSchemaManager {
   private static final Logger LOG =
       LoggerFactory.getLogger(ContainerHealthSchemaManager.class);
-  private static final int BATCH_INSERT_CHUNK_SIZE = 1000;
+  private static final int BATCH_INSERT_CHUNK_SIZE = 5_000;
 
   /**
    * Maximum number of container IDs to include in a single
@@ -205,28 +205,75 @@ public void replaceUnhealthyContainerRecordsAtomically(
 
   private int deleteScmStatesForContainers(DSLContext dslContext,
       List<Long> containerIds) {
+    if (containerIds.isEmpty()) {
+      return 0;
+    }
+
+    List<Long> sortedIds = containerIds.stream()
+        .distinct()
+        .sorted()
+        .collect(Collectors.toList());
+
     int totalDeleted = 0;
+    List<Long> inClauseBatch = new ArrayList<>(MAX_IN_CLAUSE_CHUNK_SIZE);
+
+    for (int i = 0; i < sortedIds.size(); ) {
+      int rangeStart = i;
+      while (i + 1 < sortedIds.size()
+          && sortedIds.get(i + 1) == sortedIds.get(i) + 1) {
+        i++;
+      }
+
+      if (i > rangeStart) {
+        if (!inClauseBatch.isEmpty()) {
+          totalDeleted += deleteScmStatesForContainerIdInClause(dslContext, inClauseBatch);
+          inClauseBatch.clear();
+        }
+        totalDeleted += deleteScmStatesForContainerIdRange(dslContext,
+            sortedIds.get(rangeStart), sortedIds.get(i));
+      } else {
+        inClauseBatch.add(sortedIds.get(i));
+        if (inClauseBatch.size() >= MAX_IN_CLAUSE_CHUNK_SIZE) {
+          totalDeleted += deleteScmStatesForContainerIdInClause(dslContext, inClauseBatch);
+          inClauseBatch.clear();
+        }
+      }
+      i++;
+    }
 
-    for (int from = 0; from < containerIds.size(); from += MAX_IN_CLAUSE_CHUNK_SIZE) {
-      int to = Math.min(from + MAX_IN_CLAUSE_CHUNK_SIZE, containerIds.size());
-      List<Long> chunk = containerIds.subList(from, to);
-
-      int deleted = dslContext.deleteFrom(UNHEALTHY_CONTAINERS)
-          .where(UNHEALTHY_CONTAINERS.CONTAINER_ID.in(chunk))
-          .and(UNHEALTHY_CONTAINERS.CONTAINER_STATE.in(
-              UnHealthyContainerStates.MISSING.toString(),
-              UnHealthyContainerStates.EMPTY_MISSING.toString(),
-              UnHealthyContainerStates.UNDER_REPLICATED.toString(),
-              UnHealthyContainerStates.OVER_REPLICATED.toString(),
-              UnHealthyContainerStates.MIS_REPLICATED.toString(),
-              UnHealthyContainerStates.NEGATIVE_SIZE.toString(),
-              UnHealthyContainerStates.REPLICA_MISMATCH.toString()))
-          .execute();
-      totalDeleted += deleted;
+    if (!inClauseBatch.isEmpty()) {
+      totalDeleted += deleteScmStatesForContainerIdInClause(dslContext, inClauseBatch);
     }
     return totalDeleted;
   }
 
+  private int deleteScmStatesForContainerIdRange(DSLContext dslContext,
+      long startContainerId, long endContainerId) {
+    return dslContext.deleteFrom(UNHEALTHY_CONTAINERS)
+        .where(UNHEALTHY_CONTAINERS.CONTAINER_ID.between(startContainerId, endContainerId))
+        .and(scmGeneratedStateFilter())
+        .execute();
+  }
+
+  private int deleteScmStatesForContainerIdInClause(DSLContext dslContext,
+      List<Long> containerIds) {
+    return dslContext.deleteFrom(UNHEALTHY_CONTAINERS)
+        .where(UNHEALTHY_CONTAINERS.CONTAINER_ID.in(containerIds))
+        .and(scmGeneratedStateFilter())
+        .execute();
+  }
+
+  private org.jooq.Condition scmGeneratedStateFilter() {
+    return UNHEALTHY_CONTAINERS.CONTAINER_STATE.in(
+        UnHealthyContainerStates.MISSING.toString(),
+        UnHealthyContainerStates.EMPTY_MISSING.toString(),
+        UnHealthyContainerStates.UNDER_REPLICATED.toString(),
+        UnHealthyContainerStates.OVER_REPLICATED.toString(),
+        UnHealthyContainerStates.MIS_REPLICATED.toString(),
+        UnHealthyContainerStates.NEGATIVE_SIZE.toString(),
+        UnHealthyContainerStates.REPLICA_MISMATCH.toString());
+  }
+
   /**
    * Returns previous in-state-since timestamps for tracked unhealthy states.
    * The key is a stable containerId + state tuple.