perf: optimization on using special maps to reduce RAM and GC pressure

Author: lvca

engine/src/main/java/com/arcadedb/graph/olap/DeltaOverlay.java

@@ -19,6 +19,7 @@
 package com.arcadedb.graph.olap;
 
 import com.arcadedb.database.RID;
+import com.arcadedb.utility.IntIntHashMap;
 
 import java.util.*;
 
@@ -58,8 +59,8 @@ class DeltaOverlay {
   private final Map<String, Set<Long>>      deletedEdgesPerType;
 
   // Per-node deleted edge counts for O(1) lookup: edgeType -> nodeId -> count
-  private final Map<String, Map<Integer, Integer>> deletedOutEdgeCounts;
-  private final Map<String, Map<Integer, Integer>> deletedInEdgeCounts;
+  private final Map<String, IntIntHashMap> deletedOutEdgeCounts;
+  private final Map<String, IntIntHashMap> deletedInEdgeCounts;
 
   // Property overrides for base nodes: globalId -> (propName -> value)
   private final Map<Integer, Map<String, Object>> propertyOverrides;
@@ -97,8 +98,8 @@ private DeltaOverlay(final int baseNodeCount,
       final BitSet deletedBaseNodes, final BitSet deletedOverflowNodes,
       final Map<String, List<long[]>> addedEdgesPerType,
       final Map<String, Set<Long>> deletedEdgesPerType,
-      final Map<String, Map<Integer, Integer>> deletedOutEdgeCounts,
-      final Map<String, Map<Integer, Integer>> deletedInEdgeCounts,
+      final Map<String, IntIntHashMap> deletedOutEdgeCounts,
+      final Map<String, IntIntHashMap> deletedInEdgeCounts,
       final Map<Integer, Map<String, Object>> propertyOverrides,
       final Map<String, Map<Integer, int[]>> outNeighborIndex,
       final Map<String, Map<Integer, int[]>> inNeighborIndex,
@@ -204,8 +205,8 @@ DeltaOverlay merge(final TxDelta delta, final NodeIdMapping baseMapping) {
     final Map<String, Map<Integer, int[]>> newInIndex = buildNeighborIndex(newAddedEdges, false);
 
     // Build per-node deleted edge count indexes for O(1) lookup
-    final Map<String, Map<Integer, Integer>> newDelOutCounts = buildDeletedEdgeCounts(newDeletedEdges, true);
-    final Map<String, Map<Integer, Integer>> newDelInCounts = buildDeletedEdgeCounts(newDeletedEdges, false);
+    final Map<String, IntIntHashMap> newDelOutCounts = buildDeletedEdgeCounts(newDeletedEdges, true);
+    final Map<String, IntIntHashMap> newDelInCounts = buildDeletedEdgeCounts(newDeletedEdges, false);
 
     return new DeltaOverlay(baseNodeCount,
         Collections.unmodifiableMap(newOverflowIds),
@@ -266,20 +267,20 @@ boolean isEdgeDeleted(final String edgeType, final int srcId, final int tgtId) {
    * Counts the number of deleted outgoing edges from {@code nodeId} for the given edge type. O(1).
    */
   int countDeletedOutEdges(final int nodeId, final String edgeType) {
-    final Map<Integer, Integer> counts = deletedOutEdgeCounts.get(edgeType);
+    final IntIntHashMap counts = deletedOutEdgeCounts.get(edgeType);
     if (counts == null)
       return 0;
-    return counts.getOrDefault(nodeId, 0);
+    return counts.get(nodeId, 0);
   }
 
   /**
    * Counts the number of deleted incoming edges to {@code nodeId} for the given edge type. O(1).
    */
   int countDeletedInEdges(final int nodeId, final String edgeType) {
-    final Map<Integer, Integer> counts = deletedInEdgeCounts.get(edgeType);
+    final IntIntHashMap counts = deletedInEdgeCounts.get(edgeType);
     if (counts == null)
       return 0;
-    return counts.getOrDefault(nodeId, 0);
+    return counts.get(nodeId, 0);
   }
 
   /**
@@ -355,24 +356,25 @@ private static Map<String, Map<Integer, int[]>> buildNeighborIndex(
       final List<long[]> edges = entry.getValue();
 
       // Pass 1: count neighbors per node
-      final HashMap<Integer, Integer> counts = new HashMap<>();
+      final IntIntHashMap counts = new IntIntHashMap();
       for (final long[] pair : edges) {
         final int key = (int) (outgoing ? pair[0] : pair[1]);
-        counts.merge(key, 1, Integer::sum);
+        counts.increment(key);
       }
 
       // Allocate exact-size arrays
       final Map<Integer, int[]> perNode = new HashMap<>(counts.size());
-      for (final var e : counts.entrySet())
-        perNode.put(e.getKey(), new int[e.getValue()]);
+      counts.forEach((key, count) -> perNode.put(key, new int[count]));
 
-      // Pass 2: fill arrays (reuse counts map as fill-position tracker)
-      counts.replaceAll((k, v) -> 0);
+      // Pass 2: fill arrays (use a fresh map as fill-position tracker)
+      final IntIntHashMap positions = new IntIntHashMap(counts.size());
       for (final long[] pair : edges) {
         final int key = (int) (outgoing ? pair[0] : pair[1]);
         final int neighbor = (int) (outgoing ? pair[1] : pair[0]);
         final int[] arr = perNode.get(key);
-        arr[counts.merge(key, 1, Integer::sum) - 1] = neighbor;
+        final int pos = positions.get(key, 0);
+        arr[pos] = neighbor;
+        positions.put(key, pos + 1);
       }
 
       result.put(entry.getKey(), perNode);
@@ -385,16 +387,16 @@ private static Map<String, Map<Integer, int[]>> buildNeighborIndex(
    *
    * @param outgoing true for outgoing counts (keyed by source), false for incoming (keyed by target)
    */
-  private static Map<String, Map<Integer, Integer>> buildDeletedEdgeCounts(
+  private static Map<String, IntIntHashMap> buildDeletedEdgeCounts(
       final Map<String, Set<Long>> deletedEdges, final boolean outgoing) {
     if (deletedEdges.isEmpty())
       return Collections.emptyMap();
-    final Map<String, Map<Integer, Integer>> result = new HashMap<>();
+    final Map<String, IntIntHashMap> result = new HashMap<>();
     for (final var entry : deletedEdges.entrySet()) {
-      final Map<Integer, Integer> counts = new HashMap<>();
+      final IntIntHashMap counts = new IntIntHashMap();
       for (final long packed : entry.getValue()) {
         final int nodeId = outgoing ? (int) (packed >>> 32) : (int) packed;
-        counts.merge(nodeId, 1, Integer::sum);
+        counts.increment(nodeId);
       }
       result.put(entry.getKey(), counts);
     }

engine/src/main/java/com/arcadedb/query/opencypher/executor/steps/PairHashJoinOp.java

@@ -26,6 +26,7 @@
 import com.arcadedb.graph.Vertex;
 import com.arcadedb.schema.DocumentType;
 import com.arcadedb.schema.VertexType;
+import com.arcadedb.utility.LongLongHashMap;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -96,7 +97,7 @@ public long execute(final GraphTraversalProvider provider, final Database db) {
     // BUILD: for single-hop arms, use NeighborView for direct edge iteration
     // instead of per-node walkArm (avoids ~6M getNeighborIds method calls).
     // For multi-hop arms, fall back to per-node walkArm.
-    final HashMap<Long, Long> pairCounts = new HashMap<>();
+    final LongLongHashMap pairCounts = new LongLongHashMap();
 
     final boolean arm1SingleHop = arm1EdgeTypes.length == 1;
     final boolean arm2SingleHop = arm2EdgeTypes.length == 1;
@@ -145,29 +146,22 @@ public long execute(final GraphTraversalProvider provider, final Database db) {
           continue;
         for (final int ep1 : ep1Ids)
           for (final int ep2 : ep2Ids)
-            pairCounts.merge(CSRCountUtils.packPair(ep1, ep2), 1L, Long::sum);
+            pairCounts.increment(CSRCountUtils.packPair(ep1, ep2));
       }
     }
 
     final NeighborView probeViewFallback = probeView != null ? probeView : provider.getNeighborView(probeDirection, probeEdgeType);
     long total = 0;
     if (probeViewFallback != null) {
       final int[] probeNbrs = probeViewFallback.neighbors();
-      for (int p1 = 0; p1 < nodeCount; p1++) {
-        for (int j = probeViewFallback.offset(p1), end = probeViewFallback.offsetEnd(p1); j < end; j++) {
-          final Long cnt = pairCounts.get(CSRCountUtils.packPair(p1, probeNbrs[j]));
-          if (cnt != null)
-            total += cnt;
-        }
-      }
+      for (int p1 = 0; p1 < nodeCount; p1++)
+        for (int j = probeViewFallback.offset(p1), end = probeViewFallback.offsetEnd(p1); j < end; j++)
+          total += pairCounts.get(CSRCountUtils.packPair(p1, probeNbrs[j]), 0);
     } else {
       for (int p1 = 0; p1 < nodeCount; p1++) {
         final int[] neighbors = provider.getNeighborIds(p1, probeDirection, probeEdgeType);
-        for (final int p2 : neighbors) {
-          final Long cnt = pairCounts.get(CSRCountUtils.packPair(p1, p2));
-          if (cnt != null)
-            total += cnt;
-        }
+        for (final int p2 : neighbors)
+          total += pairCounts.get(CSRCountUtils.packPair(p1, p2), 0);
       }
     }
     return total;
@@ -273,7 +267,7 @@ private long buildAndProbeInline(final NeighborView arm1View, final NeighborView
    * But for simpler pair-joins where both arms are 1 hop, this avoids all per-node method calls.
    */
   private void buildWithViews(final NeighborView arm1View, final NeighborView arm2View,
-      final Set<Integer>[] arm2Buckets, final HashMap<Long, Long> pairCounts,
+      final Set<Integer>[] arm2Buckets, final LongLongHashMap pairCounts,
       final int nodeCount, final GraphTraversalProvider provider) {
     final int[] arm1Nbrs = arm1View.neighbors();
     final int[] arm2Nbrs = arm2View.neighbors();
@@ -287,7 +281,7 @@ private void buildWithViews(final NeighborView arm1View, final NeighborView arm2
 
       for (int i = a1Start; i < a1End; i++)
         for (int j = a2Start; j < a2End; j++)
-          pairCounts.merge(CSRCountUtils.packPair(arm1Nbrs[i], arm2Nbrs[j]), 1L, Long::sum);
+          pairCounts.increment(CSRCountUtils.packPair(arm1Nbrs[i], arm2Nbrs[j]));
     }
   }
 
@@ -297,7 +291,7 @@ private void buildWithViews(final NeighborView arm1View, final NeighborView arm2
    * For Q2: arm1=HAS_CREATOR OUT (Comment→Person), arm2=REPLY_OF+HAS_CREATOR (Comment→Post→Person).
    */
   private void buildWithArm1View(final NeighborView arm1View, final GraphTraversalProvider provider,
-      final Set<Integer>[] arm2Buckets, final HashMap<Long, Long> pairCounts,
+      final Set<Integer>[] arm2Buckets, final LongLongHashMap pairCounts,
       final int nodeCount) {
     final int[] arm1Nbrs = arm1View.neighbors();
 
@@ -332,7 +326,7 @@ private void buildWithArm1View(final NeighborView arm1View, final GraphTraversal
 
       for (int i = a1Start; i < a1End; i++)
         for (final int ep2 : ep2Ids)
-          pairCounts.merge(CSRCountUtils.packPair(arm1Nbrs[i], ep2), 1L, Long::sum);
+          pairCounts.increment(CSRCountUtils.packPair(arm1Nbrs[i], ep2));
     }
   }
 

engine/src/main/java/com/arcadedb/query/opencypher/procedures/algo/AlgoLabelPropagation.java

@@ -28,8 +28,9 @@
 import com.arcadedb.query.sql.executor.Result;
 import com.arcadedb.query.sql.executor.ResultInternal;
 
+import com.arcadedb.utility.IntIntHashMap;
+
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -165,19 +166,19 @@ private Stream<Result> executeWithOLTP(final Database db, final int maxIteration
         }
 
         // Count labels of neighbors
-        final Map<Integer, Integer> labelCount = new HashMap<>();
+        final IntIntHashMap labelCount = new IntIntHashMap();
         for (final int neighborIdx : neighbors)
-          labelCount.merge(label[neighborIdx], 1, Integer::sum);
+          labelCount.increment(label[neighborIdx]);
 
         // Find most frequent label (ties broken by smallest label)
-        int bestLabel = label[i];
-        int bestCount = 0;
-        for (final Map.Entry<Integer, Integer> entry : labelCount.entrySet()) {
-          if (entry.getValue() > bestCount || (entry.getValue() == bestCount && entry.getKey() < bestLabel)) {
-            bestCount = entry.getValue();
-            bestLabel = entry.getKey();
+        final int[] best = { label[i], 0 }; // [bestLabel, bestCount]
+        labelCount.forEach((lbl, cnt) -> {
+          if (cnt > best[1] || (cnt == best[1] && lbl < best[0])) {
+            best[1] = cnt;
+            best[0] = lbl;
           }
-        }
+        });
+        int bestLabel = best[0];
 
         newLabel[i] = bestLabel;
         if (bestLabel != label[i])

engine/src/main/java/com/arcadedb/query/opencypher/procedures/algo/AlgoLeiden.java

@@ -25,9 +25,9 @@
 import com.arcadedb.query.sql.executor.Result;
 import com.arcadedb.query.sql.executor.ResultInternal;
 
-import java.util.HashMap;
+import com.arcadedb.utility.IntIntHashMap;
+
 import java.util.List;
-import java.util.Map;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
@@ -125,30 +125,29 @@ public Stream<Result> execute(final Object[] args, final Result inputRow, final
         final long ki = degree[i];
 
         // Count weights to each neighboring community
-        final Map<Integer, Integer> commWeights = new HashMap<>();
-        for (final int j : adj[i]) {
-          final int jComm = community[j];
-          commWeights.merge(jComm, 1, Integer::sum);
-        }
+        final IntIntHashMap commWeights = new IntIntHashMap();
+        for (final int j : adj[i])
+          commWeights.increment(community[j]);
 
         int bestComm = currentComm;
-        double bestGain = 0.0;
+        final double[] bestGain = { 0.0 };
+        final int[] bestCommHolder = { currentComm };
 
-        for (final Map.Entry<Integer, Integer> entry : commWeights.entrySet()) {
-          final int candidateComm = entry.getKey();
+        commWeights.forEach((candidateComm, weight) -> {
           if (candidateComm == currentComm)
-            continue;
+            return;
 
-          final double eIc = entry.getValue();
+          final double eIc = weight;
           final double dc = communityDegree[candidateComm];
           // Modularity gain = [e_ic / m] - γ * [k_i * d_c / (2m²)]
           final double gain = eIc / m - resolution * ki * dc / (2.0 * m * m);
 
-          if (gain > bestGain) {
-            bestGain = gain;
-            bestComm = candidateComm;
+          if (gain > bestGain[0]) {
+            bestGain[0] = gain;
+            bestCommHolder[0] = candidateComm;
           }
-        }
+        });
+        bestComm = bestCommHolder[0];
 
         if (bestComm != currentComm) {
           communityDegree[currentComm] -= ki;
@@ -166,14 +165,14 @@ public Stream<Result> execute(final Object[] args, final Result inputRow, final
           final int currentComm = community[i];
           final long ki = degree[i];
 
-          final Map<Integer, Integer> sameCommWeights = new HashMap<>();
+          final IntIntHashMap sameCommWeights = new IntIntHashMap();
           for (final int j : adj[i]) {
             if (community[j] == currentComm)
-              sameCommWeights.merge(community[j], 1, Integer::sum);
+              sameCommWeights.increment(community[j]);
           }
 
           // Try removing from current community
-          final int internalEdges = sameCommWeights.getOrDefault(currentComm, 0);
+          final int internalEdges = sameCommWeights.get(currentComm, 0);
           final double removeGain = -(internalEdges / m - resolution * ki * communityDegree[currentComm] / (2.0 * m * m));
 
           if (removeGain > 0.0) {
@@ -196,7 +195,7 @@ public Stream<Result> execute(final Object[] args, final Result inputRow, final
     }
 
     // Remap communities to sequential IDs
-    final Map<Integer, Integer> remap = new HashMap<>();
+    final IntIntHashMap remap = new IntIntHashMap();
     int nextId = 0;
     for (int i = 0; i < n; i++) {
       if (!remap.containsKey(community[i]))
@@ -206,7 +205,7 @@ public Stream<Result> execute(final Object[] args, final Result inputRow, final
     return IntStream.range(0, n).mapToObj(i -> {
       final ResultInternal r = new ResultInternal();
       r.setProperty("nodeId", graph.getRID(i));
-      r.setProperty("community", remap.get(community[i]));
+      r.setProperty("community", remap.get(community[i], -1));
       return (Result) r;
     });
   }

engine/src/main/java/com/arcadedb/query/opencypher/procedures/algo/AlgoLouvain.java

@@ -25,6 +25,8 @@
 import com.arcadedb.query.sql.executor.Result;
 import com.arcadedb.query.sql.executor.ResultInternal;
 
+import com.arcadedb.utility.IntIntHashMap;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -201,7 +203,7 @@ public Stream<Result> execute(final Object[] args, final Result inputRow, final
     final double finalModularity = computeModularity(vertices, community, vertexIndex, nodeDegree, totalWeight, weightProperty);
 
     // Remap community IDs to be sequential starting from 0
-    final Map<Integer, Integer> communityRemap = new HashMap<>();
+    final IntIntHashMap communityRemap = new IntIntHashMap();
     int nextId = 0;
     for (int i = 0; i < n; i++) {
       if (!communityRemap.containsKey(community[i]))
@@ -211,7 +213,7 @@ public Stream<Result> execute(final Object[] args, final Result inputRow, final
     return IntStream.range(0, n).mapToObj(i -> {
       final ResultInternal result = new ResultInternal();
       result.setProperty("node", vertices.get(i).getIdentity());
-      result.setProperty("communityId", communityRemap.get(community[i]));
+      result.setProperty("communityId", communityRemap.get(community[i], -1));
       result.setProperty("modularity", finalModularity);
       return (Result) result;
     });