Load: Optimized the partial path split logic in modifications coverage judgment (apache#16212)

* optimize

* shit

* test

* revert

* opti

* fix

Author: Caideyipi

integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java

@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.it.utils;
 
+import org.apache.iotdb.commons.conf.IoTDBConstant;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
@@ -248,6 +249,36 @@ private void generateTEXT(final Object obj, final int row) {
         new Binary(String.format("test point %d", random.nextInt()), TSFileConfig.STRING_CHARSET);
   }
 
+  public void generateDeletion(final String device) throws IOException, IllegalPathException {
+    try (final ModificationFile modificationFile =
+        new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) {
+      modificationFile.write(
+          new Deletion(
+              new PartialPath(
+                  device + TsFileConstant.PATH_SEPARATOR + IoTDBConstant.ONE_LEVEL_PATH_WILDCARD),
+              tsFile.length(),
+              Long.MIN_VALUE,
+              Long.MAX_VALUE));
+      device2TimeSet.remove(device);
+      device2MeasurementSchema.remove(device);
+    }
+  }
+
+  public void generateDeletion(final String device, final MeasurementSchema measurement)
+      throws IOException, IllegalPathException {
+    try (final ModificationFile modificationFile =
+        new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) {
+      modificationFile.write(
+          new Deletion(
+              new PartialPath(
+                  device + TsFileConstant.PATH_SEPARATOR + measurement.getMeasurementId()),
+              tsFile.length(),
+              Long.MIN_VALUE,
+              Long.MAX_VALUE));
+      device2MeasurementSchema.get(device).remove(measurement);
+    }
+  }
+
   public void generateDeletion(final String device, final int number)
       throws IOException, IllegalPathException {
     try (final ModificationFile modificationFile =

integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java

@@ -20,6 +20,7 @@
 package org.apache.iotdb.db.it;
 
 import org.apache.iotdb.commons.auth.entity.PrivilegeType;
+import org.apache.iotdb.db.it.utils.TestUtils;
 import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.it.framework.IoTDBTestRunner;
@@ -781,16 +782,19 @@ public void testLoadWithMods() throws Exception {
       generator.registerTimeseries(
           SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
       generator.registerAlignedTimeseries(
-          SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
+          SchemaConfig.DEVICE_4,
+          new ArrayList<>(Arrays.asList(SchemaConfig.MEASUREMENT_30, SchemaConfig.MEASUREMENT_40)));
       generator.generateData(SchemaConfig.DEVICE_2, 100, PARTITION_INTERVAL / 10_000, false);
       generator.generateData(SchemaConfig.DEVICE_3, 100, PARTITION_INTERVAL / 10_000, false);
+      generator.generateDeletion(SchemaConfig.DEVICE_3);
       generator.generateData(SchemaConfig.DEVICE_4, 100, PARTITION_INTERVAL / 10_000, true);
       generator.generateDeletion(SchemaConfig.DEVICE_2, 2);
       generator.generateDeletion(SchemaConfig.DEVICE_4, 2);
       generator.generateData(SchemaConfig.DEVICE_2, 100, PARTITION_INTERVAL / 10_000, false);
       generator.generateData(SchemaConfig.DEVICE_4, 100, PARTITION_INTERVAL / 10_000, true);
       generator.generateDeletion(SchemaConfig.DEVICE_2, 2);
       generator.generateDeletion(SchemaConfig.DEVICE_4, 2);
+      generator.generateDeletion(SchemaConfig.DEVICE_4, SchemaConfig.MEASUREMENT_30);
       writtenPoint2 = generator.getTotalNumber();
     }
 
@@ -810,6 +814,10 @@ public void testLoadWithMods() throws Exception {
           Assert.fail("This ResultSet is empty.");
         }
       }
+
+      TestUtils.assertSingleResultSetEqual(
+          TestUtils.executeQueryWithRetry(statement, "count timeSeries"),
+          Collections.singletonMap("count(timeseries)", "18"));
     }
   }
 

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java

@@ -28,6 +28,7 @@
 import org.apache.iotdb.commons.consensus.ConfigRegionId;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PatternTreeMap;
 import org.apache.iotdb.commons.schema.SchemaConstant;
 import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
 import org.apache.iotdb.commons.utils.RetryUtils;
@@ -58,7 +59,6 @@
 import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
 import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaStatement;
 import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement;
-import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
@@ -72,6 +72,7 @@
 import org.apache.iotdb.db.utils.ModificationUtils;
 import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
 import org.apache.iotdb.db.utils.constant.SqlConstant;
+import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
 import org.apache.iotdb.rpc.RpcUtils;
 import org.apache.iotdb.rpc.TSStatusCode;
 
@@ -96,7 +97,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -582,10 +582,10 @@ public void setCurrentTimeIndex(final ITimeIndex timeIndex) {
 
     public void autoCreateAndVerify(
         TsFileSequenceReader reader,
-        Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
+        Map<IDeviceID, List<TimeseriesMetadata>> device2TimeSeriesMetadataList)
         throws IOException, AuthException, LoadAnalyzeTypeMismatchException {
       for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
-          device2TimeseriesMetadataList.entrySet()) {
+          device2TimeSeriesMetadataList.entrySet()) {
         final IDeviceID device = entry.getKey();
 
         try {
@@ -601,15 +601,15 @@ public void autoCreateAndVerify(
 
         for (final TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
           try {
-            if (schemaCache.isTimeseriesDeletedByMods(device, timeseriesMetadata)) {
+            if (schemaCache.isTimeSeriesDeletedByMods(device, timeseriesMetadata)) {
               continue;
             }
           } catch (IllegalPathException e) {
             // In aligned devices, there may be empty measurements which will cause
             // IllegalPathException.
             if (!timeseriesMetadata.getMeasurementId().isEmpty()) {
               LOGGER.warn(
-                  "Failed to check if device {}, timeseries {} is deleted by mods. Will see it as not deleted.",
+                  "Failed to check if device {}, timeSeries {} is deleted by mods. Will see it as not deleted.",
                   device,
                   timeseriesMetadata.getMeasurementId(),
                   e);
@@ -974,7 +974,7 @@ private static class LoadTsFileAnalyzeSchemaCache {
     private Map<IDeviceID, Boolean> tsFileDevice2IsAligned;
     private Set<PartialPath> alreadySetDatabases;
 
-    private Collection<Modification> currentModifications;
+    private PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> currentModifications;
     private ITimeIndex currentTimeIndex;
 
     private long batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes = 0;
@@ -992,7 +992,7 @@ public LoadTsFileAnalyzeSchemaCache() throws LoadRuntimeOutOfMemoryException {
       this.currentBatchDevice2TimeSeriesSchemas = new HashMap<>();
       this.tsFileDevice2IsAligned = new HashMap<>();
       this.alreadySetDatabases = new HashSet<>();
-      this.currentModifications = new ArrayList<>();
+      this.currentModifications = PatternTreeMapFactory.getModsPatternTreeMap();
     }
 
     public Map<IDeviceID, Set<MeasurementSchema>> getDevice2TimeSeries() {
@@ -1051,10 +1051,13 @@ public void addIsAlignedCache(IDeviceID device, boolean isAligned, boolean addIf
     public void setCurrentModificationsAndTimeIndex(TsFileResource resource) throws IOException {
       clearModificationsAndTimeIndex();
 
-      currentModifications = resource.getModFile().getModifications();
-      for (final Modification modification : currentModifications) {
-        currentModificationsMemoryUsageSizeInBytes += ((Deletion) modification).getSerializedSize();
-      }
+      resource
+          .getModFile()
+          .getModifications()
+          .forEach(
+              modification -> currentModifications.append(modification.getPath(), modification));
+
+      currentModificationsMemoryUsageSizeInBytes = currentModifications.ramBytesUsed();
       block.addMemoryUsage(currentModificationsMemoryUsageSizeInBytes);
 
       if (resource.resourceFileExists()) {
@@ -1076,9 +1079,9 @@ public boolean isDeviceDeletedByMods(IDeviceID device) throws IllegalPathExcepti
           currentModifications, currentTimeIndex, device);
     }
 
-    public boolean isTimeseriesDeletedByMods(
+    public boolean isTimeSeriesDeletedByMods(
         IDeviceID device, TimeseriesMetadata timeseriesMetadata) throws IllegalPathException {
-      return ModificationUtils.isTimeseriesDeletedByMods(
+      return ModificationUtils.isTimeSeriesDeletedByMods(
           currentModifications,
           device,
           timeseriesMetadata.getMeasurementId(),
@@ -1116,7 +1119,7 @@ public void clearTimeSeries() {
     }
 
     public void clearModificationsAndTimeIndex() {
-      currentModifications.clear();
+      currentModifications = PatternTreeMapFactory.getModsPatternTreeMap();
       currentTimeIndex = null;
       block.reduceMemoryUsage(currentModificationsMemoryUsageSizeInBytes);
       block.reduceMemoryUsage(currentTimeIndexMemoryUsageSizeInBytes);

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java

@@ -19,16 +19,18 @@
 
 package org.apache.iotdb.db.utils;
 
-import org.apache.iotdb.commons.conf.IoTDBConstant;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.AlignedPath;
 import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PatternTreeMap;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.impl.SettleSelectorImpl;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
+import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
 
 import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
 import org.apache.tsfile.file.metadata.IChunkMetadata;
@@ -37,7 +39,6 @@
 import org.apache.tsfile.utils.Pair;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 
@@ -195,33 +196,31 @@ public static boolean isPointDeleted(long timestamp, List<TimeRange> deletionLis
    * There are some slight differences from that in {@link SettleSelectorImpl}.
    */
   public static boolean isDeviceDeletedByMods(
-      Collection<Modification> modifications, IDeviceID device, long startTime, long endTime)
+      final PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> modifications,
+      final IDeviceID device,
+      final long startTime,
+      final long endTime)
       throws IllegalPathException {
-    for (Modification modification : modifications) {
-      PartialPath path = modification.getPath();
-      if (path.include(new PartialPath(device, IoTDBConstant.ONE_LEVEL_PATH_WILDCARD))
-          && ((Deletion) modification).getTimeRange().contains(startTime, endTime)) {
-        return true;
-      }
-    }
-    return false;
+    final List<Modification> mods =
+        modifications.getOverlapped(
+            CompactionPathUtils.getPath(device, AlignedPath.VECTOR_PLACEHOLDER));
+    return mods.stream()
+        .anyMatch(
+            modification -> ((Deletion) modification).getTimeRange().contains(startTime, endTime));
   }
 
-  public static boolean isTimeseriesDeletedByMods(
-      Collection<Modification> modifications,
-      IDeviceID device,
-      String timeseriesId,
-      long startTime,
-      long endTime)
+  public static boolean isTimeSeriesDeletedByMods(
+      final PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> modifications,
+      final IDeviceID device,
+      final String timeSeriesId,
+      final long startTime,
+      final long endTime)
       throws IllegalPathException {
-    for (Modification modification : modifications) {
-      PartialPath path = modification.getPath();
-      if (path.include(new PartialPath(device, timeseriesId))
-          && ((Deletion) modification).getTimeRange().contains(startTime, endTime)) {
-        return true;
-      }
-    }
-    return false;
+    final List<Modification> mods =
+        modifications.getOverlapped(CompactionPathUtils.getPath(device, timeSeriesId));
+    return mods.stream()
+        .anyMatch(
+            modification -> ((Deletion) modification).getTimeRange().contains(startTime, endTime));
   }
 
   private static void doModifyChunkMetaData(Modification modification, IChunkMetadata metaData) {
@@ -298,7 +297,9 @@ private static List<Modification> getModificationsForMemtable(
   }
 
   public static boolean isDeviceDeletedByMods(
-      Collection<Modification> currentModifications, ITimeIndex currentTimeIndex, IDeviceID device)
+      PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> currentModifications,
+      ITimeIndex currentTimeIndex,
+      IDeviceID device)
       throws IllegalPathException {
     return isDeviceDeletedByMods(
         currentModifications,