filter table deletions

Author: shuwenwei

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java

@@ -364,6 +364,9 @@ protected PatternTreeMap<ModEntry, PatternTreeMapFactory.ModsSerializer> getAllM
               }
               return null;
             });
+    if (cachedResult == null) {
+      LOGGER.error("cached num: {}, mem cost: {}", fileModCache.size(), cachedModEntriesSize.get());
+    }
     return cachedResult == null ? atomicReference.get() : cachedResult;
   }
 
@@ -526,6 +529,16 @@ public void setTimeFilterForTableModel(Filter timeFilter) {
     }
   }
 
+  @Override
+  public boolean collectTable(String table) {
+    boolean added = super.collectTable(table);
+    if (added && memoryReservationManager != null) {
+      memoryReservationManager.reserveMemoryCumulatively(
+          RamUsageEstimator.sizeOf(table) + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY);
+    }
+    return added;
+  }
+
   public IDataRegionForQuery getDataRegion() {
     return dataRegion;
   }
@@ -842,6 +855,7 @@ public synchronized void releaseResource() {
     releaseTVListOwnedByQuery();
 
     fileModCache = null;
+    tables = null;
     dataRegion = null;
     globalTimeFilter = null;
     sharedQueryDataSource = null;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java

@@ -23,6 +23,7 @@
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.path.PatternTreeMap;
 import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
+import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.db.utils.ModificationUtils;
@@ -76,6 +77,8 @@ public class QueryContext {
   // referenced TVLists for the query
   protected final Set<TVList> tvListSet = new HashSet<>();
 
+  protected Set<String> tables;
+
   public QueryContext() {}
 
   public QueryContext(long queryId) {
@@ -90,6 +93,18 @@ public QueryContext(long queryId, boolean debug, long startTime, long timeout) {
     this.timeout = timeout;
   }
 
+  // Only used for query with table data(Tree view is not included)
+  public boolean collectTable(String table) {
+    if (tables == null) {
+      tables = Collections.singleton(table);
+      return true;
+    }
+    if (!(tables instanceof HashSet)) {
+      tables = new HashSet<>(tables);
+    }
+    return tables.add(table);
+  }
+
   // if the mods file does not exist, do not add it to the cache
   protected boolean checkIfModificationExists(TsFileResource tsFileResource) {
     // The exists state of ModificationFile is maintained in memory, and ModificationFile instance
@@ -106,7 +121,15 @@ public PatternTreeMap<ModEntry, ModsSerializer> loadAllModificationsFromDisk(
       TsFileResource resource) {
     PatternTreeMap<ModEntry, ModsSerializer> modifications =
         PatternTreeMapFactory.getModsPatternTreeMap();
-    for (ModEntry modification : resource.getAllModEntries()) {
+    TsFileResource.ModIterator modEntryIterator = resource.getModEntryIterator();
+    while (modEntryIterator.hasNext()) {
+      ModEntry modification = modEntryIterator.next();
+      if (tables != null && modification instanceof TableDeletionEntry) {
+        String tableName = ((TableDeletionEntry) modification).getTableName();
+        if (!tables.contains(tableName)) {
+          continue;
+        }
+      }
       modifications.append(modification.keyOfPatternTree(), modification);
     }
     return modifications;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java

@@ -1207,6 +1207,7 @@ public Operator visitDeviceTableScan(
 
     TableScanOperator tableScanOperator = new TableScanOperator(parameter);
 
+    context.getInstanceContext().collectTable(node.getQualifiedObjectName().getObjectName());
     addSource(
         tableScanOperator,
         context,
@@ -3021,6 +3022,7 @@ public Operator visitAggregationTableScan(
     } else {
       DefaultAggTableScanOperator aggTableScanOperator = new DefaultAggTableScanOperator(parameter);
 
+      context.getInstanceContext().collectTable(node.getQualifiedObjectName().getObjectName());
       addSource(
           aggTableScanOperator,
           context,