Added flags to mark whether a device has device descendants to optimize query like select xx from xxx.** (#17672) (#17802)

* flag-

* Fix

* Address device descendant flag review comments

(cherry picked from commit a565a06)

Author: Caideyipi

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java

@@ -174,6 +174,59 @@ public synchronized boolean createSnapshot(File snapshotDir) {
     return store.createSnapshot(snapshotDir);
   }
 
+  private void applySubtreeMeasurementDelta(IMemMNode startNode, final long delta) {
+    if (delta == 0 || startNode == null) {
+      return;
+    }
+    IMemMNode current = startNode;
+    while (current != null) {
+      current.setSubtreeMeasurementCount(current.getSubtreeMeasurementCount() + delta);
+      current = current.getParent();
+    }
+  }
+
+  private IDeviceMNode<IMemMNode> setToEntityAndUpdateFlags(final IMemMNode node) {
+    final boolean wasDevice = node.isDevice();
+    final IDeviceMNode<IMemMNode> deviceMNode = store.setToEntity(node);
+    if (!wasDevice) {
+      markAncestorsHavingDeviceDescendant(node);
+    }
+    return deviceMNode;
+  }
+
+  private void markAncestorsHavingDeviceDescendant(final IMemMNode deviceNode) {
+    IMemMNode current = deviceNode.getParent();
+    while (current != null && !current.hasDeviceDescendant()) {
+      current.setHasDeviceDescendant(true);
+      current = current.getParent();
+    }
+  }
+
+  private boolean hasDeviceDescendantInChildren(final IMemMNode node) {
+    try (final IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(node)) {
+      while (iterator.hasNext()) {
+        final IMemMNode child = iterator.next();
+        if (child.isDevice() || child.hasDeviceDescendant()) {
+          return true;
+        }
+      }
+      return false;
+    }
+  }
+
+  private void refreshAncestorsHavingDeviceDescendant(IMemMNode startNode) {
+    IMemMNode current = startNode;
+    while (current != null) {
+      current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current));
+      current = current.getParent();
+    }
+  }
+
+  private long getTemplateMeasurementCount(final int templateId) {
+    final Template template = ClusterTemplateManager.getInstance().getTemplate(templateId);
+    return template == null ? 0L : template.getMeasurementNumber();
+  }
+
   public static MTreeBelowSGMemoryImpl loadFromSnapshot(
       File snapshotDir,
       String storageGroupFullPath,
@@ -184,13 +237,16 @@ public static MTreeBelowSGMemoryImpl loadFromSnapshot(
       Function<IMeasurementMNode<IMemMNode>, Map<String, String>> tagGetter,
       Function<IMeasurementMNode<IMemMNode>, Map<String, String>> attributeGetter)
       throws IOException, IllegalPathException {
-    return new MTreeBelowSGMemoryImpl(
-        new PartialPath(storageGroupFullPath),
-        MemMTreeStore.loadFromSnapshot(
-            snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric),
-        tagGetter,
-        attributeGetter,
-        regionStatistics);
+    final MTreeBelowSGMemoryImpl mtree =
+        new MTreeBelowSGMemoryImpl(
+            new PartialPath(storageGroupFullPath),
+            MemMTreeStore.loadFromSnapshot(
+                snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric),
+            tagGetter,
+            attributeGetter,
+            regionStatistics);
+    mtree.rebuildSubtreeMeasurementCount();
+    return mtree;
   }
 
   // endregion
@@ -268,7 +324,7 @@ public IMeasurementMNode<IMemMNode> createTimeSeries(
       if (device.isDevice()) {
         entityMNode = device.getAsDeviceMNode();
       } else {
-        entityMNode = store.setToEntity(device);
+        entityMNode = setToEntityAndUpdateFlags(device);
       }
 
       // create a non-aligned time series
@@ -290,6 +346,7 @@ public IMeasurementMNode<IMemMNode> createTimeSeries(
         entityMNode.addAlias(alias, measurementMNode);
       }
 
+      applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L);
       return measurementMNode;
     }
   }
@@ -360,7 +417,7 @@ public List<IMeasurementMNode<IMemMNode>> createAlignedTimeSeries(
       if (device.isDevice()) {
         entityMNode = device.getAsDeviceMNode();
       } else {
-        entityMNode = store.setToEntity(device);
+        entityMNode = setToEntityAndUpdateFlags(device);
         entityMNode.setAligned(true);
       }
 
@@ -386,6 +443,7 @@ public List<IMeasurementMNode<IMemMNode>> createAlignedTimeSeries(
         if (aliasList != null && aliasList.get(i) != null) {
           entityMNode.addAlias(aliasList.get(i), measurementMNode);
         }
+        applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L);
         measurementMNodeList.add(measurementMNode);
       }
       return measurementMNodeList;
@@ -539,6 +597,7 @@ public IMeasurementMNode<IMemMNode> deleteTimeseries(PartialPath path) throws Me
       if (deletedNode.getAlias() != null) {
         parent.getAsDeviceMNode().deleteAliasChild(deletedNode.getAlias());
       }
+      applySubtreeMeasurementDelta(parent, -1L);
     }
     deleteEmptyInternalMNode(parent.getAsDeviceMNode());
     return deletedNode;
@@ -551,14 +610,15 @@ public void deleteEmptyInternalMNode(IDeviceMNode<IMemMNode> entityMNode) {
       boolean hasMeasurement = false;
       boolean hasNonViewMeasurement = false;
       IMemMNode child;
-      IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(curNode);
-      while (iterator.hasNext()) {
-        child = iterator.next();
-        if (child.isMeasurement()) {
-          hasMeasurement = true;
-          if (!child.getAsMeasurementMNode().isLogicalView()) {
-            hasNonViewMeasurement = true;
-            break;
+      try (final IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(curNode)) {
+        while (iterator.hasNext()) {
+          child = iterator.next();
+          if (child.isMeasurement()) {
+            hasMeasurement = true;
+            if (!child.getAsMeasurementMNode().isLogicalView()) {
+              hasNonViewMeasurement = true;
+              break;
+            }
           }
         }
       }
@@ -567,6 +627,7 @@ public void deleteEmptyInternalMNode(IDeviceMNode<IMemMNode> entityMNode) {
         synchronized (this) {
           curNode = store.setToInternal(entityMNode);
         }
+        refreshAncestorsHavingDeviceDescendant(curNode.getParent());
       } else if (!hasNonViewMeasurement) {
         // has some measurement but they are all logical view
         entityMNode.setAligned(null);
@@ -889,7 +950,7 @@ public void activateTemplate(PartialPath activatePath, Template template)
       if (cur.isDevice()) {
         entityMNode = cur.getAsDeviceMNode();
       } else {
-        entityMNode = store.setToEntity(cur);
+        entityMNode = setToEntityAndUpdateFlags(cur);
       }
     }
 
@@ -907,6 +968,7 @@ public void activateTemplate(PartialPath activatePath, Template template)
     entityMNode.setUseTemplate(true);
     entityMNode.setSchemaTemplateId(template.getId());
     regionStatistics.activateTemplate(template.getId());
+    applySubtreeMeasurementDelta(entityMNode.getAsMNode(), (long) template.getMeasurementNumber());
   }
 
   public Map<PartialPath, List<Integer>> constructSchemaBlackListWithTemplate(
@@ -968,6 +1030,8 @@ protected void updateEntity(IDeviceMNode<IMemMNode> node) {
                 resultTemplateSetInfo.put(
                     node.getPartialPath(), Collections.singletonList(node.getSchemaTemplateId()));
                 regionStatistics.deactivateTemplate(node.getSchemaTemplateId());
+                applySubtreeMeasurementDelta(
+                    node.getAsMNode(), -getTemplateMeasurementCount(node.getSchemaTemplateId()));
                 node.deactivateTemplate();
                 deleteEmptyInternalMNode(node);
               }
@@ -991,7 +1055,7 @@ public void activateTemplateWithoutCheck(
     if (cur.isDevice()) {
       entityMNode = cur.getAsDeviceMNode();
     } else {
-      entityMNode = store.setToEntity(cur);
+      entityMNode = setToEntityAndUpdateFlags(cur);
     }
 
     if (!entityMNode.isAligned()) {
@@ -1000,6 +1064,7 @@ public void activateTemplateWithoutCheck(
     entityMNode.setUseTemplate(true);
     entityMNode.setSchemaTemplateId(templateId);
     regionStatistics.activateTemplate(templateId);
+    applySubtreeMeasurementDelta(entityMNode.getAsMNode(), getTemplateMeasurementCount(templateId));
   }
 
   public long countPathsUsingTemplate(PartialPath pathPattern, int templateId)
@@ -1011,6 +1076,30 @@ public long countPathsUsingTemplate(PartialPath pathPattern, int templateId)
     }
   }
 
+  public void rebuildSubtreeMeasurementCount() {
+    rebuildSubtreeMeasurementCountFromNode(rootNode);
+  }
+
+  private long rebuildSubtreeMeasurementCountFromNode(final IMemMNode node) {
+    long count = node.isMeasurement() ? 1L : 0L;
+    boolean hasDeviceDescendant = false;
+    try (final IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(node)) {
+      while (iterator.hasNext()) {
+        final IMemMNode child = iterator.next();
+        count += rebuildSubtreeMeasurementCountFromNode(child);
+        if (child.isDevice() || child.hasDeviceDescendant()) {
+          hasDeviceDescendant = true;
+        }
+      }
+    }
+    if (node.isDevice() && node.getAsDeviceMNode().isUseTemplate()) {
+      count += getTemplateMeasurementCount(node.getAsDeviceMNode().getSchemaTemplateId());
+    }
+    node.setSubtreeMeasurementCount(count);
+    node.setHasDeviceDescendant(hasDeviceDescendant);
+    return count;
+  }
+
   // endregion
 
   // region Interfaces for schema reader
@@ -1278,7 +1367,7 @@ public IMeasurementMNode<IMemMNode> createLogicalView(
       if (device.isDevice()) {
         entityMNode = device.getAsDeviceMNode();
       } else {
-        entityMNode = store.setToEntity(device);
+        entityMNode = setToEntityAndUpdateFlags(device);
         // this parent has no measurement before. The leafName is his first child who is a logical
         // view.
         entityMNode.setAligned(null);
@@ -1287,6 +1376,7 @@ public IMeasurementMNode<IMemMNode> createLogicalView(
       measurementMNode.setParent(entityMNode.getAsMNode());
       store.addChild(entityMNode.getAsMNode(), leafName, measurementMNode.getAsMNode());
 
+      applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L);
       return measurementMNode;
     }
   }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java

@@ -19,5 +19,33 @@
 package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode;
 
 import org.apache.iotdb.commons.schema.node.IMNode;
+import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.basic.BasicMNode;
 
-public interface IMemMNode extends IMNode<IMemMNode> {}
+public interface IMemMNode extends IMNode<IMemMNode> {
+
+  BasicMNode getBasicMNode();
+
+  /**
+   * The count of measurement nodes contained in the subtree rooted at this node. The counter is
+   * maintained in memory only.
+   */
+  default long getSubtreeMeasurementCount() {
+    return getBasicMNode().getSubtreeMeasurementCount();
+  }
+
+  default void setSubtreeMeasurementCount(final long subtreeMeasurementCount) {
+    getBasicMNode().setSubtreeMeasurementCount(subtreeMeasurementCount);
+  }
+
+  /**
+   * Whether there is any device node in the subtree rooted at this node, excluding the node itself.
+   * This flag is maintained in memory only.
+   */
+  default boolean hasDeviceDescendant() {
+    return getBasicMNode().hasDeviceDescendant();
+  }
+
+  default void setHasDeviceDescendant(final boolean hasDeviceDescendant) {
+    getBasicMNode().setHasDeviceDescendant(hasDeviceDescendant);
+  }
+}

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java

@@ -46,6 +46,12 @@ public class BasicMNode implements IMemMNode {
   private IMemMNode parent;
   private final BasicMNodeInfo basicMNodeInfo;
 
+  /** Cached count of measurements in this node's subtree, rebuilt on restart. */
+  private long subtreeMeasurementCount = 0L;
+
+  /** Cached flag showing whether there is any device in the subtree below this node. */
+  private boolean hasDeviceDescendant = false;
+
   /** from root to this node, only be set when used once for InternalMNode */
   private String fullPath;
 
@@ -55,6 +61,11 @@ public BasicMNode(IMemMNode parent, String name) {
     this.basicMNodeInfo = new BasicMNodeInfo(name);
   }
 
+  @Override
+  public BasicMNode getBasicMNode() {
+    return this;
+  }
+
   @Override
   public String getName() {
     return basicMNodeInfo.getName();
@@ -98,6 +109,25 @@ public void setFullPath(String fullPath) {
     this.fullPath = fullPath;
   }
 
+  public long getSubtreeMeasurementCount() {
+    return subtreeMeasurementCount;
+  }
+
+  @Override
+  public void setSubtreeMeasurementCount(final long subtreeMeasurementCount) {
+    this.subtreeMeasurementCount = subtreeMeasurementCount;
+  }
+
+  @Override
+  public boolean hasDeviceDescendant() {
+    return hasDeviceDescendant;
+  }
+
+  @Override
+  public void setHasDeviceDescendant(final boolean hasDeviceDescendant) {
+    this.hasDeviceDescendant = hasDeviceDescendant;
+  }
+
   @Override
   public PartialPath getPartialPath() {
     List<String> detachedPath = new ArrayList<>();
@@ -224,6 +254,8 @@ public <R, C> R accept(MNodeVisitor<R, C> visitor, C context) {
    *         <li>basicMNodeInfo reference, 8B
    *         <li>parent reference, 8B
    *         <li>fullPath reference, 8B
+   *         <li>subtreeMeasurementCount, 8B
+   *         <li>hasDeviceDescendant, 1B
    *       </ol>
    *   <li>MapEntry in parent
    *       <ol>
@@ -235,7 +267,7 @@ public <R, C> R accept(MNodeVisitor<R, C> visitor, C context) {
    */
   @Override
   public int estimateSize() {
-    return 8 + 8 + 8 + 8 + 8 + 8 + 28 + basicMNodeInfo.estimateSize();
+    return 8 + 8 + 8 + 8 + 8 + 1 + 8 + 8 + 28 + basicMNodeInfo.estimateSize();
   }
 
   @Override