diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index d872a1130a4ee..f5f7373c9b130 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -207,6 +207,43 @@ private void applySubtreeMeasurementDelta(IMemMNode startNode, final long delta) } } + private IDeviceMNode setToEntityAndUpdateFlags(final IMemMNode node) { + final boolean wasDevice = node.isDevice(); + final IDeviceMNode deviceMNode = store.setToEntity(node); + if (!wasDevice) { + markAncestorsHavingDeviceDescendant(node); + } + return deviceMNode; + } + + private void markAncestorsHavingDeviceDescendant(final IMemMNode deviceNode) { + IMemMNode current = deviceNode.getParent(); + while (current != null && !current.hasDeviceDescendant()) { + current.setHasDeviceDescendant(true); + current = current.getParent(); + } + } + + private boolean hasDeviceDescendantInChildren(final IMemMNode node) { + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final IMemMNode child = iterator.next(); + if (child.isDevice() || child.hasDeviceDescendant()) { + return true; + } + } + return false; + } + } + + private void refreshAncestorsHavingDeviceDescendant(IMemMNode startNode) { + IMemMNode current = startNode; + while (current != null) { + current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current)); + current = current.getParent(); + } + } + private long getTemplateMeasurementCount(final int templateId) { final Template template = ClusterTemplateManager.getInstance().getTemplate(templateId); return template == null ? 0L : template.getMeasurementNumber(); @@ -315,7 +352,7 @@ public IMeasurementMNode createTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); } // create a non-aligned time series @@ -409,7 +446,7 @@ public List> createAlignedTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); entityMNode.setAligned(true); } @@ -657,14 +694,15 @@ public void deleteEmptyInternalMNode(final IDeviceMNode entityMNode) boolean hasMeasurement = false; boolean hasNonViewMeasurement = false; IMemMNode child; - IMNodeIterator iterator = store.getChildrenIterator(curNode); - while (iterator.hasNext()) { - child = iterator.next(); - if (child.isMeasurement()) { - hasMeasurement = true; - if (!child.getAsMeasurementMNode().isLogicalView()) { - hasNonViewMeasurement = true; - break; + try (final IMNodeIterator iterator = store.getChildrenIterator(curNode)) { + while (iterator.hasNext()) { + child = iterator.next(); + if (child.isMeasurement()) { + hasMeasurement = true; + if (!child.getAsMeasurementMNode().isLogicalView()) { + hasNonViewMeasurement = true; + break; + } } } } @@ -673,6 +711,7 @@ public void deleteEmptyInternalMNode(final IDeviceMNode entityMNode) synchronized (this) { curNode = store.setToInternal(entityMNode); } + refreshAncestorsHavingDeviceDescendant(curNode.getParent()); } else if (!hasNonViewMeasurement) { // has some measurement but they are all logical view entityMNode.setAligned(null); @@ -1033,7 +1072,7 @@ public void activateTemplate(final PartialPath activatePath, final Template temp if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } } @@ -1141,7 +1180,7 @@ public void activateTemplateWithoutCheck( if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } if (!entityMNode.isAligned()) { @@ -1196,14 +1235,21 @@ public void rebuildSubtreeMeasurementCount() { private long rebuildSubtreeMeasurementCountFromNode(final IMemMNode node) { long count = node.isMeasurement() ? 1L : 0L; - final IMNodeIterator iterator = store.getChildrenIterator(node); - while (iterator.hasNext()) { - count += rebuildSubtreeMeasurementCountFromNode(iterator.next()); + boolean hasDeviceDescendant = false; + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final IMemMNode child = iterator.next(); + count += rebuildSubtreeMeasurementCountFromNode(child); + if (child.isDevice() || child.hasDeviceDescendant()) { + hasDeviceDescendant = true; + } + } } if (node.isDevice() && node.getAsDeviceMNode().isUseTemplate()) { count += getTemplateMeasurementCount(node.getAsDeviceMNode().getSchemaTemplateId()); } node.setSubtreeMeasurementCount(count); + node.setHasDeviceDescendant(hasDeviceDescendant); return count; } @@ -1786,7 +1832,7 @@ public IMeasurementMNode createLogicalView( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); // this parent has no measurement before. The leafName is his first child who is a logical // view. entityMNode.setAligned(null); @@ -1923,7 +1969,7 @@ public void createOrUpdateTableDevice( (TableDeviceInfo) entityMNode.getDeviceInfo(); attributeUpdater.accept(deviceInfo.getAttributePointer()); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); final TableDeviceInfo deviceInfo = new TableDeviceInfo<>(); deviceInfo.setAttributePointer(attributePointerGetter.getAsInt()); entityMNode.getAsInternalMNode().setDeviceInfo(deviceInfo); @@ -2040,6 +2086,7 @@ protected Void collectMNode(final IMemMNode node) { collector.traverse(); } databaseMNode.deleteChild(tableName); + refreshAncestorsHavingDeviceDescendant(databaseMNode); regionStatistics.resetTableDevice(tableName); store.releaseMemory(memoryReleased.get()); return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java index 6cd14800f3e3f..257d7caeef8ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java @@ -19,14 +19,33 @@ package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode; import org.apache.iotdb.commons.schema.node.IMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.basic.BasicMNode; public interface IMemMNode extends IMNode { + BasicMNode getBasicMNode(); + /** * The count of measurement nodes contained in the subtree rooted at this node. The counter is * maintained in memory only. */ - long getSubtreeMeasurementCount(); + default long getSubtreeMeasurementCount() { + return getBasicMNode().getSubtreeMeasurementCount(); + } + + default void setSubtreeMeasurementCount(final long subtreeMeasurementCount) { + getBasicMNode().setSubtreeMeasurementCount(subtreeMeasurementCount); + } + + /** + * Whether there is any device node in the subtree rooted at this node, excluding the node itself. + * This flag is maintained in memory only. + */ + default boolean hasDeviceDescendant() { + return getBasicMNode().hasDeviceDescendant(); + } - void setSubtreeMeasurementCount(long subtreeMeasurementCount); + default void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + getBasicMNode().setHasDeviceDescendant(hasDeviceDescendant); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java index 2eab69749fd79..1b9ae51c19345 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java @@ -49,6 +49,9 @@ public class BasicMNode implements IMemMNode { /** Cached count of measurements in this node's subtree, rebuilt on restart. */ private long subtreeMeasurementCount = 0L; + /** Cached flag showing whether there is any device in the subtree below this node. */ + private boolean hasDeviceDescendant = false; + /** from root to this node, only be set when used once for InternalMNode */ private String fullPath; @@ -58,6 +61,11 @@ public BasicMNode(IMemMNode parent, String name) { this.basicMNodeInfo = new BasicMNodeInfo(name); } + @Override + public BasicMNode getBasicMNode() { + return this; + } + @Override public String getName() { return basicMNodeInfo.getName(); @@ -112,6 +120,16 @@ public void setSubtreeMeasurementCount(final long subtreeMeasurementCount) { this.subtreeMeasurementCount = subtreeMeasurementCount; } + @Override + public boolean hasDeviceDescendant() { + return hasDeviceDescendant; + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + this.hasDeviceDescendant = hasDeviceDescendant; + } + @Override public PartialPath getPartialPath() { final List detachedPath = new ArrayList<>(); @@ -239,6 +257,7 @@ public R accept(final MNodeVisitor visitor, final C context) { *
  • parent reference, 8B *
  • fullPath reference, 8B *
  • subtreeMeasurementCount, 8B + *
  • hasDeviceDescendant, 1B * *
  • MapEntry in parent *
      @@ -250,7 +269,7 @@ public R accept(final MNodeVisitor visitor, final C context) { */ @Override public int estimateSize() { - return 8 + 8 + 8 + 8 + 8 + 8 + 8 + 28 + basicMNodeInfo.estimateSize(); + return 8 + 8 + 8 + 8 + 8 + 1 + 8 + 8 + 28 + basicMNodeInfo.estimateSize(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java index 87144d4954a5d..cff30d8b8c462 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java @@ -33,14 +33,4 @@ public AboveDatabaseMNode(IMemMNode parent, String name) { public IMemMNode getAsMNode() { return this; } - - @Override - public long getSubtreeMeasurementCount() { - return basicMNode.getSubtreeMeasurementCount(); - } - - @Override - public void setSubtreeMeasurementCount(long subtreeMeasurementCount) { - basicMNode.setSubtreeMeasurementCount(subtreeMeasurementCount); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java index 290cf4273601b..c6b2f5e2427bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java @@ -45,14 +45,4 @@ public IDeviceInfo getDeviceInfo() { public void setDeviceInfo(IDeviceInfo deviceInfo) { basicMNode.setDeviceInfo(deviceInfo); } - - @Override - public long getSubtreeMeasurementCount() { - return basicMNode.getSubtreeMeasurementCount(); - } - - @Override - public void setSubtreeMeasurementCount(long subtreeMeasurementCount) { - basicMNode.setSubtreeMeasurementCount(subtreeMeasurementCount); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java index d2a2cbd80c90c..a40cbe6bc0fc9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java @@ -52,14 +52,4 @@ public IMNodeContainer getChildren() { public final boolean isLogicalView() { return false; } - - @Override - public long getSubtreeMeasurementCount() { - return basicMNode.getSubtreeMeasurementCount(); - } - - @Override - public void setSubtreeMeasurementCount(long subtreeMeasurementCount) { - basicMNode.setSubtreeMeasurementCount(subtreeMeasurementCount); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index 8ca06b83f7dcf..8ee6e1034d5e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -137,6 +137,68 @@ public class MTreeBelowSGCachedImpl { private final int levelOfDB; private final CachedSchemaRegionStatistics regionStatistics; + private IDeviceMNode setToEntityAndUpdateFlags(final ICachedMNode node) + throws MetadataException { + final boolean wasDevice = node.isDevice(); + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + final IDeviceMNode deviceMNode = store.setToEntity(node); + if (!wasDevice) { + markAncestorsHavingDeviceDescendant(node); + } + return deviceMNode; + } + + private void markAncestorsHavingDeviceDescendant(final ICachedMNode deviceNode) { + ICachedMNode current = deviceNode.getParent(); + while (current != null && !current.hasDeviceDescendant()) { + current.setHasDeviceDescendant(true); + current.setDeviceDescendantComputed(true); + current = current.getParent(); + } + } + + private boolean hasDeviceDescendantInChildren(final ICachedMNode node) throws MetadataException { + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final ICachedMNode child = iterator.next(); + try { + if (child.isDevice() || hasDeviceDescendant(child)) { + return true; + } + } finally { + unPinMNode(child); + } + } + return false; + } + } + + private boolean hasDeviceDescendant(final ICachedMNode node) throws MetadataException { + if (node.isMeasurement()) { + node.setHasDeviceDescendant(false); + node.setDeviceDescendantComputed(true); + return false; + } + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + return node.hasDeviceDescendant(); + } + + private void refreshAncestorsHavingDeviceDescendant(ICachedMNode startNode) + throws MetadataException { + ICachedMNode current = startNode; + while (current != null) { + current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current)); + current.setDeviceDescendantComputed(true); + current = current.getParent(); + } + } + // region MTree initialization, clear and serialization public MTreeBelowSGCachedImpl( PartialPath storageGroupPath, @@ -353,7 +415,7 @@ public IMeasurementMNode createTimeSeriesWithPinnedReturn( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); device = entityMNode.getAsMNode(); } @@ -458,7 +520,7 @@ public List> createAlignedTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); entityMNode.setAligned(true); device = entityMNode.getAsMNode(); } @@ -684,8 +746,7 @@ private void deleteAndUnpinEmptyInternalMNode(IDeviceMNode entityM boolean hasMeasurement = false; boolean hasNonViewMeasurement = false; ICachedMNode child; - IMNodeIterator iterator = store.getChildrenIterator(curNode); - try { + try (final IMNodeIterator iterator = store.getChildrenIterator(curNode)) { while (iterator.hasNext()) { child = iterator.next(); unPinMNode(child); @@ -697,12 +758,11 @@ private void deleteAndUnpinEmptyInternalMNode(IDeviceMNode entityM } } } - } finally { - iterator.close(); } if (!hasMeasurement) { curNode = store.setToInternal(entityMNode); + refreshAncestorsHavingDeviceDescendant(curNode.getParent()); } else if (!hasNonViewMeasurement) { // has some measurement but they are all logical view store.updateMNode(entityMNode.getAsMNode(), o -> o.getAsDeviceMNode().setAligned(null)); @@ -729,14 +789,11 @@ private void deleteAndUnpinEmptyInternalMNode(IDeviceMNode entityM } private boolean isEmptyInternalMNode(ICachedMNode node) throws MetadataException { - IMNodeIterator iterator = store.getChildrenIterator(node); - try { + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { return !IoTDBConstant.PATH_ROOT.equals(node.getName()) && !node.isMeasurement() && !(node.isDevice() && node.getAsDeviceMNode().isUseTemplate()) && !iterator.hasNext(); - } finally { - iterator.close(); } } @@ -1075,7 +1132,7 @@ public void createLogicalView(PartialPath path, ViewExpression viewExpression) if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); // this parent has no measurement before. The leafName is his first child who is a // logical // view. @@ -1197,7 +1254,7 @@ public void activateTemplate(PartialPath activatePath, Template template) if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } if (entityMNode.isUseTemplate()) { @@ -1243,7 +1300,7 @@ public void activateTemplateWithoutCheck( if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } store.updateMNode( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java index 489b1a642aad9..244e077013c57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java @@ -22,8 +22,27 @@ import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.lock.LockEntry; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.cache.CacheEntry; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.basic.CachedBasicMNode; public interface ICachedMNode extends IMNode { + CachedBasicMNode getBasicMNode(); + + default boolean hasDeviceDescendant() { + return getBasicMNode().hasDeviceDescendant(); + } + + default void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + getBasicMNode().setHasDeviceDescendant(hasDeviceDescendant); + } + + default boolean isDeviceDescendantComputed() { + return getBasicMNode().isDeviceDescendantComputed(); + } + + default void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + getBasicMNode().setDeviceDescendantComputed(deviceDescendantComputed); + } + CacheEntry getCacheEntry(); void setCacheEntry(CacheEntry cacheEntry); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java index d34fde474abee..14bb9f3609b78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java @@ -47,6 +47,17 @@ public class CachedBasicMNode implements ICachedMNode { private ICachedMNode parent; private final CacheMNodeInfo cacheMNodeInfo; + /** Cached flag showing whether there is any device in the subtree below this node. */ + private boolean hasDeviceDescendant = false; + + /** + * Whether {@link #hasDeviceDescendant} is trusted for the current in-memory node instance. + * + *

      This state is intentionally not persisted. A node reloaded from PBTree can lazily recompute + * it when the wildcard-suffix optimization needs it. + */ + private boolean deviceDescendantComputed = false; + /** from root to this node, only be set when used once for InternalMNode */ private String fullPath; @@ -56,6 +67,11 @@ public CachedBasicMNode(ICachedMNode parent, String name) { this.cacheMNodeInfo = new CacheMNodeInfo(name); } + @Override + public CachedBasicMNode getBasicMNode() { + return this; + } + @Override public String getName() { return cacheMNodeInfo.getName(); @@ -99,6 +115,26 @@ public void setFullPath(String fullPath) { this.fullPath = fullPath; } + @Override + public boolean hasDeviceDescendant() { + return hasDeviceDescendant; + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + this.hasDeviceDescendant = hasDeviceDescendant; + } + + @Override + public boolean isDeviceDescendantComputed() { + return deviceDescendantComputed; + } + + @Override + public void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + this.deviceDescendantComputed = deviceDescendantComputed; + } + @Override public PartialPath getPartialPath() { List detachedPath = new ArrayList<>(); @@ -245,6 +281,8 @@ public void setLockEntry(LockEntry lockEntry) { *

    1. basicMNodeInfo reference, 8B *
    2. parent reference, 8B *
    3. fullPath reference, 8B + *
    4. hasDeviceDescendant, 1B + *
    5. deviceDescendantComputed, 1B *
    *
  • MapEntry in parent *
      @@ -256,7 +294,7 @@ public void setLockEntry(LockEntry lockEntry) { */ @Override public int estimateSize() { - return 8 + 8 + 8 + 8 + 8 + 8 + 28 + cacheMNodeInfo.estimateSize(); + return 8 + 8 + 8 + 8 + 1 + 1 + 8 + 8 + 28 + cacheMNodeInfo.estimateSize(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java index 24b7e93939e03..93c0d9e31c70e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.path.fa.IFAState; import org.apache.iotdb.commons.path.fa.IFATransition; +import org.apache.iotdb.commons.path.fa.match.IStateMatchInfo; import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; @@ -32,18 +33,22 @@ import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.MNodeIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.ReentrantReadOnlyCachedMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MNodeUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.Set; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; @@ -247,10 +252,107 @@ protected Iterator getChildrenIterator(N parent) throws MetadataException { if (parent.isAboveDatabase()) { return new MNodeIterator<>(parent.getChildren().values().iterator()); } else { + final Iterator optimizedIterator = + shouldUseLeafDeviceMeasurementOptimization() + ? getOptimizedChildrenIterator(parent) + : null; + if (optimizedIterator != null) { + return optimizedIterator; + } return store.getTraverserIterator(parent, templateMap, skipPreDeletedSchema); } } + protected boolean shouldUseLeafDeviceMeasurementOptimization() { + return false; + } + + @SuppressWarnings("unchecked") + private Iterator getOptimizedChildrenIterator(final N parent) throws MetadataException { + if (!parent.isDevice()) { + return null; + } + + if (parent instanceof IMemMNode) { + if (((IMemMNode) parent).hasDeviceDescendant()) { + return null; + } + } else if (parent instanceof ICachedMNode) { + if (hasDeviceDescendant((ICachedMNode) parent)) { + return null; + } + } else { + return null; + } + + final IStateMatchInfo stateMatchInfo = getCurrentStateMatchInfo(); + final Set candidateNames = new LinkedHashSet<>(); + + for (int i = 0; i < stateMatchInfo.getMatchedStateSize(); i++) { + final IFAState matchedState = stateMatchInfo.getMatchedState(i); + + for (final IFATransition transition : + patternFA.getPreciseMatchTransition(matchedState).values()) { + if (patternFA.getNextState(matchedState, transition).isFinal()) { + candidateNames.add(transition.getAcceptEvent()); + } + } + + final Iterator fuzzyTransitionIterator = + patternFA.getFuzzyMatchTransitionIterator(matchedState); + while (fuzzyTransitionIterator.hasNext()) { + if (patternFA.getNextState(matchedState, fuzzyTransitionIterator.next()).isFinal()) { + return null; + } + } + } + + if (candidateNames.isEmpty()) { + return null; + } + + // For leaf devices, `**.measurement` does not need to enumerate every measurement child. + try { + return getChildrenIterator(parent, candidateNames.iterator()); + } catch (final MetadataException e) { + throw e; + } catch (final Exception e) { + throw new MetadataException(e.getMessage(), e); + } + } + + @SuppressWarnings("unchecked") + private boolean hasDeviceDescendant(final ICachedMNode node) throws MetadataException { + if (node.isMeasurement()) { + node.setHasDeviceDescendant(false); + node.setDeviceDescendantComputed(true); + return false; + } + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + return node.hasDeviceDescendant(); + } + + @SuppressWarnings("unchecked") + private boolean hasDeviceDescendantInChildren(final ICachedMNode node) throws MetadataException { + final IMTreeStore cachedStore = (IMTreeStore) store; + try (final IMNodeIterator iterator = cachedStore.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final ICachedMNode child = iterator.next(); + try { + if (child.isDevice() || hasDeviceDescendant(child)) { + return true; + } + } finally { + cachedStore.unPin(child); + } + } + return false; + } + } + @Override protected void releaseNodeIterator(Iterator nodeIterator) { if (nodeIterator instanceof IMNodeIterator) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java index 4996e3f0c8b36..490145d5371fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java @@ -84,4 +84,9 @@ protected boolean shouldVisitSubtreeOfFullMatchedNode(N node) { protected boolean shouldVisitSubtreeOfInternalMatchedNode(N node) { return !node.isMeasurement(); } + + @Override + protected boolean shouldUseLeafDeviceMeasurementOptimization() { + return true; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java index 601233d9d6094..3de059127d3c0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java @@ -1185,6 +1185,28 @@ public void testGetMatchedDevicesWithSpecialPattern() throws Exception { Assert.assertEquals(expectedPathList, actualPathList); } + @Test + public void testShowTimeseriesWildcardSuffixWithNestedAndLeafDevices() throws Exception { + final ISchemaRegion schemaRegion = getSchemaRegion("root.test", 0); + + SchemaRegionTestUtil.createSimpleTimeSeriesByList( + schemaRegion, + Arrays.asList( + "root.test.d1.s1", "root.test.d1.a.d2.s1", "root.test.d3.s1", "root.test.d4.s2")); + + final List result = + SchemaRegionTestUtil.showTimeseries(schemaRegion, new PartialPath("root.test.**.s1")); + final Set expectedPathList = + new HashSet<>(Arrays.asList("root.test.d1.s1", "root.test.d1.a.d2.s1", "root.test.d3.s1")); + Assert.assertEquals(expectedPathList.size(), result.size()); + + final Set actualPathList = new HashSet<>(); + for (final ITimeSeriesSchemaInfo timeSeriesSchemaInfo : result) { + actualPathList.add(timeSeriesSchemaInfo.getFullPath()); + } + Assert.assertEquals(expectedPathList, actualPathList); + } + @Test public void testGetMatchedDevicesWithSpecialPattern2() throws Exception { final ISchemaRegion schemaRegion = getSchemaRegion("root.test", 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java new file mode 100644 index 0000000000000..1dd3fa456b791 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; +import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaEngineStatistics; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MeasurementCollector; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class MTreeBelowSGMemoryImplTest { + + @Test + public void testDeviceDescendantFlagIsMaintainedAcrossCreateDeleteAndRebuild() throws Exception { + final MTreeBelowSGMemoryImpl mtree = newMTree(); + + mtree.createTimeSeries( + new MeasurementPath("root.sg.a.b.s1"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + + IMemMNode database = mtree.getNodeByPath(new PartialPath("root.sg")); + IMemMNode aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + IMemMNode bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(bNode.isDevice()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + + mtree.createTimeSeries( + new MeasurementPath("root.sg.a.s0"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + mtree.createTimeSeries( + new MeasurementPath("root.sg.c.d.s2"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + IMemMNode cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertTrue(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(cNode.hasDeviceDescendant()); + + database.setHasDeviceDescendant(false); + aNode.setHasDeviceDescendant(false); + bNode.setHasDeviceDescendant(true); + cNode.setHasDeviceDescendant(false); + mtree.rebuildSubtreeMeasurementCount(); + + database = mtree.getNodeByPath(new PartialPath("root.sg")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + Assert.assertTrue(cNode.hasDeviceDescendant()); + + mtree.deleteTimeSeries(new PartialPath("root.sg.a.s0")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + + mtree.deleteTimeSeries(new PartialPath("root.sg.a.b.s1")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertTrue(database.hasDeviceDescendant()); + assertPathNotExist(mtree, new PartialPath("root.sg.a")); + + mtree.deleteTimeSeries(new PartialPath("root.sg.c.d.s2")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertFalse(database.hasDeviceDescendant()); + assertPathNotExist(mtree, new PartialPath("root.sg.c")); + } + + @Test + public void testDeviceDescendantFlagIsMaintainedAcrossTableDeviceDrop() throws Exception { + final MTreeBelowSGMemoryImpl mtree = newMTree(); + final AtomicInteger attributePointer = new AtomicInteger(); + + mtree.createOrUpdateTableDevice( + "t", + new String[] {"hebei", "p_1", "d_0"}, + attributePointer::getAndIncrement, + ignored -> {}); + + IMemMNode database = mtree.getNodeByPath(new PartialPath("root.sg")); + final IMemMNode table = mtree.getNodeByPath(new PartialPath("root.sg.t")); + final IMemMNode province = mtree.getNodeByPath(new PartialPath("root.sg.t.hebei")); + final IMemMNode device = mtree.getNodeByPath(new PartialPath("root.sg.t.hebei.p_1.d_0")); + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(table.hasDeviceDescendant()); + Assert.assertTrue(province.hasDeviceDescendant()); + Assert.assertTrue(device.isDevice()); + Assert.assertFalse(device.hasDeviceDescendant()); + + Assert.assertTrue(mtree.deleteTableDevice("t", ignored -> {})); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertFalse(database.hasDeviceDescendant()); + assertPathNotExist(mtree, new PartialPath("root.sg.t")); + } + + @Test + public void testLeafDeviceWildcardSuffixUsesDirectMeasurementLookup() throws Exception { + final PartialPath databasePath = PartialPath.getQualifiedDatabasePartialPath("root.sg"); + final MemSchemaRegionStatistics regionStatistics = newRegionStatistics(); + final CountingMemMTreeStore store = + new CountingMemMTreeStore( + databasePath, regionStatistics, new SchemaRegionMemMetric(regionStatistics, "root.sg")); + final IMNodeFactory nodeFactory = + MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); + + final IMemMNode databaseMNode = store.getRoot(); + final IMemMNode rootNode = store.generatePrefix(databasePath); + final IMemMNode deviceNode = + store.addChild(databaseMNode, "d1", nodeFactory.createInternalMNode(databaseMNode, "d1")); + store.setToEntity(deviceNode); + + for (int i = 0; i < 64; i++) { + final String measurement = i == 0 ? "target" : "s" + i; + final IMeasurementMNode measurementMNode = + nodeFactory.createMeasurementMNode( + deviceNode.getAsDeviceMNode(), + measurement, + new MeasurementSchema( + measurement, TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY), + null); + store.addChild(deviceNode, measurement, measurementMNode.getAsMNode()); + } + + store.watchParent(deviceNode); + final List matchedPaths = new ArrayList<>(); + try (final MeasurementCollector collector = + new MeasurementCollector( + rootNode, + new PartialPath("root.sg.**.target"), + store, + false, + SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected Void collectMeasurement(final IMeasurementMNode node) { + matchedPaths.add(getCurrentMeasurementPathInTraverse(node).getFullPath()); + return null; + } + }) { + collector.traverse(); + } + + Assert.assertEquals(Collections.singletonList("root.sg.d1.target"), matchedPaths); + Assert.assertEquals(0, store.getWatchedTraverserIteratorCount()); + Assert.assertEquals(1, store.getWatchedChildLookupCount()); + } + + private static MTreeBelowSGMemoryImpl newMTree() throws Exception { + final MemSchemaRegionStatistics regionStatistics = newRegionStatistics(); + return new MTreeBelowSGMemoryImpl( + PartialPath.getQualifiedDatabasePartialPath("root.sg"), + node -> Collections.emptyMap(), + node -> Collections.emptyMap(), + regionStatistics, + new SchemaRegionMemMetric(regionStatistics, "root.sg")); + } + + private static MemSchemaRegionStatistics newRegionStatistics() { + return new MemSchemaRegionStatistics(0, new MemSchemaEngineStatistics()); + } + + private static void assertPathNotExist(final MTreeBelowSGMemoryImpl mtree, final PartialPath path) + throws MetadataException { + try { + mtree.getNodeByPath(path); + Assert.fail("Expected path not exist: " + path.getFullPath()); + } catch (final PathNotExistException ignored) { + // expected + } + } + + private static class CountingMemMTreeStore extends MemMTreeStore { + private IMemMNode watchedParent; + private int watchedTraverserIteratorCount; + private int watchedChildLookupCount; + + private CountingMemMTreeStore( + final PartialPath rootPath, + final MemSchemaRegionStatistics regionStatistics, + final SchemaRegionMemMetric metric) { + super(rootPath, regionStatistics, metric); + } + + private void watchParent(final IMemMNode parent) { + this.watchedParent = parent; + this.watchedTraverserIteratorCount = 0; + this.watchedChildLookupCount = 0; + } + + private int getWatchedTraverserIteratorCount() { + return watchedTraverserIteratorCount; + } + + private int getWatchedChildLookupCount() { + return watchedChildLookupCount; + } + + @Override + public IMemMNode getChild(final IMemMNode parent, final String name) { + if (parent == watchedParent) { + watchedChildLookupCount++; + } + return super.getChild(parent, name); + } + + @Override + public IMNodeIterator getTraverserIterator( + final IMemMNode parent, + final Map templateMap, + final boolean skipPreDeletedSchema) + throws MetadataException { + if (parent == watchedParent) { + watchedTraverserIteratorCount++; + } + return super.getTraverserIterator(parent, templateMap, skipPreDeletedSchema); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java new file mode 100644 index 0000000000000..81936640b4946 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; +import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaEngineStatistics; +import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MeasurementCollector; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.external.commons.io.FileUtils; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class MTreeBelowSGCachedImplTest { + + private static final Field STORE_FIELD; + private static final Field ROOT_NODE_FIELD; + + static { + try { + STORE_FIELD = MTreeBelowSGCachedImpl.class.getDeclaredField("store"); + STORE_FIELD.setAccessible(true); + ROOT_NODE_FIELD = MTreeBelowSGCachedImpl.class.getDeclaredField("rootNode"); + ROOT_NODE_FIELD.setAccessible(true); + } catch (final NoSuchFieldException e) { + throw new ExceptionInInitializerError(e); + } + } + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private int rawCachedMNodeSize; + private MTreeBelowSGCachedImpl mtree; + + @Before + public void setUp() { + rawCachedMNodeSize = config.getCachedMNodeSizeInPBTreeMode(); + config.setCachedMNodeSizeInPBTreeMode(10000); + ReleaseFlushMonitor.getInstance().clear(); + ReleaseFlushMonitor.getInstance().init(new CachedSchemaEngineStatistics()); + } + + @After + public void tearDown() throws Exception { + if (mtree != null) { + mtree.clear(); + mtree = null; + } + ReleaseFlushMonitor.getInstance().clear(); + FileUtils.deleteDirectory(new File(config.getSchemaDir())); + config.setCachedMNodeSizeInPBTreeMode(rawCachedMNodeSize); + } + + @Test + public void testDeviceDescendantFlagIsMaintainedAcrossCreateDeleteAndLazyRecompute() + throws Exception { + mtree = newMTree(); + + createBooleanTimeSeries("root.sg.a.b.s1"); + + ICachedMNode database = mtree.getNodeByPath(new PartialPath("root.sg")); + ICachedMNode aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + ICachedMNode bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + Assert.assertTrue(bNode.isDevice()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + Assert.assertTrue(bNode.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + mtree.unPinMNode(aNode); + mtree.unPinMNode(bNode); + + createBooleanTimeSeries("root.sg.a.s0"); + createBooleanTimeSeries("root.sg.c.d.s2"); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + ICachedMNode cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertTrue(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + Assert.assertTrue(cNode.hasDeviceDescendant()); + Assert.assertTrue(cNode.isDeviceDescendantComputed()); + + aNode.setHasDeviceDescendant(false); + aNode.setDeviceDescendantComputed(false); + mtree.unPinMNode(aNode); + cNode.setHasDeviceDescendant(false); + cNode.setDeviceDescendantComputed(false); + mtree.unPinMNode(cNode); + + Assert.assertEquals( + Collections.singletonList("root.sg.a.b.s1"), + collectMeasurementPaths(new PartialPath("root.sg.a.**.s1"))); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + mtree.unPinMNode(aNode); + cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertFalse(cNode.isDeviceDescendantComputed()); + mtree.unPinMNode(cNode); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.s0")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + mtree.unPinMNode(aNode); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.b.s1")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + assertPathNotExist(new PartialPath("root.sg.a")); + + mtree.deleteTimeseries(new PartialPath("root.sg.c.d.s2")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertFalse(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + assertPathNotExist(new PartialPath("root.sg.c")); + } + + private MTreeBelowSGCachedImpl newMTree() throws Exception { + final CachedSchemaRegionStatistics regionStatistics = + new CachedSchemaRegionStatistics(0, new CachedSchemaEngineStatistics()); + return new MTreeBelowSGCachedImpl( + PartialPath.getQualifiedDatabasePartialPath("root.sg"), + node -> Collections.emptyMap(), + node -> Collections.emptyMap(), + () -> {}, + node -> {}, + node -> {}, + 0, + regionStatistics, + new SchemaRegionCachedMetric(regionStatistics, "root.sg")); + } + + private void createBooleanTimeSeries(final String path) throws MetadataException { + mtree.createTimeSeries( + new MeasurementPath(path), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null); + } + + private List collectMeasurementPaths(final PartialPath pattern) throws Exception { + final ICachedMNode rootNode = (ICachedMNode) ROOT_NODE_FIELD.get(mtree); + final CachedMTreeStore store = (CachedMTreeStore) STORE_FIELD.get(mtree); + final List matchedPaths = new ArrayList<>(); + try (MeasurementCollector collector = + new MeasurementCollector( + rootNode, pattern, store, false, SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected Void collectMeasurement(final IMeasurementMNode node) { + matchedPaths.add(getCurrentMeasurementPathInTraverse(node).getFullPath()); + return null; + } + }) { + collector.traverse(); + } + return matchedPaths; + } + + private void assertPathNotExist(final PartialPath path) throws MetadataException { + try { + mtree.getNodeByPath(path); + Assert.fail("Expected path not exist: " + path.getFullPath()); + } catch (final PathNotExistException ignored) { + // expected + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java index ab0592480e5e2..fc379e34e5b02 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java @@ -37,6 +37,10 @@ public AbstractAboveDatabaseMNode(BasicNode basicMNode) { this.basicMNode = basicMNode; } + public BasicNode getBasicMNode() { + return basicMNode; + } + @Override public String getName() { return basicMNode.getName(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java index 75fd9d40c67dc..d1f410c546015 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java @@ -22,7 +22,7 @@ import java.util.Iterator; -public interface IMNodeIterator> extends Iterator { +public interface IMNodeIterator> extends Iterator, AutoCloseable { void skipTemplateChildren(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java index a60a4ff121ad7..512270c379a87 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java @@ -415,6 +415,10 @@ protected Iterator getCurrentChildrenIterator() { } } + protected final IStateMatchInfo getCurrentStateMatchInfo() { + return currentStateMatchInfo; + } + // Release a child node. protected void releaseNode(N node) {}