Skip to content

[To dev/1.3] Added flags to mark whether a device has device descendants to optimize query like select xx from xxx.** #17802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jt2594838 merged 1 commit into from
Jun 2, 2026
Merged

[To dev/1.3] Added flags to mark whether a device has device descendants to optimize query like select xx from xxx.** #17802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 110 additions & 20 deletions .../org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,59 @@ public synchronized boolean createSnapshot(File snapshotDir) {
return store.createSnapshot(snapshotDir);
}

private void applySubtreeMeasurementDelta(IMemMNode startNode, final long delta) {
if (delta == 0 || startNode == null) {
return;
}
IMemMNode current = startNode;
while (current != null) {
current.setSubtreeMeasurementCount(current.getSubtreeMeasurementCount() + delta);
current = current.getParent();
}
}

private IDeviceMNode<IMemMNode> setToEntityAndUpdateFlags(final IMemMNode node) {
final boolean wasDevice = node.isDevice();
final IDeviceMNode<IMemMNode> deviceMNode = store.setToEntity(node);
if (!wasDevice) {
markAncestorsHavingDeviceDescendant(node);
}
return deviceMNode;
}

private void markAncestorsHavingDeviceDescendant(final IMemMNode deviceNode) {
IMemMNode current = deviceNode.getParent();
while (current != null && !current.hasDeviceDescendant()) {
current.setHasDeviceDescendant(true);
current = current.getParent();
}
}

private boolean hasDeviceDescendantInChildren(final IMemMNode node) {
try (final IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(node)) {
while (iterator.hasNext()) {
final IMemMNode child = iterator.next();
if (child.isDevice() || child.hasDeviceDescendant()) {
return true;
}
}
return false;
}
}

private void refreshAncestorsHavingDeviceDescendant(IMemMNode startNode) {
IMemMNode current = startNode;
while (current != null) {
current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current));
current = current.getParent();
}
}

private long getTemplateMeasurementCount(final int templateId) {
final Template template = ClusterTemplateManager.getInstance().getTemplate(templateId);
return template == null ? 0L : template.getMeasurementNumber();
}

public static MTreeBelowSGMemoryImpl loadFromSnapshot(
File snapshotDir,
String storageGroupFullPath,
Expand All @@ -184,13 +237,16 @@ public static MTreeBelowSGMemoryImpl loadFromSnapshot(
Function<IMeasurementMNode<IMemMNode>, Map<String, String>> tagGetter,
Function<IMeasurementMNode<IMemMNode>, Map<String, String>> attributeGetter)
throws IOException, IllegalPathException {
return new MTreeBelowSGMemoryImpl(
new PartialPath(storageGroupFullPath),
MemMTreeStore.loadFromSnapshot(
snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric),
tagGetter,
attributeGetter,
regionStatistics);
final MTreeBelowSGMemoryImpl mtree =
new MTreeBelowSGMemoryImpl(
new PartialPath(storageGroupFullPath),
MemMTreeStore.loadFromSnapshot(
snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric),
tagGetter,
attributeGetter,
regionStatistics);
mtree.rebuildSubtreeMeasurementCount();
return mtree;
}

// endregion
Expand Down Expand Up @@ -268,7 +324,7 @@ public IMeasurementMNode<IMemMNode> createTimeSeries(
if (device.isDevice()) {
entityMNode = device.getAsDeviceMNode();
} else {
entityMNode = store.setToEntity(device);
entityMNode = setToEntityAndUpdateFlags(device);
}

// create a non-aligned time series
Expand All @@ -290,6 +346,7 @@ public IMeasurementMNode<IMemMNode> createTimeSeries(
entityMNode.addAlias(alias, measurementMNode);
}

applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L);
return measurementMNode;
}
}
Expand Down Expand Up @@ -360,7 +417,7 @@ public List<IMeasurementMNode<IMemMNode>> createAlignedTimeSeries(
if (device.isDevice()) {
entityMNode = device.getAsDeviceMNode();
} else {
entityMNode = store.setToEntity(device);
entityMNode = setToEntityAndUpdateFlags(device);
entityMNode.setAligned(true);
}

Expand All @@ -386,6 +443,7 @@ public List<IMeasurementMNode<IMemMNode>> createAlignedTimeSeries(
if (aliasList != null && aliasList.get(i) != null) {
entityMNode.addAlias(aliasList.get(i), measurementMNode);
}
applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L);
measurementMNodeList.add(measurementMNode);
}
return measurementMNodeList;
Expand Down Expand Up @@ -539,6 +597,7 @@ public IMeasurementMNode<IMemMNode> deleteTimeseries(PartialPath path) throws Me
if (deletedNode.getAlias() != null) {
parent.getAsDeviceMNode().deleteAliasChild(deletedNode.getAlias());
}
applySubtreeMeasurementDelta(parent, -1L);
}
deleteEmptyInternalMNode(parent.getAsDeviceMNode());
return deletedNode;
Expand All @@ -551,14 +610,15 @@ public void deleteEmptyInternalMNode(IDeviceMNode<IMemMNode> entityMNode) {
boolean hasMeasurement = false;
boolean hasNonViewMeasurement = false;
IMemMNode child;
IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(curNode);
while (iterator.hasNext()) {
child = iterator.next();
if (child.isMeasurement()) {
hasMeasurement = true;
if (!child.getAsMeasurementMNode().isLogicalView()) {
hasNonViewMeasurement = true;
break;
try (final IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(curNode)) {
while (iterator.hasNext()) {
child = iterator.next();
if (child.isMeasurement()) {
hasMeasurement = true;
if (!child.getAsMeasurementMNode().isLogicalView()) {
hasNonViewMeasurement = true;
break;
}
}
}
}
Expand All @@ -567,6 +627,7 @@ public void deleteEmptyInternalMNode(IDeviceMNode<IMemMNode> entityMNode) {
synchronized (this) {
curNode = store.setToInternal(entityMNode);
}
refreshAncestorsHavingDeviceDescendant(curNode.getParent());
} else if (!hasNonViewMeasurement) {
// has some measurement but they are all logical view
entityMNode.setAligned(null);
Expand Down Expand Up @@ -889,7 +950,7 @@ public void activateTemplate(PartialPath activatePath, Template template)
if (cur.isDevice()) {
entityMNode = cur.getAsDeviceMNode();
} else {
entityMNode = store.setToEntity(cur);
entityMNode = setToEntityAndUpdateFlags(cur);
}
}

Expand All @@ -907,6 +968,7 @@ public void activateTemplate(PartialPath activatePath, Template template)
entityMNode.setUseTemplate(true);
entityMNode.setSchemaTemplateId(template.getId());
regionStatistics.activateTemplate(template.getId());
applySubtreeMeasurementDelta(entityMNode.getAsMNode(), (long) template.getMeasurementNumber());
}

public Map<PartialPath, List<Integer>> constructSchemaBlackListWithTemplate(
Expand Down Expand Up @@ -968,6 +1030,8 @@ protected void updateEntity(IDeviceMNode<IMemMNode> node) {
resultTemplateSetInfo.put(
node.getPartialPath(), Collections.singletonList(node.getSchemaTemplateId()));
regionStatistics.deactivateTemplate(node.getSchemaTemplateId());
applySubtreeMeasurementDelta(
node.getAsMNode(), -getTemplateMeasurementCount(node.getSchemaTemplateId()));
node.deactivateTemplate();
deleteEmptyInternalMNode(node);
}
Expand All @@ -991,7 +1055,7 @@ public void activateTemplateWithoutCheck(
if (cur.isDevice()) {
entityMNode = cur.getAsDeviceMNode();
} else {
entityMNode = store.setToEntity(cur);
entityMNode = setToEntityAndUpdateFlags(cur);
}

if (!entityMNode.isAligned()) {
Expand All @@ -1000,6 +1064,7 @@ public void activateTemplateWithoutCheck(
entityMNode.setUseTemplate(true);
entityMNode.setSchemaTemplateId(templateId);
regionStatistics.activateTemplate(templateId);
applySubtreeMeasurementDelta(entityMNode.getAsMNode(), getTemplateMeasurementCount(templateId));
}

public long countPathsUsingTemplate(PartialPath pathPattern, int templateId)
Expand All @@ -1011,6 +1076,30 @@ public long countPathsUsingTemplate(PartialPath pathPattern, int templateId)
}
}

public void rebuildSubtreeMeasurementCount() {
rebuildSubtreeMeasurementCountFromNode(rootNode);
}

private long rebuildSubtreeMeasurementCountFromNode(final IMemMNode node) {
long count = node.isMeasurement() ? 1L : 0L;
boolean hasDeviceDescendant = false;
try (final IMNodeIterator<IMemMNode> iterator = store.getChildrenIterator(node)) {
while (iterator.hasNext()) {
final IMemMNode child = iterator.next();
count += rebuildSubtreeMeasurementCountFromNode(child);
if (child.isDevice() || child.hasDeviceDescendant()) {
hasDeviceDescendant = true;
}
}
}
if (node.isDevice() && node.getAsDeviceMNode().isUseTemplate()) {
count += getTemplateMeasurementCount(node.getAsDeviceMNode().getSchemaTemplateId());
}
node.setSubtreeMeasurementCount(count);
node.setHasDeviceDescendant(hasDeviceDescendant);
return count;
}

// endregion

// region Interfaces for schema reader
Expand Down Expand Up @@ -1278,7 +1367,7 @@ public IMeasurementMNode<IMemMNode> createLogicalView(
if (device.isDevice()) {
entityMNode = device.getAsDeviceMNode();
} else {
entityMNode = store.setToEntity(device);
entityMNode = setToEntityAndUpdateFlags(device);
// this parent has no measurement before. The leafName is his first child who is a logical
// view.
entityMNode.setAligned(null);
Expand All @@ -1287,6 +1376,7 @@ public IMeasurementMNode<IMemMNode> createLogicalView(
measurementMNode.setParent(entityMNode.getAsMNode());
store.addChild(entityMNode.getAsMNode(), leafName, measurementMNode.getAsMNode());

applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L);
return measurementMNode;
}
}
Expand Down
30 changes: 29 additions & 1 deletion ...in/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,33 @@
package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode;

import org.apache.iotdb.commons.schema.node.IMNode;
import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.basic.BasicMNode;

public interface IMemMNode extends IMNode<IMemMNode> {}
public interface IMemMNode extends IMNode<IMemMNode> {

BasicMNode getBasicMNode();

/**
* The count of measurement nodes contained in the subtree rooted at this node. The counter is
* maintained in memory only.
*/
default long getSubtreeMeasurementCount() {
return getBasicMNode().getSubtreeMeasurementCount();
}

default void setSubtreeMeasurementCount(final long subtreeMeasurementCount) {
getBasicMNode().setSubtreeMeasurementCount(subtreeMeasurementCount);
}

/**
* Whether there is any device node in the subtree rooted at this node, excluding the node itself.
* This flag is maintained in memory only.
*/
default boolean hasDeviceDescendant() {
return getBasicMNode().hasDeviceDescendant();
}

default void setHasDeviceDescendant(final boolean hasDeviceDescendant) {
getBasicMNode().setHasDeviceDescendant(hasDeviceDescendant);
}
}
34 changes: 33 additions & 1 deletion .../org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ public class BasicMNode implements IMemMNode {
private IMemMNode parent;
private final BasicMNodeInfo basicMNodeInfo;

/** Cached count of measurements in this node's subtree, rebuilt on restart. */
private long subtreeMeasurementCount = 0L;

/** Cached flag showing whether there is any device in the subtree below this node. */
private boolean hasDeviceDescendant = false;

/** from root to this node, only be set when used once for InternalMNode */
private String fullPath;

Expand All @@ -55,6 +61,11 @@ public BasicMNode(IMemMNode parent, String name) {
this.basicMNodeInfo = new BasicMNodeInfo(name);
}

@Override
public BasicMNode getBasicMNode() {
return this;
}

@Override
public String getName() {
return basicMNodeInfo.getName();
Expand Down Expand Up @@ -98,6 +109,25 @@ public void setFullPath(String fullPath) {
this.fullPath = fullPath;
}

public long getSubtreeMeasurementCount() {
return subtreeMeasurementCount;
}

@Override
public void setSubtreeMeasurementCount(final long subtreeMeasurementCount) {
this.subtreeMeasurementCount = subtreeMeasurementCount;
}

@Override
public boolean hasDeviceDescendant() {
return hasDeviceDescendant;
}

@Override
public void setHasDeviceDescendant(final boolean hasDeviceDescendant) {
this.hasDeviceDescendant = hasDeviceDescendant;
}

@Override
public PartialPath getPartialPath() {
List<String> detachedPath = new ArrayList<>();
Expand Down Expand Up @@ -224,6 +254,8 @@ public <R, C> R accept(MNodeVisitor<R, C> visitor, C context) {
* <li>basicMNodeInfo reference, 8B
* <li>parent reference, 8B
* <li>fullPath reference, 8B
* <li>subtreeMeasurementCount, 8B
* <li>hasDeviceDescendant, 1B
* </ol>
* <li>MapEntry in parent
* <ol>
Expand All @@ -235,7 +267,7 @@ public <R, C> R accept(MNodeVisitor<R, C> visitor, C context) {
*/
@Override
public int estimateSize() {
return 8 + 8 + 8 + 8 + 8 + 8 + 28 + basicMNodeInfo.estimateSize();
return 8 + 8 + 8 + 8 + 8 + 1 + 8 + 8 + 28 + basicMNodeInfo.estimateSize();
}

@Override
Expand Down
Loading
Loading