Skip to content

Commit 3b85dde

Browse files
shuwenweiJackieTien97
shuwenwei
authored and
JackieTien97
committed
Skip TTL check when there is no TTL and no mods file in the data region (#16110)
* Skip TTL check when there is no ttl in the data region and no mods file * check back quote (cherry picked from commit 77f1dcc)
1 parent e4b097a commit 3b85dde

5 files changed

Lines changed: 168 additions & 4 deletions

File tree

  • iotdb-core

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,8 @@ public long getDatabaseMaxTTL(final String database) {
169169
lock.readLock().lock();
170170
try {
171171
return ttlCache.getDatabaseMaxTTL(database);
172+
} catch (IllegalPathException e) {
173+
return TTLCache.NULL_TTL;
172174
} finally {
173175
lock.readLock().unlock();
174176
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,15 @@ public long getTTLInMSForTree(String[] path) {
109109
}
110110
}
111111

112+
public boolean dataInDatabaseMayHaveTTL(String db) throws IllegalPathException {
113+
lock.readLock().lock();
114+
try {
115+
return treeModelTTLCache.dataInDatabaseMayHaveTTL(db);
116+
} finally {
117+
lock.readLock().unlock();
118+
}
119+
}
120+
112121
/**
113122
* Get ttl of one specific path node without time precision conversion. If this node does not set
114123
* ttl, then return -1.

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.iotdb.commons.path.IFullPath;
3232
import org.apache.iotdb.commons.path.MeasurementPath;
3333
import org.apache.iotdb.commons.schema.SchemaConstant;
34+
import org.apache.iotdb.commons.schema.table.InformationSchema;
3435
import org.apache.iotdb.commons.schema.table.TsTable;
3536
import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil;
3637
import org.apache.iotdb.commons.service.metric.MetricService;
@@ -42,6 +43,8 @@
4243
import org.apache.iotdb.commons.utils.TestOnly;
4344
import org.apache.iotdb.commons.utils.TimePartitionUtils;
4445
import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp;
46+
import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp;
47+
import org.apache.iotdb.confignode.rpc.thrift.TTableInfo;
4548
import org.apache.iotdb.consensus.ConsensusFactory;
4649
import org.apache.iotdb.db.conf.IoTDBConfig;
4750
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -62,6 +65,7 @@
6265
import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager;
6366
import org.apache.iotdb.db.pipe.consensus.deletion.persist.PageCacheDeletionBuffer;
6467
import org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
68+
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
6569
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
6670
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
6771
import org.apache.iotdb.db.queryengine.common.DeviceContext;
@@ -201,6 +205,7 @@
201205
import java.util.concurrent.locks.ReadWriteLock;
202206
import java.util.concurrent.locks.ReentrantReadWriteLock;
203207
import java.util.stream.Collectors;
208+
import java.util.stream.Stream;
204209

205210
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
206211
import static org.apache.iotdb.commons.utils.PathUtils.isTableModelDatabase;
@@ -2904,9 +2909,12 @@ public int executeTTLCheck() throws InterruptedException {
29042909
// wait until success
29052910
Thread.sleep(500);
29062911
}
2907-
logger.info("[TTL] {}-{} Start ttl checking.", databaseName, dataRegionId);
29082912
int trySubmitCount = 0;
29092913
try {
2914+
if (skipCurrentTTLAndModificationCheck()) {
2915+
return 0;
2916+
}
2917+
logger.info("[TTL] {}-{} Start ttl and modification checking.", databaseName, dataRegionId);
29102918
CompactionScheduleContext context = new CompactionScheduleContext();
29112919
List<Long> timePartitions = new ArrayList<>(tsFileManager.getTimePartitions());
29122920
// Sort the time partition from smallest to largest
@@ -2942,6 +2950,49 @@ public int executeTTLCheck() throws InterruptedException {
29422950
return trySubmitCount;
29432951
}
29442952

2953+
private boolean skipCurrentTTLAndModificationCheck() {
2954+
if (this.databaseName.equals(InformationSchema.INFORMATION_DATABASE)) {
2955+
return true;
2956+
}
2957+
for (Long timePartition : getTimePartitions()) {
2958+
List<TsFileResource> seqFiles = tsFileManager.getTsFileListSnapshot(timePartition, true);
2959+
List<TsFileResource> unseqFiles = tsFileManager.getTsFileListSnapshot(timePartition, false);
2960+
boolean modFileExists =
2961+
Stream.concat(seqFiles.stream(), unseqFiles.stream())
2962+
.anyMatch(TsFileResource::anyModFileExists);
2963+
if (modFileExists) {
2964+
return false;
2965+
}
2966+
}
2967+
boolean isTableModel = isTableModelDatabase(this.databaseName);
2968+
if (isTableModel) {
2969+
try (final ConfigNodeClient configNodeClient =
2970+
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
2971+
TShowTableResp resp = configNodeClient.showTables(databaseName, false);
2972+
for (TTableInfo tTableInfo : resp.getTableInfoList()) {
2973+
String ttl = tTableInfo.getTTL();
2974+
if (ttl == null || ttl.equals(IoTDBConstant.TTL_INFINITE)) {
2975+
continue;
2976+
}
2977+
long ttlValue = Long.parseLong(ttl);
2978+
if (ttlValue < 0 || ttlValue == Long.MAX_VALUE) {
2979+
continue;
2980+
}
2981+
return false;
2982+
}
2983+
return true;
2984+
} catch (Exception ignored) {
2985+
return false;
2986+
}
2987+
} else {
2988+
try {
2989+
return !DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL(databaseName);
2990+
} catch (Exception ignored) {
2991+
return false;
2992+
}
2993+
}
2994+
}
2995+
29452996
protected int[] executeInsertionCompaction(
29462997
List<Long> timePartitions, CompactionScheduleContext context) throws InterruptedException {
29472998
int[] trySubmitCountOfTimePartitions = new int[timePartitions.size()];

‎iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.apache.tsfile.read.common.block.TsBlock;
6161
import org.apache.tsfile.write.schema.MeasurementSchema;
6262
import org.junit.After;
63+
import org.junit.Assert;
6364
import org.junit.Before;
6465
import org.junit.Test;
6566

@@ -259,6 +260,57 @@ public void testTTLRead()
259260
assertTrue(cnt == 0);
260261
}
261262

263+
@Test
264+
public void testTTLRead2() throws IllegalPathException {
265+
DataNodeTTLCache.getInstance().setTTLForTree("root.test1.**", 500);
266+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
267+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
268+
269+
DataNodeTTLCache.getInstance().setTTLForTree("root.**", 500);
270+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
271+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
272+
273+
DataNodeTTLCache.getInstance().setTTLForTree("root.test1.d1.**", 500);
274+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
275+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
276+
277+
DataNodeTTLCache.getInstance().setTTLForTree("root.test2.**", 500);
278+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
279+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
280+
281+
DataNodeTTLCache.getInstance().setTTLForTree("root.test2.d1.**", 500);
282+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
283+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
284+
285+
DataNodeTTLCache.getInstance().setTTLForTree("root.**", 500);
286+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
287+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
288+
289+
DataNodeTTLCache.getInstance().setTTLForTree("root.test.**", 500);
290+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
291+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
292+
293+
DataNodeTTLCache.getInstance().setTTLForTree("root.test1.**", 500);
294+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
295+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
296+
297+
DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg1.**", 500);
298+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
299+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
300+
301+
DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg2.**", 500);
302+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
303+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
304+
305+
DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg1.d1.**", 500);
306+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
307+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
308+
309+
DataNodeTTLCache.getInstance().setTTLForTree("root.`1.1`.**", 500);
310+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.`1.1`"));
311+
DataNodeTTLCache.getInstance().clearAllTTLForTree();
312+
}
313+
262314
private NonAlignedFullPath mockMeasurementPath() {
263315
return new NonAlignedFullPath(
264316
IDeviceID.Factory.DEFAULT_FACTORY.create(sg1Device),

‎iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java‎

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.iotdb.commons.utils.StatusUtils;
2727
import org.apache.iotdb.rpc.TSStatusCode;
2828

29+
import org.apache.tsfile.common.constant.TsFileConstant;
2930
import org.apache.tsfile.utils.ReadWriteIOUtils;
3031

3132
import javax.annotation.concurrent.NotThreadSafe;
@@ -181,7 +182,7 @@ public long getLastNodeTTL(String[] nodes) {
181182
* @return the maximum ttl of the subtree of the corresponding database. return NULL_TTL if the
182183
* TTL is not set or the database does not exist.
183184
*/
184-
public long getDatabaseMaxTTL(String database) {
185+
public long getDatabaseMaxTTL(String database) throws IllegalPathException {
185186
CacheNode node = ttlCacheTree.searchChild(database);
186187
if (node == null) {
187188
return NULL_TTL;
@@ -199,6 +200,55 @@ public long getDatabaseMaxTTL(String database) {
199200
return maxTTL;
200201
}
201202

203+
public boolean dataInDatabaseMayHaveTTL(String database) throws IllegalPathException {
204+
String[] nodeNames = split(database);
205+
CacheNode current = ttlCacheTree;
206+
for (String nodeName : nodeNames) {
207+
if (hasValidTTLOnCurrentLevel(current)) {
208+
return true;
209+
}
210+
if (nodeName.equals("root")) {
211+
continue;
212+
}
213+
current = current.getChild(nodeName);
214+
if (current == null) {
215+
return false;
216+
}
217+
}
218+
219+
if (hasValidTTLOnCurrentLevel(current)) {
220+
return true;
221+
}
222+
223+
Queue<CacheNode> queue = new LinkedList<>();
224+
queue.add(current);
225+
while (!queue.isEmpty()) {
226+
current = queue.poll();
227+
for (CacheNode child : current.getChildren().values()) {
228+
if (child.ttl >= 0 && child.ttl != Long.MAX_VALUE) {
229+
return true;
230+
}
231+
queue.add(child);
232+
}
233+
}
234+
return false;
235+
}
236+
237+
private boolean hasValidTTLOnCurrentLevel(CacheNode current) {
238+
if (current.ttl >= 0 && current.ttl != Long.MAX_VALUE) {
239+
return true;
240+
}
241+
CacheNode wildcardChild = current.getChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD);
242+
return wildcardChild != null && wildcardChild.ttl >= 0 && wildcardChild.ttl != Long.MAX_VALUE;
243+
}
244+
245+
private static String[] split(String path) throws IllegalPathException {
246+
if (!path.contains(TsFileConstant.BACK_QUOTE_STRING)) {
247+
return path.split(TsFileConstant.PATH_SEPARATER_NO_REGEX);
248+
}
249+
return PathUtils.splitPathToDetachedNodes(path);
250+
}
251+
202252
/**
203253
* @return key is path contains wildcard between each node
204254
*/
@@ -306,8 +356,8 @@ public CacheNode getChild(String name) {
306356
* @param name the name corresponding to the child node, use '.' to separate each node
307357
* @return the child node if it exists, otherwise return null
308358
*/
309-
public CacheNode searchChild(String name) {
310-
String[] nodeNames = name.split("\\.");
359+
public CacheNode searchChild(String name) throws IllegalPathException {
360+
String[] nodeNames = split(name);
311361
CacheNode current = this;
312362
for (String nodeName : nodeNames) {
313363
if (nodeName.equals("root")) {

0 commit comments

Comments
 (0)