[to dev/1.3] Cherry pick 'estimating inner compaction task memory during selection (#15257)' & Fix estimating memory cost for string type in compaction (#16048)

* estimating inner compaction task memory during selection (#15257)

* estimating inner compaction task during selection

* use tsfile id

* rename

* spotless

* modify CompactionEstimateUtils

* fix bug

* fix compile

* fix ut

* fix bug

* fix ut

* fix review

* modify CompactionEstimateUtils

* modify ut

* fix ut

* add log

* add log

* fix estimating memory cost for string type

* modify SystemInfo

* remove Collections.synchronizedMap

Author: shuwenwei

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java

@@ -587,9 +587,6 @@ public class IoTDBConfig {
    */
   private volatile double innerCompactionTaskSelectionDiskRedundancy = 0.05;
 
-  /** The size of global compaction estimation file info cahce. */
-  private int globalCompactionFileInfoCacheSize = 1000;
-
   /** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
   private boolean metaDataCacheEnable = true;
 
@@ -3949,14 +3946,6 @@ public void setCandidateCompactionTaskQueueSize(int candidateCompactionTaskQueue
     this.candidateCompactionTaskQueueSize = candidateCompactionTaskQueueSize;
   }
 
-  public int getGlobalCompactionFileInfoCacheSize() {
-    return globalCompactionFileInfoCacheSize;
-  }
-
-  public void setGlobalCompactionFileInfoCacheSize(int globalCompactionFileInfoCacheSize) {
-    this.globalCompactionFileInfoCacheSize = globalCompactionFileInfoCacheSize;
-  }
-
   public boolean isEnableAuditLog() {
     return enableAuditLog;
   }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/ICrossCompactionPerformer.java

@@ -19,11 +19,17 @@
 
 package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer;
 
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.AbstractCrossSpaceEstimator;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 
 import java.util.List;
+import java.util.Optional;
 
 public interface ICrossCompactionPerformer extends ICompactionPerformer {
   @Override
   void setSourceFiles(List<TsFileResource> seqFiles, List<TsFileResource> unseqFiles);
+
+  default Optional<AbstractCrossSpaceEstimator> getCrossSpaceEstimator() {
+    return Optional.empty();
+  }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/IInnerCompactionPerformer.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer;
+
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.AbstractInnerSpaceEstimator;
+
+import java.util.Optional;
+
+public interface IInnerCompactionPerformer extends ICompactionPerformer {
+  default Optional<AbstractInnerSpaceEstimator> getInnerSpaceEstimator() {
+    return Optional.empty();
+  }
+}

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/ISeqCompactionPerformer.java

@@ -23,7 +23,7 @@
 
 import java.util.List;
 
-public interface ISeqCompactionPerformer extends ICompactionPerformer {
+public interface ISeqCompactionPerformer extends IInnerCompactionPerformer {
   @Override
   void setSourceFiles(List<TsFileResource> seqFiles);
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/IUnseqCompactionPerformer.java

@@ -23,7 +23,7 @@
 
 import java.util.List;
 
-public interface IUnseqCompactionPerformer extends ICompactionPerformer {
+public interface IUnseqCompactionPerformer extends IInnerCompactionPerformer {
   @Override
   void setSourceFiles(List<TsFileResource> unseqFiles);
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java

@@ -39,6 +39,10 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.FastCrossCompactionWriter;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.FastInnerCompactionWriter;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.AbstractCrossSpaceEstimator;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.AbstractInnerSpaceEstimator;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.FastCompactionInnerCompactionEstimator;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.FastCrossSpaceCompactionEstimator;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -61,6 +65,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -364,4 +369,20 @@ private void readModification(List<TsFileResource> resources) {
       modificationCache.put(resource.getTsFile().getName(), modifications);
     }
   }
+
+  public String getDatabaseName() {
+    return !seqFiles.isEmpty()
+        ? seqFiles.get(0).getDatabaseName()
+        : unseqFiles.get(0).getDatabaseName();
+  }
+
+  @Override
+  public Optional<AbstractInnerSpaceEstimator> getInnerSpaceEstimator() {
+    return Optional.of(new FastCompactionInnerCompactionEstimator());
+  }
+
+  @Override
+  public Optional<AbstractCrossSpaceEstimator> getCrossSpaceEstimator() {
+    return Optional.of(new FastCrossSpaceCompactionEstimator());
+  }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java

@@ -30,6 +30,8 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.readchunk.SingleSeriesCompactionExecutor;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.AbstractInnerSpaceEstimator;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.ReadChunkInnerCompactionEstimator;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
 
@@ -45,6 +47,7 @@
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Optional;
 
 public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer {
   private List<TsFileResource> seqFiles;
@@ -286,4 +289,9 @@ private void compactNotAlignedSeries(
   public void setSourceFiles(List<TsFileResource> seqFiles) {
     this.seqFiles = seqFiles;
   }
+
+  @Override
+  public Optional<AbstractInnerSpaceEstimator> getInnerSpaceEstimator() {
+    return Optional.of(new ReadChunkInnerCompactionEstimator());
+  }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java

@@ -38,6 +38,8 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.ReadPointCrossCompactionWriter;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.ReadPointInnerCompactionWriter;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.AbstractInnerSpaceEstimator;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.RepairUnsortedFileCompactionEstimator;
 import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
 import org.apache.iotdb.db.storageengine.dataregion.read.control.QueryResourceManager;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -61,6 +63,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
@@ -323,4 +326,9 @@ public void setSourceFiles(List<TsFileResource> seqFiles, List<TsFileResource> u
   public void setSourceFiles(List<TsFileResource> unseqFiles) {
     this.unseqFiles = unseqFiles;
   }
+
+  @Override
+  public Optional<AbstractInnerSpaceEstimator> getInnerSpaceEstimator() {
+    return Optional.of(new RepairUnsortedFileCompactionEstimator());
+  }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java

@@ -74,6 +74,7 @@ public abstract class AbstractCompactionTask {
   protected CompactionTaskSummary summary;
   protected long serialId;
   protected CompactionTaskStage taskStage;
+  protected long roughMemoryCost = -1L;
   protected long memoryCost = 0L;
 
   protected boolean recoverMemoryStatus;
@@ -256,6 +257,15 @@ public void transitSourceFilesToMerging() throws FileCannotTransitToCompactingEx
     }
   }
 
+  @TestOnly
+  public long getRoughMemoryCost() {
+    return roughMemoryCost;
+  }
+
+  public void setRoughMemoryCost(long memoryCost) {
+    this.roughMemoryCost = memoryCost;
+  }
+
   public abstract long getEstimatedMemoryCost();
 
   public abstract int getProcessedFileNum();

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java

@@ -28,8 +28,8 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionSourceFileDeletedException;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.IInnerCompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer;
@@ -39,8 +39,6 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.AbstractInnerSpaceEstimator;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.CompactionEstimateUtils;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.FastCompactionInnerCompactionEstimator;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.ReadChunkInnerCompactionEstimator;
 import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -674,20 +672,14 @@ protected void releaseAllLocks() {
   @Override
   public long getEstimatedMemoryCost() {
     if (innerSpaceEstimator == null) {
-      if (this.performer instanceof ReadChunkCompactionPerformer) {
-        innerSpaceEstimator = new ReadChunkInnerCompactionEstimator();
-      } else if (this.performer instanceof FastCompactionPerformer) {
-        innerSpaceEstimator = new FastCompactionInnerCompactionEstimator();
-      }
+      innerSpaceEstimator =
+          ((IInnerCompactionPerformer) this.performer).getInnerSpaceEstimator().orElse(null);
     }
     if (innerSpaceEstimator != null && memoryCost == 0L) {
       try {
-        long roughEstimatedMemoryCost =
-            innerSpaceEstimator.roughEstimateInnerCompactionMemory(
-                filesView.sourceFilesInCompactionPerformer);
         memoryCost =
-            CompactionEstimateUtils.shouldUseRoughEstimatedResult(roughEstimatedMemoryCost)
-                ? roughEstimatedMemoryCost
+            CompactionEstimateUtils.shouldUseRoughEstimatedResult(roughMemoryCost)
+                ? roughMemoryCost
                 : innerSpaceEstimator.estimateInnerCompactionMemory(
                     filesView.sourceFilesInCompactionPerformer);
       } catch (CompactionSourceFileDeletedException e) {