[To dev/1.3] Load: Harden LOAD TSFILE source path validation (#17624)

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java

@@ -61,8 +61,10 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -324,6 +326,14 @@ public class IoTDBConfig {
     tierDataDirs[0][0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME
   };
 
+  private String[] loadTsFileAllowedDirs = new String[0];
+
+  private CanonicalPaths loadTsFileDirCanonicalPaths = canonicalPaths(loadTsFileDirs);
+
+  private CanonicalPaths loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs);
+
+  private boolean loadTsFileSourcePathCheckEnabled = false;
+
   /** Strategy of multiple directories. */
   private String multiDirStrategyClassName = null;
 
@@ -1394,6 +1404,10 @@ private void formulateFolders() {
     for (int i = 0; i < loadActiveListeningDirs.length; i++) {
       loadActiveListeningDirs[i] = addDataHomeDir(loadActiveListeningDirs[i]);
     }
+    for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
+      loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
+    }
+    loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs);
     loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir);
     loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir);
     udfDir = addDataHomeDir(udfDir);
@@ -1589,6 +1603,36 @@ public String[] getLoadTsFileDirs() {
     return this.loadTsFileDirs;
   }
 
+  public String[] getLoadTsFileAllowedDirs() {
+    return this.loadTsFileAllowedDirs.length == 0
+        ? getLoadTsFileDirs()
+        : this.loadTsFileAllowedDirs;
+  }
+
+  public Path[] getLoadTsFileAllowedDirCanonicalPaths() throws FileNotFoundException {
+    return (this.loadTsFileAllowedDirs.length == 0
+            ? this.loadTsFileDirCanonicalPaths
+            : this.loadTsFileAllowedDirCanonicalPaths)
+        .getPaths();
+  }
+
+  public boolean isLoadTsFileSourcePathCheckEnabled() {
+    return loadTsFileSourcePathCheckEnabled;
+  }
+
+  public void setLoadTsFileSourcePathCheckEnabled(boolean loadTsFileSourcePathCheckEnabled) {
+    this.loadTsFileSourcePathCheckEnabled = loadTsFileSourcePathCheckEnabled;
+  }
+
+  public void setLoadTsFileAllowedDirs(String[] loadTsFileAllowedDirs) {
+    final String[] newLoadTsFileAllowedDirs = new String[loadTsFileAllowedDirs.length];
+    for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
+      newLoadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
+    }
+    this.loadTsFileAllowedDirs = newLoadTsFileAllowedDirs;
+    this.loadTsFileAllowedDirCanonicalPaths = canonicalPaths(newLoadTsFileAllowedDirs);
+  }
+
   public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
     if (tierDataDirs.length < 1) {
       logger.warn("No data directory is set. loadTsFileDirs is kept as the default value.");
@@ -1606,6 +1650,45 @@ public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
     // or the newLoadTsFileDirs will be used in the middle of the process
     // and cause the undefined behavior.
     this.loadTsFileDirs = newLoadTsFileDirs;
+    this.loadTsFileDirCanonicalPaths = canonicalPaths(newLoadTsFileDirs);
+  }
+
+  private static CanonicalPaths canonicalPaths(final String[] dirs) {
+    final Path[] paths = new Path[dirs.length];
+    for (int i = 0; i < dirs.length; i++) {
+      try {
+        paths[i] = new File(dirs[i]).getCanonicalFile().toPath();
+      } catch (final IOException e) {
+        return new CanonicalPaths(
+            String.format(
+                "Failed to resolve canonical path for Load TsFile allowed directory %s: %s",
+                dirs[i], e.getMessage()));
+      }
+    }
+    return new CanonicalPaths(paths);
+  }
+
+  private static class CanonicalPaths {
+
+    private final Path[] paths;
+    private final String errorMessage;
+
+    private CanonicalPaths(final Path[] paths) {
+      this.paths = paths;
+      this.errorMessage = null;
+    }
+
+    private CanonicalPaths(final String errorMessage) {
+      this.paths = new Path[0];
+      this.errorMessage = errorMessage;
+    }
+
+    private Path[] getPaths() throws FileNotFoundException {
+      if (errorMessage != null) {
+        throw new FileNotFoundException(errorMessage);
+      }
+      return paths;
+    }
   }
 
   public String getSchemaDir() {

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java

@@ -2449,6 +2449,17 @@
             properties.getProperty(
                 "load_write_throughput_bytes_per_second",
                 String.valueOf(conf.getLoadWriteThroughputBytesPerSecond()))));
+
+    conf.setLoadTsFileAllowedDirs(
+        Arrays.stream(properties.getProperty("load_tsfile_allowed_dirs", "").trim().split(","))
+            .filter(dir -> !dir.isEmpty())
+            .toArray(String[]::new));
+    conf.setLoadTsFileSourcePathCheckEnabled(
+        Boolean.parseBoolean(
+            properties.getProperty(
+                "load_tsfile_source_path_check_enable",
+                Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled()))));
+
     conf.setLoadTabletConversionThresholdBytes(
         Long.parseLong(
             properties.getProperty(
@@ -2560,6 +2571,25 @@
                 ConfigurationFileUtils.getConfigurationDefaultValue(
                     "load_write_throughput_bytes_per_second"))));
 
+    conf.setLoadTsFileAllowedDirs(
+        Arrays.stream(
+                properties
+                    .getProperty(
+                        "load_tsfile_allowed_dirs",
+                        Optional.ofNullable(
+                                ConfigurationFileUtils.getConfigurationDefaultValue(
+                                    "load_tsfile_allowed_dirs"))
+                            .orElse(""))
+                    .trim()
+                    .split(","))
+            .filter(dir -> !dir.isEmpty())
+            .toArray(String[]::new));
+    conf.setLoadTsFileSourcePathCheckEnabled(
+        Boolean.parseBoolean(
+            properties.getProperty(
+                "load_tsfile_source_path_check_enable",
+                Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled()))));
+
     conf.setLoadActiveListeningEnable(
         Boolean.parseBoolean(
             properties.getProperty(

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java

@@ -54,7 +54,7 @@ public TsFileLoader(File tsFile, String database) {
   @Override
   public void load() {
     try {
-      LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath());
+      LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
       statement.setDeleteAfterLoad(true);
       statement.setConvertOnTypeMismatch(true);
       statement.setDatabaseLevel(parseSgLevel());

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java

@@ -490,8 +490,7 @@ private TSStatus loadTsFileAsync(final List<String> absolutePaths) throws IOExce
   }
 
   private TSStatus loadTsFileSync(final String fileAbsolutePath) throws FileNotFoundException {
-    final LoadTsFileStatement statement = new LoadTsFileStatement(fileAbsolutePath);
-
+    final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath);
     statement.setDeleteAfterLoad(true);
     statement.setConvertOnTypeMismatch(true);
     statement.setVerifySchema(validateTsFile.get());

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java

@@ -372,7 +372,7 @@ private boolean handleSingleMiniFile(final int i) throws FileNotFoundException {
       final TSStatus status =
           loadTsFileDataTypeConverter
               .convertForTreeModel(
-                  new LoadTsFileStatement(tsFiles.get(i).getPath())
+                  LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
                       .setDeleteAfterLoad(isDeleteAfterLoad)
                       .setConvertOnTypeMismatch(isConvertOnTypeMismatch))
               .orElse(null);

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java

@@ -551,7 +551,7 @@ private void convertFailedTsFilesToTabletsAndRetry() {
         final TSStatus status =
             loadTsFileDataTypeConverter
                 .convertForTreeModel(
-                    new LoadTsFileStatement(filePath)
+                    LoadTsFileStatement.createUnchecked(filePath)
                         .setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
                         .setConvertOnTypeMismatch(true))
                 .orElse(null);

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java

@@ -21,6 +21,7 @@
 
 import org.apache.iotdb.common.rpc.thrift.TSStatus;
 import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.queryengine.plan.statement.Statement;
 import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
@@ -34,7 +35,9 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -56,6 +59,15 @@ public class LoadTsFileStatement extends Statement {
   private List<Long> writePointCountList;
 
   public LoadTsFileStatement(String filePath) throws FileNotFoundException {
+    this(filePath, true);
+  }
+
+  public static LoadTsFileStatement createUnchecked(String filePath) throws FileNotFoundException {
+    return new LoadTsFileStatement(filePath, false);
+  }
+
+  private LoadTsFileStatement(String filePath, boolean validateSourcePath)
+      throws FileNotFoundException {
     this.file = new File(filePath).getAbsoluteFile();
     this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel();
     this.verifySchema = true;
@@ -65,13 +77,22 @@ public LoadTsFileStatement(String filePath) throws FileNotFoundException {
         IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes();
     this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled();
 
-    this.tsFiles = processTsFile(file);
+    this.tsFiles = processTsFile(file, validateSourcePath);
     this.resources = new ArrayList<>();
     this.writePointCountList = new ArrayList<>();
     this.statementType = StatementType.MULTI_BATCH_INSERT;
   }
 
   public static List<File> processTsFile(final File file) throws FileNotFoundException {
+    return processTsFile(file, true);
+  }
+
+  public static List<File> processTsFile(final File file, final boolean validateSourcePath)
+      throws FileNotFoundException {
+    if (validateSourcePath) {
+      validateLoadSourcePath(file);
+    }
+
     final List<File> tsFiles = new ArrayList<>();
     if (file.isFile()) {
       tsFiles.add(file);
@@ -82,7 +103,7 @@ public static List<File> processTsFile(final File file) throws FileNotFoundExcep
                 "Can not find %s on this machine, notice that load can only handle files on this machine.",
                 file.getPath()));
       }
-      tsFiles.addAll(findAllTsFile(file));
+      tsFiles.addAll(findAllTsFile(file, validateSourcePath));
     }
     sortTsFiles(tsFiles);
     return tsFiles;
@@ -101,23 +122,64 @@ protected LoadTsFileStatement() {
     this.statementType = StatementType.MULTI_BATCH_INSERT;
   }
 
-  private static List<File> findAllTsFile(File file) {
+  private static List<File> findAllTsFile(File file, boolean validateSourcePath)
+      throws FileNotFoundException {
     final File[] files = file.listFiles();
     if (files == null) {
       return Collections.emptyList();
     }
 
     final List<File> tsFiles = new ArrayList<>();
     for (File nowFile : files) {
+      if (validateSourcePath) {
+        validateLoadSourcePath(nowFile);
+      }
       if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
         tsFiles.add(nowFile);
       } else if (nowFile.isDirectory()) {
-        tsFiles.addAll(findAllTsFile(nowFile));
+        tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath));
       }
     }
     return tsFiles;
   }
 
+  public static void validateLoadSourcePath(final String filePath) throws FileNotFoundException {
+    validateLoadSourcePath(new File(filePath));
+  }
+
+  private static void validateLoadSourcePath(final File file) throws FileNotFoundException {
+    final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+    if (!config.isLoadTsFileSourcePathCheckEnabled()) {
+      return;
+    }
+
+    final Path sourcePath = canonicalPath(file);
+    final String[] allowedDirs = config.getLoadTsFileAllowedDirs();
+    final Path[] allowedDirCanonicalPaths = config.getLoadTsFileAllowedDirCanonicalPaths();
+
+    for (final Path allowedDirCanonicalPath : allowedDirCanonicalPaths) {
+      if (sourcePath.startsWith(allowedDirCanonicalPath)) {
+        return;
+      }
+    }
+
+    throw new FileNotFoundException(
+        String.format(
+            "Load TsFile source path %s is outside allowed directories %s.",
+            sourcePath, Arrays.toString(allowedDirs)));
+  }
+
+  private static Path canonicalPath(final File file) throws FileNotFoundException {
+    try {
+      return file.getCanonicalFile().toPath();
+    } catch (final IOException e) {
+      throw new FileNotFoundException(
+          String.format(
+              "Failed to resolve canonical path for Load TsFile source %s: %s",
+              file.getPath(), e.getMessage()));
+    }
+  }
+
   private static void sortTsFiles(List<File> files) {
     files.sort(
         (o1, o2) -> {

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java

@@ -218,7 +218,8 @@ private TSStatus loadTsFile(
       final ActiveLoadPendingQueue.ActiveLoadEntry entry, final IClientSession session)
       throws FileNotFoundException {
     final File tsFile = new File(entry.getFile());
-    final LoadTsFileStatement statement = new LoadTsFileStatement(entry.getFile());
+    final LoadTsFileStatement statement =
+        LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
 
     statement.setDeleteAfterLoad(true);
     statement.setAutoCreateDatabase(