Skip to content

Commit 5ec7ee2

Browse files
CaideyipiCaideyipi
authored
[To dev/1.3] Load: Harden LOAD TSFILE source path validation (#17624) (#17654)
* Load: Harden LOAD TSFILE source path validation (#17624) * Load pri * sp * MAINTAIN * rollback * Add * change * canonical * line * Pre * Update LoadTsFileStatementTest.java * Update LoadTsFileStatementTest.java * Update IoTDBDescriptor.java
1 parent 96754ec commit 5ec7ee2

10 files changed

Lines changed: 307 additions & 10 deletions

File tree

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,10 @@
6161
import org.slf4j.LoggerFactory;
6262

6363
import java.io.File;
64+
import java.io.FileNotFoundException;
6465
import java.io.IOException;
6566
import java.lang.reflect.Field;
67+
import java.nio.file.Path;
6668
import java.util.ArrayList;
6769
import java.util.Arrays;
6870
import java.util.List;
@@ -324,6 +326,14 @@ public class IoTDBConfig {
324326
tierDataDirs[0][0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME
325327
};
326328

329+
private String[] loadTsFileAllowedDirs = new String[0];
330+
331+
private CanonicalPaths loadTsFileDirCanonicalPaths = canonicalPaths(loadTsFileDirs);
332+
333+
private CanonicalPaths loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs);
334+
335+
private boolean loadTsFileSourcePathCheckEnabled = false;
336+
327337
/** Strategy of multiple directories. */
328338
private String multiDirStrategyClassName = null;
329339

@@ -1394,6 +1404,10 @@ private void formulateFolders() {
13941404
for (int i = 0; i < loadActiveListeningDirs.length; i++) {
13951405
loadActiveListeningDirs[i] = addDataHomeDir(loadActiveListeningDirs[i]);
13961406
}
1407+
for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
1408+
loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
1409+
}
1410+
loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs);
13971411
loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir);
13981412
loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir);
13991413
udfDir = addDataHomeDir(udfDir);
@@ -1589,6 +1603,36 @@ public String[] getLoadTsFileDirs() {
15891603
return this.loadTsFileDirs;
15901604
}
15911605

1606+
public String[] getLoadTsFileAllowedDirs() {
1607+
return this.loadTsFileAllowedDirs.length == 0
1608+
? getLoadTsFileDirs()
1609+
: this.loadTsFileAllowedDirs;
1610+
}
1611+
1612+
public Path[] getLoadTsFileAllowedDirCanonicalPaths() throws FileNotFoundException {
1613+
return (this.loadTsFileAllowedDirs.length == 0
1614+
? this.loadTsFileDirCanonicalPaths
1615+
: this.loadTsFileAllowedDirCanonicalPaths)
1616+
.getPaths();
1617+
}
1618+
1619+
public boolean isLoadTsFileSourcePathCheckEnabled() {
1620+
return loadTsFileSourcePathCheckEnabled;
1621+
}
1622+
1623+
public void setLoadTsFileSourcePathCheckEnabled(boolean loadTsFileSourcePathCheckEnabled) {
1624+
this.loadTsFileSourcePathCheckEnabled = loadTsFileSourcePathCheckEnabled;
1625+
}
1626+
1627+
public void setLoadTsFileAllowedDirs(String[] loadTsFileAllowedDirs) {
1628+
final String[] newLoadTsFileAllowedDirs = new String[loadTsFileAllowedDirs.length];
1629+
for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
1630+
newLoadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
1631+
}
1632+
this.loadTsFileAllowedDirs = newLoadTsFileAllowedDirs;
1633+
this.loadTsFileAllowedDirCanonicalPaths = canonicalPaths(newLoadTsFileAllowedDirs);
1634+
}
1635+
15921636
public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
15931637
if (tierDataDirs.length < 1) {
15941638
logger.warn("No data directory is set. loadTsFileDirs is kept as the default value.");
@@ -1606,6 +1650,45 @@ public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
16061650
// or the newLoadTsFileDirs will be used in the middle of the process
16071651
// and cause the undefined behavior.
16081652
this.loadTsFileDirs = newLoadTsFileDirs;
1653+
this.loadTsFileDirCanonicalPaths = canonicalPaths(newLoadTsFileDirs);
1654+
}
1655+
1656+
private static CanonicalPaths canonicalPaths(final String[] dirs) {
1657+
final Path[] paths = new Path[dirs.length];
1658+
for (int i = 0; i < dirs.length; i++) {
1659+
try {
1660+
paths[i] = new File(dirs[i]).getCanonicalFile().toPath();
1661+
} catch (final IOException e) {
1662+
return new CanonicalPaths(
1663+
String.format(
1664+
"Failed to resolve canonical path for Load TsFile allowed directory %s: %s",
1665+
dirs[i], e.getMessage()));
1666+
}
1667+
}
1668+
return new CanonicalPaths(paths);
1669+
}
1670+
1671+
private static class CanonicalPaths {
1672+
1673+
private final Path[] paths;
1674+
private final String errorMessage;
1675+
1676+
private CanonicalPaths(final Path[] paths) {
1677+
this.paths = paths;
1678+
this.errorMessage = null;
1679+
}
1680+
1681+
private CanonicalPaths(final String errorMessage) {
1682+
this.paths = new Path[0];
1683+
this.errorMessage = errorMessage;
1684+
}
1685+
1686+
private Path[] getPaths() throws FileNotFoundException {
1687+
if (errorMessage != null) {
1688+
throw new FileNotFoundException(errorMessage);
1689+
}
1690+
return paths;
1691+
}
16091692
}
16101693

16111694
public String getSchemaDir() {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2449,6 +2449,17 @@ private void loadLoadTsFileProps(TrimProperties properties) throws IOException {
24492449
properties.getProperty(
24502450
"load_write_throughput_bytes_per_second",
24512451
String.valueOf(conf.getLoadWriteThroughputBytesPerSecond()))));
2452+
2453+
conf.setLoadTsFileAllowedDirs(
2454+
Arrays.stream(properties.getProperty("load_tsfile_allowed_dirs", "").trim().split(","))
2455+
.filter(dir -> !dir.isEmpty())
2456+
.toArray(String[]::new));
2457+
conf.setLoadTsFileSourcePathCheckEnabled(
2458+
Boolean.parseBoolean(
2459+
properties.getProperty(
2460+
"load_tsfile_source_path_check_enable",
2461+
Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled()))));
2462+
24522463
conf.setLoadTabletConversionThresholdBytes(
24532464
Long.parseLong(
24542465
properties.getProperty(
@@ -2560,6 +2571,25 @@ private void loadLoadTsFileHotModifiedProp(TrimProperties properties) throws IOE
25602571
ConfigurationFileUtils.getConfigurationDefaultValue(
25612572
"load_write_throughput_bytes_per_second"))));
25622573

2574+
conf.setLoadTsFileAllowedDirs(
2575+
Arrays.stream(
2576+
properties
2577+
.getProperty(
2578+
"load_tsfile_allowed_dirs",
2579+
Optional.ofNullable(
2580+
ConfigurationFileUtils.getConfigurationDefaultValue(
2581+
"load_tsfile_allowed_dirs"))
2582+
.orElse(""))
2583+
.trim()
2584+
.split(","))
2585+
.filter(dir -> !dir.isEmpty())
2586+
.toArray(String[]::new));
2587+
conf.setLoadTsFileSourcePathCheckEnabled(
2588+
Boolean.parseBoolean(
2589+
properties.getProperty(
2590+
"load_tsfile_source_path_check_enable",
2591+
Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled()))));
2592+
25632593
conf.setLoadActiveListeningEnable(
25642594
Boolean.parseBoolean(
25652595
properties.getProperty(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public TsFileLoader(File tsFile, String database) {
5454
@Override
5555
public void load() {
5656
try {
57-
LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath());
57+
LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
5858
statement.setDeleteAfterLoad(true);
5959
statement.setConvertOnTypeMismatch(true);
6060
statement.setDatabaseLevel(parseSgLevel());

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -490,8 +490,7 @@ private TSStatus loadTsFileAsync(final List<String> absolutePaths) throws IOExce
490490
}
491491

492492
private TSStatus loadTsFileSync(final String fileAbsolutePath) throws FileNotFoundException {
493-
final LoadTsFileStatement statement = new LoadTsFileStatement(fileAbsolutePath);
494-
493+
final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath);
495494
statement.setDeleteAfterLoad(true);
496495
statement.setConvertOnTypeMismatch(true);
497496
statement.setVerifySchema(validateTsFile.get());

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,7 @@ private boolean handleSingleMiniFile(final int i) throws FileNotFoundException {
372372
final TSStatus status =
373373
loadTsFileDataTypeConverter
374374
.convertForTreeModel(
375-
new LoadTsFileStatement(tsFiles.get(i).getPath())
375+
LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
376376
.setDeleteAfterLoad(isDeleteAfterLoad)
377377
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
378378
.orElse(null);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -551,7 +551,7 @@ private void convertFailedTsFilesToTabletsAndRetry() {
551551
final TSStatus status =
552552
loadTsFileDataTypeConverter
553553
.convertForTreeModel(
554-
new LoadTsFileStatement(filePath)
554+
LoadTsFileStatement.createUnchecked(filePath)
555555
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
556556
.setConvertOnTypeMismatch(true))
557557
.orElse(null);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.iotdb.common.rpc.thrift.TSStatus;
2323
import org.apache.iotdb.commons.path.PartialPath;
24+
import org.apache.iotdb.db.conf.IoTDBConfig;
2425
import org.apache.iotdb.db.conf.IoTDBDescriptor;
2526
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
2627
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
@@ -34,7 +35,9 @@
3435
import java.io.File;
3536
import java.io.FileNotFoundException;
3637
import java.io.IOException;
38+
import java.nio.file.Path;
3739
import java.util.ArrayList;
40+
import java.util.Arrays;
3841
import java.util.Collections;
3942
import java.util.List;
4043
import java.util.Map;
@@ -56,6 +59,15 @@ public class LoadTsFileStatement extends Statement {
5659
private List<Long> writePointCountList;
5760

5861
public LoadTsFileStatement(String filePath) throws FileNotFoundException {
62+
this(filePath, true);
63+
}
64+
65+
public static LoadTsFileStatement createUnchecked(String filePath) throws FileNotFoundException {
66+
return new LoadTsFileStatement(filePath, false);
67+
}
68+
69+
private LoadTsFileStatement(String filePath, boolean validateSourcePath)
70+
throws FileNotFoundException {
5971
this.file = new File(filePath).getAbsoluteFile();
6072
this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel();
6173
this.verifySchema = true;
@@ -65,13 +77,22 @@ public LoadTsFileStatement(String filePath) throws FileNotFoundException {
6577
IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes();
6678
this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled();
6779

68-
this.tsFiles = processTsFile(file);
80+
this.tsFiles = processTsFile(file, validateSourcePath);
6981
this.resources = new ArrayList<>();
7082
this.writePointCountList = new ArrayList<>();
7183
this.statementType = StatementType.MULTI_BATCH_INSERT;
7284
}
7385

7486
public static List<File> processTsFile(final File file) throws FileNotFoundException {
87+
return processTsFile(file, true);
88+
}
89+
90+
public static List<File> processTsFile(final File file, final boolean validateSourcePath)
91+
throws FileNotFoundException {
92+
if (validateSourcePath) {
93+
validateLoadSourcePath(file);
94+
}
95+
7596
final List<File> tsFiles = new ArrayList<>();
7697
if (file.isFile()) {
7798
tsFiles.add(file);
@@ -82,7 +103,7 @@ public static List<File> processTsFile(final File file) throws FileNotFoundExcep
82103
"Can not find %s on this machine, notice that load can only handle files on this machine.",
83104
file.getPath()));
84105
}
85-
tsFiles.addAll(findAllTsFile(file));
106+
tsFiles.addAll(findAllTsFile(file, validateSourcePath));
86107
}
87108
sortTsFiles(tsFiles);
88109
return tsFiles;
@@ -101,23 +122,64 @@ protected LoadTsFileStatement() {
101122
this.statementType = StatementType.MULTI_BATCH_INSERT;
102123
}
103124

104-
private static List<File> findAllTsFile(File file) {
125+
private static List<File> findAllTsFile(File file, boolean validateSourcePath)
126+
throws FileNotFoundException {
105127
final File[] files = file.listFiles();
106128
if (files == null) {
107129
return Collections.emptyList();
108130
}
109131

110132
final List<File> tsFiles = new ArrayList<>();
111133
for (File nowFile : files) {
134+
if (validateSourcePath) {
135+
validateLoadSourcePath(nowFile);
136+
}
112137
if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
113138
tsFiles.add(nowFile);
114139
} else if (nowFile.isDirectory()) {
115-
tsFiles.addAll(findAllTsFile(nowFile));
140+
tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath));
116141
}
117142
}
118143
return tsFiles;
119144
}
120145

146+
public static void validateLoadSourcePath(final String filePath) throws FileNotFoundException {
147+
validateLoadSourcePath(new File(filePath));
148+
}
149+
150+
private static void validateLoadSourcePath(final File file) throws FileNotFoundException {
151+
final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
152+
if (!config.isLoadTsFileSourcePathCheckEnabled()) {
153+
return;
154+
}
155+
156+
final Path sourcePath = canonicalPath(file);
157+
final String[] allowedDirs = config.getLoadTsFileAllowedDirs();
158+
final Path[] allowedDirCanonicalPaths = config.getLoadTsFileAllowedDirCanonicalPaths();
159+
160+
for (final Path allowedDirCanonicalPath : allowedDirCanonicalPaths) {
161+
if (sourcePath.startsWith(allowedDirCanonicalPath)) {
162+
return;
163+
}
164+
}
165+
166+
throw new FileNotFoundException(
167+
String.format(
168+
"Load TsFile source path %s is outside allowed directories %s.",
169+
sourcePath, Arrays.toString(allowedDirs)));
170+
}
171+
172+
private static Path canonicalPath(final File file) throws FileNotFoundException {
173+
try {
174+
return file.getCanonicalFile().toPath();
175+
} catch (final IOException e) {
176+
throw new FileNotFoundException(
177+
String.format(
178+
"Failed to resolve canonical path for Load TsFile source %s: %s",
179+
file.getPath(), e.getMessage()));
180+
}
181+
}
182+
121183
private static void sortTsFiles(List<File> files) {
122184
files.sort(
123185
(o1, o2) -> {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,8 @@ private TSStatus loadTsFile(
218218
final ActiveLoadPendingQueue.ActiveLoadEntry entry, final IClientSession session)
219219
throws FileNotFoundException {
220220
final File tsFile = new File(entry.getFile());
221-
final LoadTsFileStatement statement = new LoadTsFileStatement(entry.getFile());
221+
final LoadTsFileStatement statement =
222+
LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
222223

223224
statement.setDeleteAfterLoad(true);
224225
statement.setAutoCreateDatabase(

0 commit comments

Comments
 (0)