Skip to content

Commit 5029a0a

Browse files
CaideyipiCaideyipi
authored
Load: Harden LOAD TSFILE source path validation (#17624)
* Load pri * sp * MAINTAIN * rollback * Add * change * canonical * line * Pre
1 parent 50cefa2 commit 5029a0a

11 files changed

Lines changed: 275 additions & 16 deletions

File tree

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,10 @@
6060
import org.slf4j.LoggerFactory;
6161

6262
import java.io.File;
63+
import java.io.FileNotFoundException;
6364
import java.io.IOException;
6465
import java.lang.reflect.Field;
66+
import java.nio.file.Path;
6567
import java.util.ArrayList;
6668
import java.util.Arrays;
6769
import java.util.Collections;
@@ -298,6 +300,14 @@ public class IoTDBConfig {
298300
tierDataDirs[0][0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME
299301
};
300302

303+
private String[] loadTsFileAllowedDirs = new String[0];
304+
305+
private CanonicalPaths loadTsFileDirCanonicalPaths = canonicalPaths(loadTsFileDirs);
306+
307+
private CanonicalPaths loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs);
308+
309+
private boolean loadTsFileSourcePathCheckEnabled = false;
310+
301311
/** Strategy of multiple directories. */
302312
private String multiDirStrategyClassName = null;
303313

@@ -1355,6 +1365,10 @@ private void formulateFolders() {
13551365
for (int i = 0; i < loadActiveListeningDirs.length; i++) {
13561366
loadActiveListeningDirs[i] = addDataHomeDir(loadActiveListeningDirs[i]);
13571367
}
1368+
for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
1369+
loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
1370+
}
1371+
loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs);
13581372
loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir);
13591373
loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir);
13601374
udfDir = addDataHomeDir(udfDir);
@@ -1560,6 +1574,36 @@ public String[] getLoadTsFileDirs() {
15601574
return this.loadTsFileDirs;
15611575
}
15621576

1577+
public String[] getLoadTsFileAllowedDirs() {
1578+
return this.loadTsFileAllowedDirs.length == 0
1579+
? getLoadTsFileDirs()
1580+
: this.loadTsFileAllowedDirs;
1581+
}
1582+
1583+
public Path[] getLoadTsFileAllowedDirCanonicalPaths() throws FileNotFoundException {
1584+
return (this.loadTsFileAllowedDirs.length == 0
1585+
? this.loadTsFileDirCanonicalPaths
1586+
: this.loadTsFileAllowedDirCanonicalPaths)
1587+
.getPaths();
1588+
}
1589+
1590+
public boolean isLoadTsFileSourcePathCheckEnabled() {
1591+
return loadTsFileSourcePathCheckEnabled;
1592+
}
1593+
1594+
public void setLoadTsFileSourcePathCheckEnabled(boolean loadTsFileSourcePathCheckEnabled) {
1595+
this.loadTsFileSourcePathCheckEnabled = loadTsFileSourcePathCheckEnabled;
1596+
}
1597+
1598+
public void setLoadTsFileAllowedDirs(String[] loadTsFileAllowedDirs) {
1599+
final String[] newLoadTsFileAllowedDirs = new String[loadTsFileAllowedDirs.length];
1600+
for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
1601+
newLoadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
1602+
}
1603+
this.loadTsFileAllowedDirs = newLoadTsFileAllowedDirs;
1604+
this.loadTsFileAllowedDirCanonicalPaths = canonicalPaths(newLoadTsFileAllowedDirs);
1605+
}
1606+
15631607
public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
15641608
if (tierDataDirs.length < 1) {
15651609
logger.warn("No data directory is set. loadTsFileDirs is kept as the default value.");
@@ -1577,6 +1621,45 @@ public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
15771621
// or the newLoadTsFileDirs will be used in the middle of the process
15781622
// and cause the undefined behavior.
15791623
this.loadTsFileDirs = newLoadTsFileDirs;
1624+
this.loadTsFileDirCanonicalPaths = canonicalPaths(newLoadTsFileDirs);
1625+
}
1626+
1627+
private static CanonicalPaths canonicalPaths(final String[] dirs) {
1628+
final Path[] paths = new Path[dirs.length];
1629+
for (int i = 0; i < dirs.length; i++) {
1630+
try {
1631+
paths[i] = new File(dirs[i]).getCanonicalFile().toPath();
1632+
} catch (final IOException e) {
1633+
return new CanonicalPaths(
1634+
String.format(
1635+
"Failed to resolve canonical path for Load TsFile allowed directory %s: %s",
1636+
dirs[i], e.getMessage()));
1637+
}
1638+
}
1639+
return new CanonicalPaths(paths);
1640+
}
1641+
1642+
private static class CanonicalPaths {
1643+
1644+
private final Path[] paths;
1645+
private final String errorMessage;
1646+
1647+
private CanonicalPaths(final Path[] paths) {
1648+
this.paths = paths;
1649+
this.errorMessage = null;
1650+
}
1651+
1652+
private CanonicalPaths(final String errorMessage) {
1653+
this.paths = new Path[0];
1654+
this.errorMessage = errorMessage;
1655+
}
1656+
1657+
private Path[] getPaths() throws FileNotFoundException {
1658+
if (errorMessage != null) {
1659+
throw new FileNotFoundException(errorMessage);
1660+
}
1661+
return paths;
1662+
}
15801663
}
15811664

15821665
public String getSchemaDir() {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2456,6 +2456,16 @@ private void loadLoadTsFileProps(TrimProperties properties) {
24562456
"load_write_throughput_bytes_per_second",
24572457
String.valueOf(conf.getLoadWriteThroughputBytesPerSecond()))));
24582458

2459+
conf.setLoadTsFileAllowedDirs(
2460+
Arrays.stream(properties.getProperty("load_tsfile_allowed_dirs", "").trim().split(","))
2461+
.filter(dir -> !dir.isEmpty())
2462+
.toArray(String[]::new));
2463+
conf.setLoadTsFileSourcePathCheckEnabled(
2464+
Boolean.parseBoolean(
2465+
properties.getProperty(
2466+
"load_tsfile_source_path_check_enable",
2467+
Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled()))));
2468+
24592469
conf.setLoadTabletConversionThresholdBytes(
24602470
Long.parseLong(
24612471
properties.getProperty(
@@ -2573,6 +2583,23 @@ private void loadLoadTsFileHotModifiedProp(TrimProperties properties) throws IOE
25732583
ConfigurationFileUtils.getConfigurationDefaultValue(
25742584
"load_write_throughput_bytes_per_second"))));
25752585

2586+
conf.setLoadTsFileAllowedDirs(
2587+
Arrays.stream(
2588+
properties
2589+
.getProperty(
2590+
"load_tsfile_allowed_dirs",
2591+
ConfigurationFileUtils.getConfigurationDefaultValue(
2592+
"load_tsfile_allowed_dirs"))
2593+
.trim()
2594+
.split(","))
2595+
.filter(dir -> !dir.isEmpty())
2596+
.toArray(String[]::new));
2597+
conf.setLoadTsFileSourcePathCheckEnabled(
2598+
Boolean.parseBoolean(
2599+
properties.getProperty(
2600+
"load_tsfile_source_path_check_enable",
2601+
Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled()))));
2602+
25762603
conf.setLoadActiveListeningEnable(
25772604
Boolean.parseBoolean(
25782605
properties.getProperty(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public TsFileLoader(File tsFile, String database) {
5555
@Override
5656
public void load() {
5757
try {
58-
LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath());
58+
LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
5959
statement.setDeleteAfterLoad(true);
6060
statement.setConvertOnTypeMismatch(true);
6161
statement.setDatabaseLevel(parseSgLevel());

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -585,7 +585,7 @@ private TSStatus loadTsFileAsync(final String dataBaseName, final List<String> a
585585

586586
private TSStatus loadTsFileSync(final String dataBaseName, final String fileAbsolutePath)
587587
throws FileNotFoundException {
588-
final LoadTsFileStatement statement = new LoadTsFileStatement(fileAbsolutePath);
588+
final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath);
589589
statement.setDeleteAfterLoad(true);
590590
statement.setConvertOnTypeMismatch(true);
591591
statement.setVerifySchema(validateTsFile.get());

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -443,14 +443,15 @@ private boolean handleSingleMiniFile(final int i) throws FileNotFoundException {
443443
isTableModelTsFile.get(i)
444444
? loadTsFileDataTypeConverter
445445
.convertForTableModel(
446-
new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap())
446+
LoadTsFile.createUnchecked(
447+
null, tsFiles.get(i).getPath(), Collections.emptyMap())
447448
.setDatabase(databaseForTableData)
448449
.setDeleteAfterLoad(isDeleteAfterLoad)
449450
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
450451
.orElse(null)
451452
: loadTsFileDataTypeConverter
452453
.convertForTreeModel(
453-
new LoadTsFileStatement(tsFiles.get(i).getPath())
454+
LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
454455
.setDeleteAfterLoad(isDeleteAfterLoad)
455456
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
456457
.orElse(null);
@@ -712,14 +713,15 @@ private void executeTabletConversionOnException(
712713
isTableModelTsFile.get(i)
713714
? loadTsFileDataTypeConverter
714715
.convertForTableModel(
715-
new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap())
716+
LoadTsFile.createUnchecked(
717+
null, tsFiles.get(i).getPath(), Collections.emptyMap())
716718
.setDatabase(databaseForTableData)
717719
.setDeleteAfterLoad(isDeleteAfterLoad)
718720
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
719721
.orElse(null)
720722
: loadTsFileDataTypeConverter
721723
.convertForTreeModel(
722-
new LoadTsFileStatement(tsFiles.get(i).getPath())
724+
LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
723725
.setDeleteAfterLoad(isDeleteAfterLoad)
724726
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
725727
.orElse(null);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,19 @@ public class LoadTsFile extends Statement {
7272
private boolean needDecode4TimeColumn;
7373

7474
public LoadTsFile(NodeLocation location, String filePath, Map<String, String> loadAttributes) {
75+
this(location, filePath, loadAttributes, true);
76+
}
77+
78+
public static LoadTsFile createUnchecked(
79+
NodeLocation location, String filePath, Map<String, String> loadAttributes) {
80+
return new LoadTsFile(location, filePath, loadAttributes, false);
81+
}
82+
83+
private LoadTsFile(
84+
NodeLocation location,
85+
String filePath,
86+
Map<String, String> loadAttributes,
87+
boolean validateSourcePath) {
7588
super(location);
7689
this.filePath = requireNonNull(filePath, "filePath is null");
7790

@@ -89,7 +102,7 @@ public LoadTsFile(NodeLocation location, String filePath, Map<String, String> lo
89102
try {
90103
this.tsFiles =
91104
org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.processTsFile(
92-
new File(filePath));
105+
new File(filePath), validateSourcePath);
93106
this.resources = new ArrayList<>();
94107
this.writePointCountList = new ArrayList<>();
95108
this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), true));
@@ -283,7 +296,7 @@ public List<LoadTsFile> getSubStatements() {
283296
final Map<String, String> properties = this.loadAttributes;
284297

285298
final LoadTsFile subStatement =
286-
new LoadTsFile(getLocation().orElse(null), filePath, properties);
299+
LoadTsFile.createUnchecked(getLocation().orElse(null), filePath, properties);
287300

288301
// Copy all configuration properties
289302
subStatement.databaseLevel = this.databaseLevel;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -586,14 +586,14 @@ private void convertFailedTsFilesToTabletsAndRetry() {
586586
failedNode.isTableModel()
587587
? loadTsFileDataTypeConverter
588588
.convertForTableModel(
589-
new LoadTsFile(null, filePath, Collections.emptyMap())
589+
LoadTsFile.createUnchecked(null, filePath, Collections.emptyMap())
590590
.setDatabase(failedNode.getDatabase())
591591
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
592592
.setConvertOnTypeMismatch(true))
593593
.orElse(null)
594594
: loadTsFileDataTypeConverter
595595
.convertForTreeModel(
596-
new LoadTsFileStatement(filePath)
596+
LoadTsFileStatement.createUnchecked(filePath)
597597
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
598598
.setConvertOnTypeMismatch(true))
599599
.orElse(null);

0 commit comments

Comments
 (0)