Skip to content

Commit e09bdbe

Browse files
LJW21-02LJW21-02
authored
Modified db parameter in data import (#16294)
* Modified implementation of exporting schema to sql file using show create table * Added double quotation marks to tableName dropped * Used constant sql + escapeSqlIdentifier in sql sentence * Changed default timeout + Allowed mfs configuration * Tracked progress of table data export * Tracked progress of both tree and table data export * Modified rowRecord to iterator for csv and sql export * Adjust updateTimeInerval to 2s * Used constant process sentence + fixed copilot review * Changed db to optional in ImportData * Fix value null bug * Added comments in import main func * Fix null bug in ExportData using iterator.isNull
1 parent 71bb8cc commit e09bdbe

7 files changed

Lines changed: 71 additions & 74 deletions

File tree

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public class Constants {
9898
public static final String DB_ARGS = "db";
9999
public static final String DB_NAME = "database";
100100
public static final String DB_DESC =
101-
"The database to be exported,only takes effect and required when sql_dialect is table .(optional)";
101+
"The database to be exported,only takes effect when sql_dialect is table and required when file_type is csv and tsfile.(optional)";
102102

103103
public static final String TABLE_ARGS = "table";
104104
public static final String TABLE_DESC =
@@ -388,4 +388,8 @@ public class Constants {
388388
"Lines per failed file,only takes effect and required when sql_dialect is table .(option)";
389389
public static final String IMPORT_COMPLETELY = "Import completely!";
390390
public static final int BATCH_POINT_SIZE = 10000;
391+
392+
public static final String IMPORT_INIT_MEET_ERROR_MSG = "Meet error when init import because ";
393+
public static final String REQUIRED_ARGS_ERROR_MSG =
394+
"Invalid args: Required values for option '%s' not provided";
391395
}

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/OptionsUtil.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -130,13 +130,7 @@ public static Options createTableImportCommonOptions() {
130130
Options options = createImportCommonOptions();
131131

132132
Option opDatabase =
133-
Option.builder(DB_ARGS)
134-
.longOpt(DB_NAME)
135-
.argName(DB_ARGS)
136-
.hasArg()
137-
.required()
138-
.desc(DB_DESC)
139-
.build();
133+
Option.builder(DB_ARGS).longOpt(DB_NAME).argName(DB_ARGS).hasArg().desc(DB_DESC).build();
140134
options.addOption(opDatabase);
141135

142136
return options;

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -207,9 +207,7 @@ public static void main(String[] args) {
207207
}
208208
} else {
209209
ioTPrinter.println(
210-
String.format(
211-
"Invalid args: Required values for option '%s' not provided",
212-
Constants.FILE_TYPE_NAME));
210+
String.format(Constants.REQUIRED_ARGS_ERROR_MSG, Constants.FILE_TYPE_NAME));
213211
System.exit(Constants.CODE_ERROR);
214212
}
215213
int exitCode = Constants.CODE_OK;

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -270,10 +270,10 @@ private void exportToCsvFile(SessionDataSet sessionDataSet, String filePath)
270270
if (curType.equalsIgnoreCase("TIMESTAMP")) {
271271
csvPrinterWrapper.print(timeTrans(iterator.getLong(curColumnIndex + 1)));
272272
} else {
273-
String columnValue = iterator.getString(curColumnIndex + 1);
274-
if (StringUtils.isEmpty(columnValue)) {
273+
if (iterator.isNull(curColumnIndex + 1)) {
275274
csvPrinterWrapper.print("");
276275
} else {
276+
String columnValue = iterator.getString(curColumnIndex + 1);
277277
if (curType.equalsIgnoreCase("TEXT") || curType.equalsIgnoreCase("STRING")) {
278278
csvPrinterWrapper.print("\"" + columnValue + "\"");
279279
} else {

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTree.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.apache.iotdb.tool.common.Constants;
2929

3030
import org.apache.commons.collections4.CollectionUtils;
31-
import org.apache.commons.lang3.StringUtils;
3231
import org.apache.thrift.TException;
3332
import org.apache.tsfile.enums.TSDataType;
3433
import org.apache.tsfile.exception.write.WriteProcessException;
@@ -201,11 +200,11 @@ private void exportToSqlFile(SessionDataSet sessionDataSet, String filePath)
201200
"SHOW TIMESERIES " + timeseries.get(index - startIndex), timeout);
202201
SessionDataSet.DataIterator iterator2 = sessionDataSet2.iterator();
203202
if (iterator2.next()) {
204-
String value = iterator.getString(index + 1);
205-
if (StringUtils.isEmpty(value)) {
203+
if (iterator.isNull(index + 1)) {
206204
headersTemp.remove(seriesList.get(index - startIndex));
207205
continue;
208206
}
207+
String value = iterator.getString(index + 1);
209208
if ("TEXT".equalsIgnoreCase(iterator2.getString(4))) {
210209
values.add("\"" + value + "\"");
211210
} else {
@@ -298,10 +297,10 @@ private void exportToCsvFile(SessionDataSet sessionDataSet, String filePath)
298297
fromOuterloop = false;
299298
csvPrinterWrapper.print(timeTrans(iterator.getLong(1)));
300299
for (int curColumnIndex = 1; curColumnIndex < totalColumns; curColumnIndex++) {
301-
String columnValue = iterator.getString(curColumnIndex + 1);
302-
if (StringUtils.isEmpty(columnValue)) {
300+
if (iterator.isNull(curColumnIndex + 1)) {
303301
csvPrinterWrapper.print("");
304302
} else {
303+
String columnValue = iterator.getString(curColumnIndex + 1);
305304
String curType = columnTypeList.get(curColumnIndex);
306305
if ((curType.equalsIgnoreCase("TEXT") || curType.equalsIgnoreCase("STRING"))
307306
&& !columnValue.startsWith("root.")) {

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,17 +99,17 @@ public static void main(String[] args) throws IoTDBConnectionException {
9999
System.exit(Constants.CODE_ERROR);
100100
}
101101
final List<String> argList = Arrays.asList(args);
102-
int helpIndex = argList.indexOf(Constants.MINUS + Constants.HELP_ARGS);
103102
int sql_dialect = argList.indexOf(Constants.MINUS + Constants.SQL_DIALECT_ARGS); // -sql_dialect
104-
if (sql_dialect >= 0
105-
&& !Constants.SQL_DIALECT_VALUE_TREE.equalsIgnoreCase(argList.get(sql_dialect + 1))) {
103+
if (sql_dialect >= 0) {
104+
// sql_dialect specified (default: tree)
106105
final String sqlDialectValue = argList.get(sql_dialect + 1);
107106
if (Constants.SQL_DIALECT_VALUE_TABLE.equalsIgnoreCase(sqlDialectValue)) {
108107
sqlDialectTree = false;
109108
tsFileOptions = OptionsUtil.createTableImportTsFileOptions();
110109
csvOptions = OptionsUtil.createTableImportCsvOptions();
111110
sqlOptions = OptionsUtil.createTableImportSqlOptions();
112-
} else {
111+
} else if (!Constants.SQL_DIALECT_VALUE_TREE.equalsIgnoreCase(sqlDialectValue)) {
112+
// sql_dialect neither tree nor table
113113
ioTPrinter.println(String.format("sql_dialect %s is not support", sqlDialectValue));
114114
printHelpOptions(
115115
Constants.IMPORT_CLI_HEAD,
@@ -126,9 +126,11 @@ public static void main(String[] args) throws IoTDBConnectionException {
126126
if (ftIndex < 0) {
127127
ftIndex = argList.indexOf(Constants.MINUS + Constants.FILE_TYPE_NAME); // -file_type
128128
}
129+
int helpIndex = argList.indexOf(Constants.MINUS + Constants.HELP_ARGS);
129130
if (helpIndex >= 0) {
130131
fileType = argList.get(helpIndex + 1);
131132
if (StringUtils.isNotBlank(fileType)) {
133+
// print help info according to file type
132134
if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) {
133135
printHelpOptions(null, Constants.IMPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false);
134136
} else if (Constants.CSV_SUFFIXS.equalsIgnoreCase(fileType)) {
@@ -147,6 +149,7 @@ public static void main(String[] args) throws IoTDBConnectionException {
147149
true);
148150
}
149151
} else {
152+
// print help info for all file types
150153
printHelpOptions(
151154
Constants.IMPORT_CLI_HEAD,
152155
Constants.IMPORT_CLI_PREFIX,
@@ -160,6 +163,7 @@ public static void main(String[] args) throws IoTDBConnectionException {
160163
} else if (ftIndex >= 0) {
161164
fileType = argList.get(ftIndex + 1);
162165
if (StringUtils.isNotBlank(fileType)) {
166+
// parse command line according to file type
163167
if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) {
164168
try {
165169
commandLine = parser.parse(tsFileOptions, args);
@@ -199,16 +203,12 @@ public static void main(String[] args) throws IoTDBConnectionException {
199203
}
200204
} else {
201205
ioTPrinter.println(
202-
String.format(
203-
"Invalid args: Required values for option '%s' not provided",
204-
Constants.FILE_TYPE_NAME));
206+
String.format(Constants.REQUIRED_ARGS_ERROR_MSG, Constants.FILE_TYPE_NAME));
205207
System.exit(Constants.CODE_ERROR);
206208
}
207209
} else {
208210
ioTPrinter.println(
209-
String.format(
210-
"Invalid args: Required values for option '%s' not provided",
211-
Constants.FILE_TYPE_NAME));
211+
String.format(Constants.REQUIRED_ARGS_ERROR_MSG, Constants.FILE_TYPE_NAME));
212212
System.exit(Constants.CODE_ERROR);
213213
}
214214

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTable.java‎

Lines changed: 48 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -83,55 +83,57 @@ public void init() throws InterruptedException {
8383
System.exit(Constants.CODE_ERROR);
8484
}
8585
// checkDataBase
86-
SessionDataSet sessionDataSet = null;
87-
try (ITableSession session = sessionPool.getSession()) {
88-
List<String> databases = new ArrayList<>();
89-
sessionDataSet = session.executeQueryStatement("show databases");
90-
while (sessionDataSet.hasNext()) {
91-
RowRecord rowRecord = sessionDataSet.next();
92-
databases.add(rowRecord.getField(0).getStringValue());
93-
}
94-
if (!databases.contains(database)) {
95-
ioTPrinter.println(String.format(Constants.TARGET_DATABASE_NOT_EXIST_MSG, database));
96-
System.exit(1);
97-
}
98-
if (Constants.CSV_SUFFIXS.equals(fileType)) {
99-
if (StringUtils.isNotBlank(table)) {
100-
sessionDataSet = session.executeQueryStatement("show tables");
101-
List<String> tables = new ArrayList<>();
102-
while (sessionDataSet.hasNext()) {
103-
RowRecord rowRecord = sessionDataSet.next();
104-
tables.add(rowRecord.getField(0).getStringValue());
105-
}
106-
if (!tables.contains(table)) {
107-
ioTPrinter.println(String.format(Constants.TARGET_TABLE_NOT_EXIST_MSG, table));
108-
System.exit(1);
109-
}
110-
sessionDataSet = session.executeQueryStatement("describe " + table);
111-
while (sessionDataSet.hasNext()) {
112-
RowRecord rowRecord = sessionDataSet.next();
113-
final String columnName = rowRecord.getField(0).getStringValue();
114-
final String category = rowRecord.getField(2).getStringValue();
115-
if (!timeColumn.equalsIgnoreCase(category)) {
116-
dataTypes.put(columnName, getType(rowRecord.getField(1).getStringValue()));
117-
columnCategory.put(columnName, getColumnCategory(category));
86+
if (!Constants.SQL_SUFFIXS.equals(fileType)) {
87+
SessionDataSet sessionDataSet = null;
88+
try (ITableSession session = sessionPool.getSession()) {
89+
List<String> databases = new ArrayList<>();
90+
sessionDataSet = session.executeQueryStatement("show databases");
91+
while (sessionDataSet.hasNext()) {
92+
RowRecord rowRecord = sessionDataSet.next();
93+
databases.add(rowRecord.getField(0).getStringValue());
94+
}
95+
if (!databases.contains(database)) {
96+
ioTPrinter.println(String.format(Constants.TARGET_DATABASE_NOT_EXIST_MSG, database));
97+
System.exit(1);
98+
}
99+
if (Constants.CSV_SUFFIXS.equals(fileType)) {
100+
if (StringUtils.isNotBlank(table)) {
101+
sessionDataSet = session.executeQueryStatement("show tables");
102+
List<String> tables = new ArrayList<>();
103+
while (sessionDataSet.hasNext()) {
104+
RowRecord rowRecord = sessionDataSet.next();
105+
tables.add(rowRecord.getField(0).getStringValue());
106+
}
107+
if (!tables.contains(table)) {
108+
ioTPrinter.println(String.format(Constants.TARGET_TABLE_NOT_EXIST_MSG, table));
109+
System.exit(1);
110+
}
111+
sessionDataSet = session.executeQueryStatement("describe " + table);
112+
while (sessionDataSet.hasNext()) {
113+
RowRecord rowRecord = sessionDataSet.next();
114+
final String columnName = rowRecord.getField(0).getStringValue();
115+
final String category = rowRecord.getField(2).getStringValue();
116+
if (!timeColumn.equalsIgnoreCase(category)) {
117+
dataTypes.put(columnName, getType(rowRecord.getField(1).getStringValue()));
118+
columnCategory.put(columnName, getColumnCategory(category));
119+
}
118120
}
121+
} else {
122+
ioTPrinter.println(String.format(Constants.TARGET_TABLE_NOT_EXIST_MSG, null));
123+
System.exit(1);
119124
}
120-
} else {
121-
ioTPrinter.println(String.format(Constants.TARGET_TABLE_NOT_EXIST_MSG, null));
122-
System.exit(1);
123125
}
124-
}
125-
} catch (StatementExecutionException e) {
126-
ioTPrinter.println(Constants.INSERT_CSV_MEET_ERROR_MSG + e.getMessage());
127-
System.exit(1);
128-
} catch (IoTDBConnectionException e) {
129-
throw new RuntimeException(e);
130-
} finally {
131-
if (ObjectUtils.isNotEmpty(sessionDataSet)) {
132-
try {
133-
sessionDataSet.close();
134-
} catch (Exception e) {
126+
} catch (StatementExecutionException e) {
127+
ioTPrinter.println(Constants.IMPORT_INIT_MEET_ERROR_MSG + e.getMessage());
128+
System.exit(1);
129+
} catch (IoTDBConnectionException e) {
130+
throw new RuntimeException(e);
131+
} finally {
132+
if (ObjectUtils.isNotEmpty(sessionDataSet)) {
133+
try {
134+
sessionDataSet.close();
135+
} catch (Exception e) {
136+
}
135137
}
136138
}
137139
}

0 commit comments

Comments
 (0)