Skip to content

Commit 7ce2cd8

Browse files
chernserchernser
authored
Merge pull request #2254 from ClickHouse/perf_jdbc_improvements
[Perf] fixed by index perf. added using ImmutableMap
2 parents 470aaa1 + 42dc545 commit 7ce2cd8

12 files changed

Lines changed: 158 additions & 109 deletions

File tree

‎client-v2/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,12 @@
8282
<scope>compile</scope>
8383
</dependency>
8484

85+
<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
86+
<dependency>
87+
<groupId>com.google.guava</groupId>
88+
<artifactId>guava</artifactId>
89+
<version>${guava.version}</version>
90+
</dependency>
8591

8692
<!-- Test Dependencies -->
8793
<dependency>

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,12 @@ private boolean readBlock() throws IOException {
7474
List<String> names = new ArrayList<>(nColumns);
7575
List<String> types = new ArrayList<>(nColumns);
7676
currentBlock = new Block(names, types, nRows);
77-
TableSchema schema = new TableSchema();
77+
List<ClickHouseColumn> columns = new ArrayList<>(nColumns);
7878
for (int i = 0; i < nColumns; i++) {
7979

80-
schema.addColumn(BinaryStreamReader.readString(input),
80+
ClickHouseColumn column = ClickHouseColumn.of(BinaryStreamReader.readString(input),
8181
BinaryStreamReader.readString(input));
82-
ClickHouseColumn column = schema.getColumns().get(i);
82+
columns.add(column);
8383

8484
names.add(column.getColumnName());
8585
types.add(column.getDataType().name());
@@ -101,6 +101,7 @@ private boolean readBlock() throws IOException {
101101
}
102102
currentBlock.add(values);
103103
}
104+
TableSchema schema = new TableSchema(columns);
104105

105106
setSchema(schema);
106107

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryWithNamesAndTypesFormatReader.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
66
import com.clickhouse.client.api.metadata.TableSchema;
77
import com.clickhouse.client.api.query.QuerySettings;
8+
import com.clickhouse.data.ClickHouseColumn;
89

910
import java.io.EOFException;
1011
import java.io.IOException;
@@ -24,8 +25,7 @@ public RowBinaryWithNamesAndTypesFormatReader(InputStream inputStream, QuerySett
2425

2526
private void readSchema() {
2627
try {
27-
TableSchema headerSchema = new TableSchema();
28-
List<String> columns = new ArrayList<>();
28+
List<String> names = new ArrayList<>();
2929
int nCol;
3030
try {
3131
nCol = BinaryStreamReader.readVarInt(input);
@@ -34,14 +34,16 @@ private void readSchema() {
3434
return;
3535
}
3636
for (int i = 0; i < nCol; i++) {
37-
columns.add(binaryStreamReader.readString());
37+
names.add(binaryStreamReader.readString());
3838
}
3939

40+
List<ClickHouseColumn> columns = new ArrayList<>(nCol);
4041
for (int i = 0; i < nCol; i++) {
41-
headerSchema.addColumn(columns.get(i), binaryStreamReader.readString());
42+
columns.add(ClickHouseColumn.of(names.get(i), binaryStreamReader.readString()));
4243
}
4344

44-
setSchema(headerSchema);
45+
TableSchema schema = new TableSchema(columns);
46+
setSchema(schema);
4547
} catch (IOException e) {
4648
throw new ClientException("Failed to read header", e);
4749
}

‎client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,38 @@
11
package com.clickhouse.client.api.internal;
22

33
import com.clickhouse.client.api.metadata.TableSchema;
4+
import com.clickhouse.data.ClickHouseColumn;
45

56
import java.io.BufferedReader;
67
import java.io.IOException;
78
import java.io.InputStream;
89
import java.io.InputStreamReader;
910
import java.io.StringReader;
11+
import java.util.ArrayList;
12+
import java.util.List;
1013
import java.util.Properties;
1114

1215
public class TableSchemaParser {
1316

1417
public static TableSchema readTSKV(InputStream content, String table, String sqlQuery, String database) {
15-
TableSchema schema = new TableSchema();
16-
schema.setTableName(table);
17-
schema.setQuery(sqlQuery);
18-
schema.setDatabaseName(database);
1918
Properties p = new Properties();
2019
try (BufferedReader r = new BufferedReader(new InputStreamReader(content))) {
2120
String line;
21+
List<ClickHouseColumn> columns = new ArrayList<>();
2222
while ((line = r.readLine()) != null) {
2323
p.clear();
24-
int lineLength = line.length();
2524
if (!line.trim().isEmpty()) {
2625
p.load(new StringReader(line.replaceAll("\t", "\n")));
27-
schema.addColumn(p.getProperty("name"), p.getProperty("type"), p.getProperty("default_type"));
26+
ClickHouseColumn column = ClickHouseColumn.of(p.getProperty("name"), p.getProperty("type"));
27+
String defaultType = p.getProperty("default_type");
28+
column.setHasDefault(defaultType != null && !defaultType.isEmpty());
29+
columns.add(column);
2830
}
2931
}
32+
33+
return new TableSchema(table, sqlQuery, database, columns);
3034
} catch (IOException e) {
3135
throw new RuntimeException("Failed to parse table schema", e);
3236
}
33-
return schema;
3437
}
3538
}

‎client-v2/src/main/java/com/clickhouse/client/api/metadata/TableSchema.java‎

Lines changed: 35 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,47 @@
11
package com.clickhouse.client.api.metadata;
22

3-
import com.clickhouse.client.api.ClientException;
43
import com.clickhouse.data.ClickHouseColumn;
4+
import com.google.common.collect.ImmutableList;
5+
import com.google.common.collect.ImmutableMap;
56

67
import java.util.ArrayList;
8+
import java.util.Collection;
79
import java.util.Collections;
810
import java.util.HashMap;
911
import java.util.List;
1012
import java.util.Map;
1113

1214
public class TableSchema {
1315

14-
private String tableName = "";
16+
private final String tableName;
1517

16-
private String query = "";
18+
private final String query;
1719

18-
private String databaseName = "";
20+
private final String databaseName;
1921

20-
private List<ClickHouseColumn> columns;
22+
private final List<ClickHouseColumn> columns;
2123

22-
private List<ClickHouseColumn> columnsView;
23-
24-
private Map<String, Map<String, Object>> metadata;
25-
26-
private Map<String, Integer> colIndex;
24+
private final Map<String, Integer> colIndex;
2725

2826
private boolean hasDefaults = false;
2927

30-
public TableSchema() {
31-
this.metadata = new HashMap<>();
32-
this.columns = new ArrayList<>();
33-
this.columnsView = Collections.unmodifiableList(this.columns);
34-
this.colIndex = new HashMap<>();
28+
public TableSchema(Collection<ClickHouseColumn> columns) {
29+
this("", "", "", columns);
30+
}
31+
public TableSchema(String tableName, String query, String databaseName, Collection<ClickHouseColumn> columns) {
32+
this.tableName = tableName;
33+
this.databaseName = databaseName;
34+
this.query = query;
35+
this.columns = ImmutableList.copyOf(columns);
36+
ImmutableMap.Builder<String, Integer> colIndexMapBuilder = ImmutableMap.builder();
37+
for (int i = 0; i < this.columns.size(); i++) {
38+
ClickHouseColumn column= this.columns.get(i);
39+
if (column.hasDefault()) {
40+
this.hasDefaults = true;
41+
}
42+
colIndexMapBuilder.put(this.columns.get(i).getColumnName(), i);
43+
}
44+
this.colIndex = colIndexMapBuilder.build();
3545
}
3646

3747
/**
@@ -40,7 +50,7 @@ public TableSchema() {
4050
* @return - collection of columns in the table
4151
*/
4252
public List<ClickHouseColumn> getColumns() {
43-
return columnsView;
53+
return columns;
4454
}
4555

4656
public String getDatabaseName() {
@@ -51,14 +61,6 @@ public String getTableName() {
5161
return tableName;
5262
}
5363

54-
public void setTableName(String tableName) {
55-
this.tableName = tableName;
56-
}
57-
58-
public void setDatabaseName(String databaseName) {
59-
this.databaseName = databaseName;
60-
}
61-
6264
public boolean hasDefaults() {
6365
return hasDefaults;
6466
}
@@ -67,30 +69,19 @@ public String getQuery() {
6769
return query;
6870
}
6971

70-
public void setQuery(String query) {
71-
this.query = query;
72-
}
73-
74-
public void addColumn(String name, String type) {
75-
addColumn(name, type, "");
76-
}
77-
public void addColumn(String name, String type, String defaultType) {
78-
ClickHouseColumn column = ClickHouseColumn.of(name, type);
79-
if (defaultType.toUpperCase().contains("DEFAULT")) {
80-
hasDefaults = true;
81-
column.setHasDefault(true);
82-
}
83-
columns.add(column);
84-
85-
Map<String, Object> columnMetadata = metadata.computeIfAbsent(name, k -> new HashMap<>());
86-
columnMetadata.put("type", type);
87-
colIndex.put(name, columns.size() - 1);
88-
}
89-
9072
public ClickHouseColumn getColumnByName(String name) {
9173
return columns.get(nameToIndex(name));
9274
}
9375

76+
/**
77+
* Returns column by index. Index starts with 1.
78+
* @param colIndex - column index;
79+
* @return
80+
*/
81+
public ClickHouseColumn getColumnByIndex(int colIndex) {
82+
return columns.get(colIndex - 1);
83+
}
84+
9485
/**
9586
* Takes absolute index (starting from 0) and returns corresponding column.
9687
*
@@ -134,7 +125,6 @@ public String toString() {
134125
"tableName='" + tableName + '\'' +
135126
", databaseName='" + databaseName + '\'' +
136127
", columns=" + columns +
137-
", metadata=" + metadata +
138128
", colIndex=" + colIndex +
139129
", hasDefaults=" + hasDefaults +
140130
'}';

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.clickhouse.client.api.query.QueryResponse;
2828
import com.clickhouse.client.api.query.QuerySettings;
2929
import com.clickhouse.client.api.query.Records;
30+
import com.clickhouse.data.ClickHouseColumn;
3031
import com.clickhouse.data.ClickHouseDataType;
3132
import com.clickhouse.data.ClickHouseFormat;
3233
import com.clickhouse.data.ClickHouseVersion;
@@ -440,10 +441,13 @@ public void testBinaryStreamReader() throws Exception {
440441
Future<QueryResponse> response = client.query("SELECT col1, col3, hostname() as host FROM " + table, settings);
441442
QueryResponse queryResponse = response.get();
442443

443-
TableSchema schema = new TableSchema();
444-
schema.addColumn("col1", "UInt32");
445-
schema.addColumn("col3", "String");
446-
schema.addColumn("host", "String");
444+
445+
TableSchema schema = new TableSchema(
446+
Arrays.asList(
447+
ClickHouseColumn.of("col1", "UInt32"),
448+
ClickHouseColumn.of("col3", "String"),
449+
ClickHouseColumn.of("host", "String")
450+
));
447451
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse, schema);
448452
int rowsCount = 0;
449453
while (reader.next() != null) {
@@ -472,10 +476,6 @@ public void testRowStreamReader() throws Exception {
472476
Future<QueryResponse> response = client.query("SELECT col1, col3, hostname() as host FROM " + table, settings);
473477

474478
QueryResponse queryResponse = response.get();
475-
TableSchema schema = new TableSchema();
476-
schema.addColumn("col1", "UInt32");
477-
schema.addColumn("col3", "String");
478-
schema.addColumn("host", "String");
479479
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse);
480480

481481
Map<String, Object> record;

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/ResultSetImpl.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import com.clickhouse.client.api.metadata.TableSchema;
1818
import com.clickhouse.client.api.query.QueryResponse;
1919
import com.clickhouse.data.ClickHouseColumn;
20+
import com.clickhouse.data.ClickHouseDataType;
2021
import com.clickhouse.jdbc.internal.ExceptionUtils;
2122
import com.clickhouse.jdbc.internal.JdbcUtils;
2223
import com.clickhouse.jdbc.types.Array;
@@ -431,7 +432,7 @@ public ResultSetMetaData getMetaData() throws SQLException {
431432

432433
@Override
433434
public Object getObject(int columnIndex) throws SQLException {
434-
return getObject(columnIndexToName(columnIndex));
435+
return getObject(columnIndex, JdbcUtils.convertToJavaClass(getSchema().getColumnByIndex(columnIndex).getDataType()));
435436
}
436437

437438
@Override
@@ -959,7 +960,8 @@ public Statement getStatement() throws SQLException {
959960

960961
@Override
961962
public Object getObject(int columnIndex, Map<String, Class<?>> map) throws SQLException {
962-
return getObject(columnIndexToName(columnIndex), map);
963+
ClickHouseDataType type = getSchema().getColumnByIndex(columnIndex).getDataType();
964+
return getObject(columnIndex, map.get(JdbcUtils.convertToSqlType(type).getName()));
963965
}
964966

965967
@Override
@@ -1493,7 +1495,24 @@ public void updateNClob(String columnLabel, Reader reader) throws SQLException {
14931495

14941496
@Override
14951497
public <T> T getObject(int columnIndex, Class<T> type) throws SQLException {
1496-
return getObject(columnIndexToName(columnIndex), type);
1498+
checkClosed();
1499+
try {
1500+
if (reader.hasValue(columnIndex)) {
1501+
wasNull = false;
1502+
if (type == null) {//As a fallback, try to get the value as is
1503+
return reader.readValue(columnIndex);
1504+
}
1505+
1506+
return (T) JdbcUtils.convert(reader.readValue(columnIndex), type);
1507+
} else {
1508+
wasNull = true;
1509+
return null;
1510+
}
1511+
} catch (Exception e) {
1512+
throw ExceptionUtils.toSqlState(String.format("Method: getObject(\"%s\", %s) encountered an exception.",
1513+
reader.getSchema().columnIndexToName(columnIndex), type),
1514+
String.format("SQL: [%s]", parentStatement.getLastSql()), e);
1515+
}
14971516
}
14981517

14991518
@Override

‎performance/src/test/com/clickhouse/benchmark/BenchmarkRunner.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,23 +39,23 @@ public static void main(String[] args) throws Exception {
3939
Map<String, String> argMap = parseArguments(args);
4040

4141
Options opt = new OptionsBuilder()
42-
.include(QueryClient.class.getName())
43-
.include(InsertClient.class.getName())
44-
.include(ConcurrentInsertClient.class.getName())
45-
.include(ConcurrentQueryClient.class.getName())
46-
.include(Compression.class.getName())
47-
.include(Serializers.class.getName())
48-
.include(Deserializers.class.getName())
49-
.include(MixedWorkload.class.getName())
50-
.include(DataTypes.class.getName())
42+
// .include(QueryClient.class.getName())
43+
// .include(InsertClient.class.getName())
44+
// .include(ConcurrentInsertClient.class.getName())
45+
// .include(ConcurrentQueryClient.class.getName())
46+
// .include(Compression.class.getName())
47+
// .include(Serializers.class.getName())
48+
// .include(Deserializers.class.getName())
49+
// .include(MixedWorkload.class.getName())
50+
// .include(DataTypes.class.getName())
5151
.include(JDBCQuery.class.getName())
5252
.include(JDBCInsert.class.getName())
5353
.forks(1) // must be a fork. No fork only for debugging
5454
.mode(Mode.SampleTime)
5555
.timeUnit(TimeUnit.MILLISECONDS)
5656
.addProfiler(GCProfiler.class)
5757
.addProfiler(MemPoolProfiler.class)
58-
.warmupIterations(3)
58+
.warmupIterations(1)
5959
.warmupTime(TimeValue.seconds(5))
6060
.measurementIterations(10)
6161
.jvmArgs("-Xms8g", "-Xmx8g")

0 commit comments

Comments
 (0)