Skip to content

Commit 832783f

Browse files
mzitnikmzitnik
authored
Add only Serializer benchmark for V1 and V2 (#2219)
* Add only Serializer benchmark for V1 and V2 * Ordering data so benchmark v2 will be comparable to v1 (data is ordered and pull from map) * Another change to match v1 ser * Small bug fix to nameToIndex * Fix serialization also in Insert test to match v1
1 parent 97610f7 commit 832783f

6 files changed

Lines changed: 122 additions & 7 deletions

File tree

‎client-v2/src/main/java/com/clickhouse/client/api/metadata/TableSchema.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ public int nameToIndex(String name) {
125125
if (index == null) {
126126
throw new NoSuchColumnException("Result has no column with name '" + name + "'");
127127
}
128-
return colIndex.get(name).intValue();
128+
return index;
129129
}
130130

131131
@Override

‎performance/src/test/com/clickhouse/benchmark/clients/Components.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,26 @@
11
package com.clickhouse.benchmark.clients;
22

33
import com.clickhouse.benchmark.data.DataSet;
4+
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
5+
import com.clickhouse.client.api.insert.InsertResponse;
46
import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream;
7+
import com.clickhouse.data.ClickHouseColumn;
8+
import com.clickhouse.data.ClickHouseDataProcessor;
9+
import com.clickhouse.data.ClickHouseFormat;
510
import com.clickhouse.data.ClickHouseOutputStream;
11+
import com.clickhouse.data.ClickHousePassThruStream;
12+
import com.clickhouse.data.ClickHouseRecord;
13+
import com.clickhouse.data.ClickHouseSerializer;
614
import com.clickhouse.data.stream.Lz4OutputStream;
715
import net.jpountz.lz4.LZ4Factory;
816
import org.openjdk.jmh.annotations.Benchmark;
917
import org.slf4j.Logger;
1018
import org.slf4j.LoggerFactory;
1119

1220
import java.io.ByteArrayOutputStream;
21+
import java.io.OutputStream;
22+
import java.util.List;
23+
import java.util.Map;
1324

1425
public class Components extends BenchmarkBase {
1526
private static final Logger LOGGER = LoggerFactory.getLogger(Components.class);
@@ -42,4 +53,70 @@ public void CompressingOutputStreamV2(DataState dataState) {
4253
LOGGER.error("Error: ", e);
4354
}
4455
}
56+
57+
private OutputStream createEmptyOutputStream() {
58+
return new OutputStream() {
59+
private long count = 0;
60+
61+
@Override
62+
public void write(int b) {
63+
count++;
64+
}
65+
66+
@Override
67+
public void write(byte[] b) {
68+
count += b.length;
69+
}
70+
71+
@Override
72+
public void write(byte[] b, int off, int len) {
73+
count += len;
74+
}
75+
76+
@Override
77+
public void flush() {
78+
79+
}
80+
81+
@Override
82+
public void close() {
83+
}
84+
};
85+
}
86+
@Benchmark
87+
public void SerializerOutputStreamV1(DataState dataState) {
88+
OutputStream empty = createEmptyOutputStream();
89+
try {
90+
ClickHouseOutputStream chos = ClickHouseOutputStream.of(empty);
91+
ClickHouseDataProcessor p = dataState.dataSet.getClickHouseDataProcessor();
92+
ClickHouseSerializer[] serializers = p.getSerializers(getClientV1().getConfig(), p.getColumns());
93+
for (ClickHouseRecord record : dataState.dataSet.getClickHouseRecords()) {
94+
for (int i = 0; i < serializers.length; i++) {
95+
serializers[i].serialize(record.getValue(i), chos);
96+
}
97+
}
98+
chos.flush();
99+
} catch (Exception e) {
100+
LOGGER.error("Error: ", e);
101+
}
102+
}
103+
104+
@Benchmark
105+
public void SerializerOutputStreamV2(DataState dataState) {
106+
OutputStream empty = createEmptyOutputStream();
107+
try {
108+
RowBinaryFormatWriter w = new RowBinaryFormatWriter(empty, dataState.dataSet.getSchema(), ClickHouseFormat.RowBinary);
109+
for (List<Object> row : dataState.dataSet.getRowsOrdered()) {
110+
int index = 1;
111+
for (Object value : row) {
112+
w.setValue(index, value);
113+
index++;
114+
}
115+
w.commitRow();
116+
}
117+
empty.flush();
118+
} catch (Exception e) {
119+
LOGGER.error("Error: ", e);
120+
}
121+
}
45122
}

‎performance/src/test/com/clickhouse/benchmark/clients/InsertClient.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,19 +162,22 @@ public void insertV2RowBinary(DataState dataState) {
162162
try (InsertResponse response = clientV2.insert(dataState.tableNameEmpty, out -> {
163163
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, dataState.dataSet.getSchema(), ClickHouseFormat.RowBinary);
164164
List<ClickHouseColumn> columns = dataState.dataSet.getSchema().getColumns();
165-
for (Map<String, Object> row : dataState.dataSet.getRows()) {
166-
for (ClickHouseColumn column : columns) {
167-
w.setValue(column.getColumnName(),row.get(column.getColumnName()));
165+
for (List<Object> row : dataState.dataSet.getRowsOrdered()) {
166+
int index = 1;
167+
for (Object value : row) {
168+
w.setValue(index, value);
169+
index++;
168170
}
169171
w.commitRow();
170172
}
171173
out.flush();
172174

173-
}, ClickHouseFormat.RowBinaryWithDefaults, new InsertSettings()).get()) {
175+
}, ClickHouseFormat.RowBinaryWithDefaults, new InsertSettings().compressClientRequest(true)).get()) {
174176
response.getWrittenRows();
175177
}
176178
} catch (Exception e) {
177179
LOGGER.error("Error: ", e);
178180
}
179181
}
182+
180183
}

‎performance/src/test/com/clickhouse/benchmark/data/DataSet.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ default InputStream getInputStream(int rowId, ClickHouseFormat format) {
4444

4545
List<ClickHouseRecord> getClickHouseRecords();
4646

47+
List<Map<String, Object>> getRowsLimit(int numRows);
48+
49+
List<ClickHouseRecord> getClickHouseRecordsLimit(int numRows);
50+
List<List<Object>> getRowsOrdered();
51+
4752
void setClickHouseRecords(List<ClickHouseRecord> records);
4853

4954
void setClickHouseDataProcessor(ClickHouseDataProcessor dataProcessor);

‎performance/src/test/com/clickhouse/benchmark/data/FileDataSet.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public class FileDataSet implements DataSet{
3131
private List<byte[]> lines =null;
3232

3333
private List<Map<String, Object>> data;
34+
private List<List<Object>> dataOrdered;
3435

3536
public FileDataSet(String filePath, int limit) {
3637
LOGGER.info("Reading file {}", filePath);
@@ -117,7 +118,18 @@ public List<byte[]> getBytesList(ClickHouseFormat format) {
117118
public List<Map<String, Object>> getRows() {
118119
return data;
119120
}
120-
121+
@Override
122+
public List<Map<String, Object>> getRowsLimit(int numRows) {
123+
return data.subList(0, numRows);
124+
}
125+
@Override
126+
public List<ClickHouseRecord> getClickHouseRecordsLimit(int numRows) {
127+
return clickHouseRecords.subList(0, numRows);
128+
}
129+
@Override
130+
public List<List<Object>> getRowsOrdered() {
131+
return dataOrdered;
132+
}
121133
@Override
122134
public ClickHouseFormat getFormat() {
123135
return ClickHouseFormat.CSV;
@@ -134,17 +146,24 @@ public List<ClickHouseRecord> getClickHouseRecords() {
134146
public void setClickHouseRecords(List<ClickHouseRecord> records) {
135147
this.clickHouseRecords = records;
136148
List<ClickHouseColumn> columns = schema.getColumns();
149+
dataOrdered = new ArrayList<>(records.size());
137150
data = new ArrayList<>(records.size());
138151
for (ClickHouseRecord record : records) {
139152
Iterator<ClickHouseValue> vIter = record.iterator();
140153
int i = 0;
141154
Map<String, Object> row = new HashMap<>();
155+
List<Object> rowOrdered = new ArrayList<>(columns.size());
142156
while (vIter.hasNext()) {
143157
ClickHouseValue v = vIter.next();
144158
row.put(columns.get(i++).getColumnName(), v.asObject());
159+
160+
rowOrdered.add(v.asObject());
145161
}
146162
data.add(row);
163+
dataOrdered.add(rowOrdered);
164+
147165
}
166+
148167
}
149168

150169
private ClickHouseDataProcessor dataProcessor;

‎performance/src/test/com/clickhouse/benchmark/data/SimpleDataSet.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,18 @@ public List<Map<String, Object>> getRows() {
150150
public List<ClickHouseRecord> getClickHouseRecords() {
151151
return clickHouseRecords;
152152
}
153-
153+
@Override
154+
public List<Map<String, Object>> getRowsLimit(int numRows) {
155+
return data.subList(0, numRows);
156+
}
157+
@Override
158+
public List<List<Object>> getRowsOrdered() {
159+
return null;
160+
}
161+
@Override
162+
public List<ClickHouseRecord> getClickHouseRecordsLimit(int numRows) {
163+
return clickHouseRecords.subList(0, numRows);
164+
}
154165
@Override
155166
public void setClickHouseRecords(List<ClickHouseRecord> records) {
156167
clickHouseRecords = records;

0 commit comments

Comments
 (0)