Skip to content

Commit 631e736

Browse files
author
Paultagoras
committed
Some initial tests for validating the results
1 parent 335caaf commit 631e736

3 files changed

Lines changed: 290 additions & 2 deletions

File tree

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ public class RowBinaryFormatWriter {
2929

3030
private final Object[] row;
3131

32+
private final boolean defaultSupport;
33+
3234
public RowBinaryFormatWriter(OutputStream out, TableSchema tableSchema, ClickHouseFormat format) {
3335
if (format != ClickHouseFormat.RowBinary && format != ClickHouseFormat.RowBinaryWithDefaults) {
3436
throw new IllegalArgumentException("Only RowBinary and RowBinaryWithDefaults are supported");
@@ -37,6 +39,7 @@ public RowBinaryFormatWriter(OutputStream out, TableSchema tableSchema, ClickHou
3739
this.out = out;
3840
this.tableSchema = tableSchema;
3941
this.row = new Object[tableSchema.getColumns().size()];
42+
this.defaultSupport = format == ClickHouseFormat.RowBinaryWithDefaults;
4043
}
4144

4245
public void setValue(String column, Object value) {
@@ -48,12 +51,11 @@ public void setValue(int colIndex, Object value) {
4851
}
4952

5053
public void commitRow() throws IOException {
51-
5254
List<ClickHouseColumn> columnList = tableSchema.getColumns();
5355
for (int i = 0; i < row.length; i++) {
5456
ClickHouseColumn column = columnList.get(i);
5557

56-
if (RowBinaryFormatSerializer.writeValuePreamble(out, true, column, row[i])) {
58+
if (RowBinaryFormatSerializer.writeValuePreamble(out, defaultSupport, column, row[i])) {
5759
SerializerUtils.serializeData(out, row[i], column);
5860
}
5961
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -741,6 +741,8 @@ public static BigDecimal convertToBigDecimal(Object value) {
741741
return (BigDecimal) value;
742742
} else if (value instanceof BigInteger) {
743743
return new BigDecimal((BigInteger) value);
744+
} else if (value instanceof BigDecimal) {
745+
return (BigDecimal) value;
744746
} else if (value instanceof Number) {
745747
return BigDecimal.valueOf(((Number) value).doubleValue());
746748
} else if (value instanceof String) {

‎client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
package com.clickhouse.client.datatypes;
2+
3+
import com.clickhouse.client.BaseIntegrationTest;
4+
import com.clickhouse.client.ClickHouseNode;
5+
import com.clickhouse.client.ClickHouseProtocol;
6+
import com.clickhouse.client.ClickHouseServerForTest;
7+
import com.clickhouse.client.api.Client;
8+
import com.clickhouse.client.api.command.CommandSettings;
9+
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
10+
import com.clickhouse.client.api.enums.Protocol;
11+
import com.clickhouse.client.api.insert.InsertResponse;
12+
import com.clickhouse.client.api.insert.InsertSettings;
13+
import com.clickhouse.client.api.internal.ServerSettings;
14+
import com.clickhouse.client.api.metadata.TableSchema;
15+
import com.clickhouse.client.api.query.GenericRecord;
16+
import com.clickhouse.data.ClickHouseFormat;
17+
import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
18+
import org.testng.annotations.BeforeMethod;
19+
import org.testng.annotations.Test;
20+
21+
import java.io.IOException;
22+
import java.math.BigDecimal;
23+
import java.math.BigInteger;
24+
import java.util.List;
25+
import java.util.Map;
26+
import java.util.Random;
27+
import java.util.UUID;
28+
import java.util.concurrent.TimeUnit;
29+
30+
import static org.testng.Assert.assertEquals;
31+
import static org.testng.Assert.assertNull;
32+
import static org.testng.Assert.assertTrue;
33+
34+
public class RowBinaryFormatWriterTest extends BaseIntegrationTest {
35+
private Client client;
36+
private InsertSettings settings;
37+
private static final int EXECUTE_CMD_TIMEOUT = 30;
38+
39+
@BeforeMethod(groups = { "integration" })
40+
public void setUp() throws IOException {
41+
int bufferSize = (7 * 65500);
42+
client = newClient()
43+
.setSocketSndbuf(bufferSize)
44+
.setSocketRcvbuf(bufferSize)
45+
.setClientNetworkBufferSize(bufferSize)
46+
.build();
47+
48+
settings = new InsertSettings()
49+
.setDeduplicationToken(RandomStringUtils.randomAlphabetic(36))
50+
.setQueryId(String.valueOf(UUID.randomUUID()));
51+
}
52+
53+
protected Client.Builder newClient() {
54+
ClickHouseNode node = getServer(ClickHouseProtocol.HTTP);
55+
boolean isSecure = isCloud();
56+
return new Client.Builder()
57+
.addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure)
58+
.setUsername("default")
59+
.setPassword(ClickHouseServerForTest.getPassword())
60+
.compressClientRequest(true)
61+
.useHttpCompression(true)
62+
.setDefaultDatabase(ClickHouseServerForTest.getDatabase())
63+
.serverSetting(ServerSettings.ASYNC_INSERT, "0")
64+
.serverSetting(ServerSettings.WAIT_END_OF_QUERY, "1");
65+
}
66+
67+
protected void initTable(String tableName, String createTableSQL, CommandSettings settings) throws Exception {
68+
if (settings == null) {
69+
settings = new CommandSettings();
70+
}
71+
72+
client.execute("DROP TABLE IF EXISTS " + tableName, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS);
73+
client.execute(createTableSQL, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS);
74+
}
75+
76+
private static void assertEqualsKinda(Object actual, Object expected) {
77+
assertEquals(String.valueOf(actual), String.valueOf(expected));
78+
}
79+
80+
81+
82+
@Test (groups = { "integration" })
83+
public void writeNumbersTest() throws Exception {
84+
String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_');
85+
String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
86+
" (id Int32, " +
87+
" int8 Int8, int8_nullable Nullable(Int8), int8_default Int8 DEFAULT 3, " +
88+
" int16 Int16, int16_nullable Nullable(Int16), int16_default Int16 DEFAULT 3, " +
89+
" int32 Int32, int32_nullable Nullable(Int32), int32_default Int32 DEFAULT 3, " +
90+
" int64 Int64, int64_nullable Nullable(Int64), int64_default Int64 DEFAULT 3, " +
91+
" int128 Int128, int128_nullable Nullable(Int128), int128_default Int128 DEFAULT 3, " +
92+
" int256 Int256, int256_nullable Nullable(Int256), int256_default Int256 DEFAULT 3, " +
93+
" uint8 UInt8, uint8_nullable Nullable(UInt8), uint8_default UInt8 DEFAULT 3, " +
94+
" uint16 UInt16, uint16_nullable Nullable(UInt16), uint16_default UInt16 DEFAULT 3, " +
95+
" uint32 UInt32, uint32_nullable Nullable(UInt32), uint32_default UInt32 DEFAULT 3, " +
96+
" uint64 UInt64, uint64_nullable Nullable(UInt64), uint64_default UInt64 DEFAULT 3, " +
97+
" uint128 UInt128, uint128_nullable Nullable(UInt128), uint128_default UInt128 DEFAULT 3, " +
98+
" uint256 UInt256, uint256_nullable Nullable(UInt256), uint256_default UInt256 DEFAULT 3, " +
99+
" float32 Float32, float32_nullable Nullable(Float32), float32_default Float32 DEFAULT 3, " +
100+
" float64 Float64, float64_nullable Nullable(Float64), float64_default Float64 DEFAULT 3, " +
101+
" decimal Decimal(4, 2), decimal_nullable Nullable(Decimal(4, 2)), decimal_default Decimal(4, 2) DEFAULT 3, " +
102+
" decimal32 Decimal(8, 4), decimal32_nullable Nullable(Decimal(8, 4)), decimal32_default Decimal(8, 4) DEFAULT 3, " +
103+
" decimal64 Decimal(18, 6), decimal64_nullable Nullable(Decimal(18, 6)), decimal64_default Decimal(18, 6) DEFAULT 3, " +
104+
" decimal128 Decimal(36, 8), decimal128_nullable Nullable(Decimal(36, 8)), decimal128_default Decimal(36, 8) DEFAULT 3, " +
105+
" decimal256 Decimal(74, 10), decimal256_nullable Nullable(Decimal(74, 10)), decimal256_default Decimal(74, 10) DEFAULT 3" +
106+
" ) Engine = MergeTree ORDER BY id";
107+
initTable(tableName, tableCreate, new CommandSettings());
108+
109+
// Insert random (valid) values
110+
long seed = System.currentTimeMillis();
111+
Random rand = new Random(seed);
112+
System.out.println("Random seed: " + seed);
113+
114+
Object[][] rows = new Object[][] {
115+
{1,
116+
rand.nextInt(256) - 128, null, null, //Int8
117+
rand.nextInt(65536) - 32768, null, null, //Int16
118+
rand.nextInt(), null, null, //Int32
119+
rand.nextLong(), null, null, //Int64
120+
new BigInteger(127, rand), null, null, //Int128
121+
new BigInteger(255, rand), null, null, //Int256
122+
rand.nextInt(256), null, null, //UInt8
123+
rand.nextInt(65536), null, null, //UInt16
124+
rand.nextInt() & 0xFFFFFFFFL, null, null, //UInt32
125+
BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE)), null, null, //UInt64
126+
new BigInteger(128, rand), null, null, //UInt128
127+
new BigInteger(256, rand), null, null, //UInt256
128+
rand.nextFloat(), null, null, //Float32
129+
rand.nextDouble(), null, null, //Float64
130+
new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(10,100)), null, null, //Decimal(4)
131+
new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(1000, 10000)), null, null, //Decimal32
132+
new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)), null, null, //Decimal64
133+
new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000)), null, null, //Decimal128
134+
new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L)), null, null //Decimal256
135+
},
136+
{2,
137+
rand.nextInt(256) - 128, null, null, //Int8
138+
rand.nextInt(65536) - 32768, null, null, //Int16
139+
rand.nextInt(), null, null, //Int32
140+
rand.nextLong(), null, null, //Int64
141+
new BigInteger(127, rand), null, null, //Int128
142+
new BigInteger(255, rand), null, null, //Int256
143+
rand.nextInt(256), null, null, //UInt8
144+
rand.nextInt(65536), null, null, //UInt16
145+
rand.nextInt() & 0xFFFFFFFFL, null, null, //UInt32
146+
BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE)), null, null, //UInt64
147+
new BigInteger(128, rand), null, null, //UInt128
148+
new BigInteger(256, rand), null, null, //UInt256
149+
rand.nextFloat(), null, null, //Float32
150+
rand.nextDouble(), null, null, //Float64
151+
new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(10,100)), null, null, //Decimal(4)
152+
new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(1000, 10000)), null, null, //Decimal32
153+
new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)), null, null, //Decimal64
154+
new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000)), null, null, //Decimal128
155+
new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L)), null, null //Decimal256
156+
},
157+
};
158+
159+
TableSchema schema = client.getTableSchema(tableName);
160+
161+
ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults;
162+
try (InsertResponse response = client.insert(tableName, out -> {
163+
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format);
164+
for (Object[] row : rows) {
165+
for (int i = 0; i < row.length; i++) {
166+
w.setValue(i + 1, row[i]);
167+
}
168+
w.commitRow();
169+
}
170+
}, format, settings).get()) {
171+
System.out.println("Rows written: " + response.getWrittenRows());
172+
}
173+
174+
List<GenericRecord> records = client.queryAll("SELECT * FROM \"" + tableName + "\"" );
175+
176+
int id = 1;
177+
for (GenericRecord record : records) {
178+
Map<String, Object> r = record.getValues();
179+
assertEquals(r.get("id"), id);
180+
assertEqualsKinda(r.get("int8"), String.valueOf(rows[id - 1][1]));
181+
assertEqualsKinda(r.get("int8_nullable"), rows[id - 1][2]);
182+
assertEqualsKinda(r.get("int8_default"), 3);
183+
assertEqualsKinda(r.get("int16"), rows[id - 1][4]);
184+
assertEqualsKinda(r.get("int16_nullable"), rows[id - 1][5]);
185+
assertEqualsKinda(r.get("int16_default"), 3);
186+
assertEqualsKinda(r.get("int32"), rows[id - 1][7]);
187+
assertEqualsKinda(r.get("int32_nullable"), rows[id - 1][8]);
188+
assertEqualsKinda(r.get("int32_default"), 3);
189+
assertEqualsKinda(r.get("int64"), rows[id - 1][10]);
190+
assertEqualsKinda(r.get("int64_nullable"), rows[id - 1][11]);
191+
assertEqualsKinda(r.get("int64_default"), 3);
192+
assertEqualsKinda(r.get("int128"), rows[id - 1][13]);
193+
assertEqualsKinda(r.get("int128_nullable"), rows[id - 1][14]);
194+
assertEqualsKinda(r.get("int128_default"), 3);
195+
assertEqualsKinda(r.get("int256"), rows[id - 1][16]);
196+
assertEqualsKinda(r.get("int256_nullable"), rows[id - 1][17]);
197+
assertEqualsKinda(r.get("int256_default"), 3);
198+
assertEqualsKinda(r.get("uint8"), rows[id - 1][19]);
199+
assertEqualsKinda(r.get("uint8_nullable"), rows[id - 1][20]);
200+
assertEqualsKinda(r.get("uint8_default"), 3);
201+
assertEqualsKinda(r.get("uint16"), rows[id - 1][22]);
202+
assertEqualsKinda(r.get("uint16_nullable"), rows[id - 1][23]);
203+
assertEqualsKinda(r.get("uint16_default"), 3);
204+
assertEqualsKinda(r.get("uint32"), rows[id - 1][25]);
205+
assertEqualsKinda(r.get("uint32_nullable"), rows[id - 1][26]);
206+
assertEqualsKinda(r.get("uint32_default"), 3);
207+
assertEqualsKinda(r.get("uint64"), rows[id - 1][28]);
208+
assertEqualsKinda(r.get("uint64_nullable"), rows[id - 1][29]);
209+
assertEqualsKinda(r.get("uint64_default"), 3);
210+
assertEqualsKinda(r.get("uint128"), rows[id - 1][31]);
211+
assertEqualsKinda(r.get("uint128_nullable"), rows[id - 1][32]);
212+
assertEqualsKinda(r.get("uint128_default"), 3);
213+
assertEqualsKinda(r.get("uint256"), rows[id - 1][34]);
214+
assertEqualsKinda(r.get("uint256_nullable"), rows[id - 1][35]);
215+
assertEqualsKinda(r.get("uint256_default"), 3);
216+
assertEqualsKinda(r.get("float32"), rows[id - 1][37]);
217+
assertEqualsKinda(r.get("float32_nullable"), rows[id - 1][38]);
218+
assertEqualsKinda(r.get("float32_default"), 3.0);
219+
assertEqualsKinda(r.get("float64"), rows[id - 1][40]);
220+
assertEqualsKinda(r.get("float64_nullable"), rows[id - 1][41]);
221+
assertEqualsKinda(r.get("float64_default"), 3.0);
222+
assertEqualsKinda(r.get("decimal"), rows[id - 1][43]);
223+
assertEqualsKinda(r.get("decimal_nullable"), rows[id - 1][44]);
224+
assertEqualsKinda(r.get("decimal_default"), "3.00");
225+
assertEqualsKinda(r.get("decimal32"), rows[id - 1][46]);
226+
assertEqualsKinda(r.get("decimal32_nullable"), rows[id - 1][47]);
227+
assertEqualsKinda(r.get("decimal32_default"), "3.0000");
228+
assertEqualsKinda(r.get("decimal64"), rows[id - 1][49]);
229+
assertEqualsKinda(r.get("decimal64_nullable"), rows[id - 1][50]);
230+
assertEqualsKinda(r.get("decimal64_default"), "3.000000");
231+
assertEqualsKinda(r.get("decimal128"), rows[id - 1][52]);
232+
assertEqualsKinda(r.get("decimal128_nullable"), rows[id - 1][53]);
233+
assertEqualsKinda(r.get("decimal128_default"), "3.00000000");
234+
assertEqualsKinda(r.get("decimal256"), rows[id - 1][55]);
235+
assertEqualsKinda(r.get("decimal256_nullable"), rows[id - 1][56]);
236+
assertEqualsKinda(r.get("decimal256_default"), "3.0000000000");
237+
id++;
238+
}
239+
}
240+
241+
// @Test
242+
// public void testAdvancedWriter() throws Exception {
243+
// String tableName = "very_long_table_name_with_uuid_" + UUID.randomUUID().toString().replace('-', '_');
244+
// String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
245+
// " (name String, " +
246+
// " v1 Float32, " +
247+
// " v2 Float32, " +
248+
// " attrs Nullable(String), " +
249+
// " corrected_time DateTime('UTC') DEFAULT now()," +
250+
// " special_attr Nullable(Int8) DEFAULT -1)" +
251+
// " Engine = MergeTree ORDER by ()";
252+
//
253+
// initTable(tableName, tableCreate);
254+
//
255+
// ZonedDateTime correctedTime = Instant.now().atZone(ZoneId.of("UTC"));
256+
// Object[][] rows = new Object[][] {
257+
// {"foo1", 0.3f, 0.6f, "a=1,b=2,c=5", correctedTime, 10},
258+
// {"foo2", 0.6f, 0.1f, "a=1,b=2,c=5", correctedTime, null},
259+
// {"foo3", 0.7f, 0.4f, "a=1,b=2,c=5", null, null},
260+
// {"foo4", 0.8f, 0.5f, null, null, null},
261+
// };
262+
//
263+
// TableSchema schema = client.getTableSchema(tableName);
264+
//
265+
// ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults;
266+
// try (InsertResponse response = client.insert(tableName, out -> {
267+
// RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format);
268+
// for (Object[] row : rows) {
269+
// for (int i = 0; i < row.length; i++) {
270+
// w.setValue(i + 1, row[i]);
271+
// }
272+
// w.commitRow();
273+
// }
274+
// }, format, new InsertSettings()).get()) {
275+
// System.out.println("Rows written: " + response.getWrittenRows());
276+
// }
277+
//
278+
// List<GenericRecord> records = client.queryAll("SELECT * FROM \"" + tableName + "\"" );
279+
//
280+
// for (GenericRecord record : records) {
281+
// System.out.println("> " + record.getString(1) + ", " + record.getFloat(2) + ", " + record.getFloat(3));
282+
// }
283+
// }
284+
}

0 commit comments

Comments
 (0)