Skip to content

[client-v2] Fixed converting integers thru decimal #2814

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chernser wants to merge 5 commits into
base: main
Choose a base branch
from
Open

[client-v2] Fixed converting integers thru decimal #2814

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
67f3c40
Fixed converting integers thru decimal
chernser Mar 31, 2026
10a54b7
deduplicated some code
chernser Apr 1, 2026
f1c7d4a
Merge branch 'main' into 03/31/26/fix_decimal_truncate
chernser Apr 1, 2026
95face3
Merge branch 'main' into 03/31/26/fix_decimal_truncate
chernser Apr 2, 2026
860ed34
Added more tests and fixed issue
chernser Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/NumberConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,12 @@ public static BigDecimal toBigDecimal(Object value) {
return (BigDecimal) value;
} else if (value instanceof BigInteger) {
return new BigDecimal((BigInteger) value);
} else if (value instanceof Number) {
} else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) {
return BigDecimal.valueOf(((Number) value).longValue());
} else if (value instanceof Float || value instanceof Double) {
return BigDecimal.valueOf(((Number) value).doubleValue());
} else if (value instanceof Number) {
return new BigDecimal(value.toString());
} else if (value instanceof String) {
return new BigDecimal((String) value);
} else if (value instanceof Boolean) {
Expand Down
39 changes: 7 additions & 32 deletions client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL;
import static org.objectweb.asm.Opcodes.RETURN;

@SuppressWarnings("deprecation")
public class SerializerUtils {

public static void serializeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
Expand Down Expand Up @@ -505,10 +506,10 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
BinaryStreamUtils.writeInt64(stream, convertToLong(value));
break;
case Int128:
BinaryStreamUtils.writeInt128(stream, convertToBigInteger(value));
BinaryStreamUtils.writeInt128(stream, NumberConverter.toBigInteger(value));
break;
case Int256:
BinaryStreamUtils.writeInt256(stream, convertToBigInteger(value));
BinaryStreamUtils.writeInt256(stream, NumberConverter.toBigInteger(value));
break;
case UInt8:
BinaryStreamUtils.writeUnsignedInt8(stream, convertToInteger(value));
Expand All @@ -520,13 +521,13 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
BinaryStreamUtils.writeUnsignedInt32(stream, convertToLong(value));
break;
case UInt64:
BinaryStreamUtils.writeUnsignedInt64(stream, convertToLong(value));
BinaryStreamUtils.writeUnsignedInt64(stream, NumberConverter.toBigInteger(value));
break;
case UInt128:
BinaryStreamUtils.writeUnsignedInt128(stream, convertToBigInteger(value));
BinaryStreamUtils.writeUnsignedInt128(stream, NumberConverter.toBigInteger(value));
break;
case UInt256:
BinaryStreamUtils.writeUnsignedInt256(stream, convertToBigInteger(value));
BinaryStreamUtils.writeUnsignedInt256(stream, NumberConverter.toBigInteger(value));
break;
case Float32:
BinaryStreamUtils.writeFloat32(stream, (float) value);
Expand All @@ -539,7 +540,7 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
case Decimal64:
case Decimal128:
case Decimal256:
BinaryStreamUtils.writeDecimal(stream, convertToBigDecimal(value), column.getPrecision(), column.getScale());
BinaryStreamUtils.writeDecimal(stream, NumberConverter.toBigDecimal(value), column.getPrecision(), column.getScale());
break;
case Bool:
BinaryStreamUtils.writeBoolean(stream, (Boolean) value);
Expand Down Expand Up @@ -765,32 +766,6 @@ public static Long convertToLong(Object value) {
}
}

public static BigInteger convertToBigInteger(Object value) {
if (value instanceof BigInteger) {
return (BigInteger) value;
} else if (value instanceof Number) {
return BigInteger.valueOf(((Number) value).longValue());
} else if (value instanceof String) {
return new BigInteger((String) value);
} else {
throw new IllegalArgumentException("Cannot convert " + value + " to BigInteger");
}
}

public static BigDecimal convertToBigDecimal(Object value) {
if (value instanceof BigDecimal) {
return (BigDecimal) value;
} else if (value instanceof BigInteger) {
return new BigDecimal((BigInteger) value);
} else if (value instanceof Number) {
return BigDecimal.valueOf(((Number) value).doubleValue());
} else if (value instanceof String) {
return new BigDecimal((String) value);
} else {
throw new IllegalArgumentException("Cannot convert " + value + " to BigDecimal");
}
}

public static String convertToString(Object value) {
return java.lang.String.valueOf(value);
}
Expand Down
9 changes: 7 additions & 2 deletions client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/ValueConverters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import java.math.BigInteger;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Collections;
Expand Down Expand Up @@ -210,7 +209,13 @@ public BigInteger convertNumberToBigInteger(Object value) {
}

public BigDecimal convertNumberToBigDecimal(Object value) {
return BigDecimal.valueOf(((Number) value).doubleValue());
Number number = (Number) value;
Copy link
Copy Markdown
Contributor

@mzitnik mzitnik Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

missing checks

  • value not null
  • instanceof Number before cast

if (number instanceof Byte || number instanceof Short || number instanceof Integer || number instanceof Long) {
return BigDecimal.valueOf(number.longValue());
} else if (number instanceof Float || number instanceof Double) {
return BigDecimal.valueOf(number.doubleValue());
}
return new BigDecimal(number.toString());
}

// Date & Time converters
Expand Down
61 changes: 61 additions & 0 deletions ...rc/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.util.TimeZone;
import java.util.function.Consumer;

@SuppressWarnings("deprecation")
public class ClickHouseBinaryFormatReaderTest {

@Test
Expand Down Expand Up @@ -83,6 +84,66 @@ public void testReadingNumbers() throws IOException {
}
}

@Test
public void testGetBigDecimalForIntegerWidths8To256() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();

String[] names = new String[] {
"i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64", "i128", "u128", "i256", "u256"
};
String[] types = new String[] {
"Int8", "UInt8", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64",
"Int128", "UInt128", "Int256", "UInt256"
};

BinaryStreamUtils.writeVarInt(out, names.length);
for (String name : names) {
BinaryStreamUtils.writeString(out, name);
}
for (String type : types) {
BinaryStreamUtils.writeString(out, type);
}

BigInteger u64 = new BigInteger("18446744073709551615");
BigInteger i128 = new BigInteger("-170141183460469231731687303715884105728");
BigInteger u128 = new BigInteger("340282366920938463463374607431768211455");
BigInteger i256 = new BigInteger("-57896044618658097711785492504343953926634992332820282019728792003956564819968");
BigInteger u256 = new BigInteger("115792089237316195423570985008687907853269984665640564039457584007913129639935");

BinaryStreamUtils.writeInt8(out, -128);
BinaryStreamUtils.writeUnsignedInt8(out, 255);
BinaryStreamUtils.writeInt16(out, -32768);
BinaryStreamUtils.writeUnsignedInt16(out, 65535);
BinaryStreamUtils.writeInt32(out, Integer.MIN_VALUE);
BinaryStreamUtils.writeUnsignedInt32(out, 4294967295L);
BinaryStreamUtils.writeInt64(out, Long.MAX_VALUE);
BinaryStreamUtils.writeUnsignedInt64(out, u64);
BinaryStreamUtils.writeInt128(out, i128);
BinaryStreamUtils.writeUnsignedInt128(out, u128);
BinaryStreamUtils.writeInt256(out, i256);
BinaryStreamUtils.writeUnsignedInt256(out, u256);

InputStream in = new ByteArrayInputStream(out.toByteArray());
QuerySettings querySettings = new QuerySettings().setUseTimeZone(TimeZone.getTimeZone("UTC").toZoneId().getId());
RowBinaryWithNamesAndTypesFormatReader reader =
new RowBinaryWithNamesAndTypesFormatReader(in, querySettings, new BinaryStreamReader.CachingByteBufferAllocator());

reader.next();

Assert.assertEquals(reader.getBigDecimal("i8"), BigDecimal.valueOf(-128));
Assert.assertEquals(reader.getBigDecimal("u8"), BigDecimal.valueOf(255));
Assert.assertEquals(reader.getBigDecimal("i16"), BigDecimal.valueOf(-32768));
Assert.assertEquals(reader.getBigDecimal("u16"), BigDecimal.valueOf(65535));
Assert.assertEquals(reader.getBigDecimal("i32"), BigDecimal.valueOf(Integer.MIN_VALUE));
Assert.assertEquals(reader.getBigDecimal("u32"), BigDecimal.valueOf(4294967295L));
Assert.assertEquals(reader.getBigDecimal("i64"), BigDecimal.valueOf(Long.MAX_VALUE));
Assert.assertEquals(reader.getBigDecimal("u64"), new BigDecimal(u64));
Assert.assertEquals(reader.getBigDecimal("i128"), new BigDecimal(i128));
Assert.assertEquals(reader.getBigDecimal("u128"), new BigDecimal(u128));
Assert.assertEquals(reader.getBigDecimal("i256"), new BigDecimal(i256));
Assert.assertEquals(reader.getBigDecimal("u256"), new BigDecimal(u256));
}

@Test
public void testReadingNumbersWithOverflow() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Expand Down
18 changes: 18 additions & 0 deletions client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,24 @@ public void testVariantWithDecimals() throws Exception {
});
}

@Test(groups = {"integration"})
public void testGetBigDecimalDoesNotTruncateLargeIntegers() throws Exception {
BigDecimal expectedInt64 = BigDecimal.valueOf(Long.MAX_VALUE);
BigDecimal expectedUInt64 = new BigDecimal("18446744073709551615");

String sql = "SELECT toInt64(" + Long.MAX_VALUE + ") AS i64, "
+ "toUInt64('" + expectedUInt64.toPlainString() + "') AS u64";

try (QueryResponse response = client.query(sql).get(3, TimeUnit.SECONDS)) {
Copy link

Copilot AI Apr 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This integration test uses a hard-coded 3-second timeout on client.query(sql).get(...), while the rest of DataTypeTests generally uses .get() without a short timeout. A small timeout can make CI runs flaky on slower environments. Consider aligning with the existing pattern (no timeout) or using a shared/longer timeout constant for integration tests in this class.

Suggested change
try (QueryResponse response = client.query(sql).get(3, TimeUnit.SECONDS)) {
try (QueryResponse response = client.query(sql).get()) {

Copilot uses AI. Check for mistakes.
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response);

Assert.assertNotNull(reader.next());
Assert.assertEquals(reader.getBigDecimal("i64"), expectedInt64);
Assert.assertEquals(reader.getBigDecimal("u64"), expectedUInt64);
Assert.assertFalse(reader.hasNext());
}
}

@Test(groups = {"integration"})
public void testVariantWithArrays() throws Exception {
testVariantWith("arrays", new String[]{"field Variant(String, Array(String))"},
Expand Down
122 changes: 122 additions & 0 deletions client-v2/src/test/java/com/clickhouse/client/internal/SerializerUtilsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,22 @@
package com.clickhouse.client.internal;

import com.clickhouse.client.api.data_formats.RowBinaryWithNamesAndTypesFormatReader;
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
import com.clickhouse.client.api.data_formats.internal.SerializerUtils;
import com.clickhouse.client.api.query.QuerySettings;
import com.clickhouse.data.ClickHouseColumn;
import com.clickhouse.data.format.BinaryStreamUtils;
import org.testng.annotations.Test;

import org.testng.Assert;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.TimeZone;

public class SerializerUtilsTests {

@Test
Expand All @@ -13,4 +25,114 @@
Assert.assertEquals(SerializerUtils.convertToInteger("1640995199").intValue(), expected);
Assert.assertEquals(SerializerUtils.convertToInteger(false).intValue(), 0);
}

@Test
public void testSerializeNumbersRoundTrip() throws IOException {
String[] names = new String[]{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
"p", "q", "r"};
String[] types = new String[]{"Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64",
"Int128", "Int256", "UInt128", "UInt256", "Float32", "Float64", "Decimal32(3)", "Decimal64(3)",
"Decimal128(4)", "Decimal256(4)"};
Object[] values = new Object[]{
(byte) 120,
(short) 120,
120,
120L,
120,
120,
120L,
BigInteger.valueOf(120),
BigInteger.valueOf(120),
BigInteger.valueOf(120),
BigInteger.valueOf(120),
BigInteger.valueOf(120),
120.0f,
Copy link
Copy Markdown
Contributor

@mzitnik mzitnik Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think if we set it to 0.1f, we can lose precision, lets add different values to test the conversion logic

120.0d,
BigDecimal.valueOf(120),
BigDecimal.valueOf(120),
BigDecimal.valueOf(120),
BigDecimal.valueOf(120)
};

RowBinaryWithNamesAndTypesFormatReader reader = serializeRow(names, types, values);
reader.next();

for (String name : names) {
Assert.assertTrue(reader.getBigDecimal(name).compareTo(BigDecimal.valueOf(120)) == 0);
}
}

@Test
public void testSerializeIntegerWidths8To256RoundTripAsBigDecimal() throws IOException {
String[] names = new String[]{
"i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64", "i128", "u128", "i256", "u256"
};
String[] types = new String[]{
"Int8", "UInt8", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64",
"Int128", "UInt128", "Int256", "UInt256"
};

BigInteger u64 = new BigInteger("18446744073709551615");
BigInteger i128 = new BigInteger("-170141183460469231731687303715884105728");
BigInteger u128 = new BigInteger("340282366920938463463374607431768211455");
BigInteger i256 = new BigInteger("-57896044618658097711785492504343953926634992332820282019728792003956564819968");
BigInteger u256 = new BigInteger("115792089237316195423570985008687907853269984665640564039457584007913129639935");

Object[] values = new Object[]{
-128,
255,
-32768,
65535,
Integer.MIN_VALUE,
4294967295L,
Long.MAX_VALUE,
u64,
i128,
u128,
i256,
u256
};

BigDecimal[] expected = new BigDecimal[]{
BigDecimal.valueOf(-128),
BigDecimal.valueOf(255),
BigDecimal.valueOf(-32768),
BigDecimal.valueOf(65535),
BigDecimal.valueOf(Integer.MIN_VALUE),
BigDecimal.valueOf(4294967295L),
BigDecimal.valueOf(Long.MAX_VALUE),
new BigDecimal(u64),
new BigDecimal(i128),
new BigDecimal(u128),
new BigDecimal(i256),
new BigDecimal(u256)
};

RowBinaryWithNamesAndTypesFormatReader reader = serializeRow(names, types, values);
reader.next();

for (int i = 0; i < names.length; i++) {
Assert.assertEquals(reader.getBigDecimal(names[i]), expected[i]);
}
}

private RowBinaryWithNamesAndTypesFormatReader serializeRow(String[] names, String[] types, Object[] values)
throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
BinaryStreamUtils.writeVarInt(out, names.length);

Check warning on line 122 in client-v2/src/test/java/com/clickhouse/client/internal/SerializerUtilsTests.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this use of "BinaryStreamUtils"; it is deprecated. 


See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ1Pf2X1qMfC8EjjbkYm&open=AZ1Pf2X1qMfC8EjjbkYm&pullRequest=2814
for (String name : names) {
BinaryStreamUtils.writeString(out, name);

Check warning on line 124 in client-v2/src/test/java/com/clickhouse/client/internal/SerializerUtilsTests.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this use of "BinaryStreamUtils"; it is deprecated. 


See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ1Pf2X1qMfC8EjjbkYn&open=AZ1Pf2X1qMfC8EjjbkYn&pullRequest=2814
}
for (String type : types) {
BinaryStreamUtils.writeString(out, type);

Check warning on line 127 in client-v2/src/test/java/com/clickhouse/client/internal/SerializerUtilsTests.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this use of "BinaryStreamUtils"; it is deprecated. 


See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ1Pf2X1qMfC8EjjbkYo&open=AZ1Pf2X1qMfC8EjjbkYo&pullRequest=2814
}
for (int i = 0; i < names.length; i++) {
SerializerUtils.serializeData(out, values[i], ClickHouseColumn.of(names[i], types[i]));
}

return new RowBinaryWithNamesAndTypesFormatReader(
new ByteArrayInputStream(out.toByteArray()),
new QuerySettings().setUseTimeZone(TimeZone.getTimeZone("UTC").toZoneId().getId()),
new BinaryStreamReader.CachingByteBufferAllocator());
}
}
12 changes: 11 additions & 1 deletion jdbc-v2/src/test/java/com/clickhouse/jdbc/types/ArrayResultSetTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ void testPrimitiveValues(Object array, ClickHouseColumn column) throws SQLExcept
assertEquals(rs.getLong(valueColumn), number.longValue());
assertEquals(rs.getFloat(valueColumn), number.floatValue());
assertEquals(rs.getDouble(valueColumn), number.doubleValue());
assertEquals(rs.getBigDecimal(valueColumn), BigDecimal.valueOf(number.doubleValue()));
assertEquals(rs.getBigDecimal(valueColumn), expectedBigDecimal(number));
} else if (itemClass == Boolean.class || itemClass == boolean.class) {
Number number = ((Boolean) value) ? 1 : 0;
assertEquals(rs.getBoolean(valueColumn), ((Boolean) value));
Expand All @@ -207,6 +207,16 @@ void testPrimitiveValues(Object array, ClickHouseColumn column) throws SQLExcept
}
}

private static BigDecimal expectedBigDecimal(Number number) {
if (number instanceof Byte || number instanceof Short || number instanceof Integer || number instanceof Long) {
return BigDecimal.valueOf(number.longValue());
} else if (number instanceof Float || number instanceof Double) {
return BigDecimal.valueOf(number.doubleValue());
}

return new BigDecimal(number.toString());
}

@DataProvider
static Object[][] testPrimitiveValues() {
return new Object[][]{
Expand Down
Loading