Skip to content

Commit 70cc2cf

Browse files
theorytheory
committed
Add support for the Decimal data type
The mappings already work for the HTTP engine, although its inserts were limited to `float8`, now fixed to use proper numeric formatting. Add the mappings for the binary engine. Use strings to convert between the Postgres and ClickHouse formats to as to avoid differences in their implementations: ClickHouse stores data as `Int32`, `Int64`, or `Int128`, while Postgres...well, I have no idea what the internals are, but using the `numeric_in` and `numeric_out` functions avoid all issues. Conversion from `Decimal` values returned from ClickHouse requires converting its internal `Int128` into a string. The clickhouse-cpp library doesn't provide this functionality, alas, so implement it here. Add a test that covers both engines.
1 parent 372652f commit 70cc2cf

4 files changed

Lines changed: 229 additions & 6 deletions

File tree

‎src/binary.cpp‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#include <sstream>
12
#include <iostream>
23
#include <cassert>
34
#include <stdexcept>
@@ -10,6 +11,7 @@
1011
#include <clickhouse/client.h>
1112
#include <clickhouse/query.h>
1213
#include <clickhouse/types/types.h>
14+
#include "absl/numeric/int128.h"
1315

1416
#if __cplusplus > 199711L
1517
#define register // Deprecated in C++11.
@@ -250,6 +252,11 @@ static Oid get_corr_postgres_type(const TypeRef & type)
250252
return FLOAT4OID;
251253
case Type::Code::Float64:
252254
return FLOAT8OID;
255+
case Type::Code::Decimal128:
256+
case Type::Code::Decimal64:
257+
case Type::Code::Decimal32:
258+
case Type::Code::Decimal:
259+
return NUMERICOID;
253260
case Type::Code::FixedString:
254261
case Type::Code::Enum8:
255262
case Type::Code::Enum16:
@@ -461,6 +468,26 @@ static void column_append(clickhouse::ColumnRef col, Datum val, Oid valtype, boo
461468
}
462469
break;
463470
}
471+
case NUMERICOID: {
472+
// Convert numeric to string and let ColumnDecimal parse it.
473+
char *s = DatumGetCString(DirectFunctionCall1(numeric_out, val));
474+
switch (col->Type()->GetCode())
475+
{
476+
case Type::Code::Decimal128:
477+
case Type::Code::Decimal64:
478+
case Type::Code::Decimal32:
479+
case Type::Code::Decimal:
480+
col->As<ColumnDecimal>()->Append(std::string(s));
481+
break;
482+
default:
483+
throw std::runtime_error(
484+
"unexpected column "
485+
"type for NUMERIC: "
486+
+ col->Type()->GetName());
487+
}
488+
pfree(s);
489+
break;
490+
}
464491
case TEXTOID: {
465492
char * s = TextDatumGetCString(val);
466493

@@ -745,6 +772,55 @@ static Datum make_datum(clickhouse::ColumnRef col, size_t row, Oid * valtype, bo
745772
*valtype = FLOAT8OID;
746773
}
747774
break;
775+
case Type::Code::Decimal128:
776+
case Type::Code::Decimal64:
777+
case Type::Code::Decimal32:
778+
case Type::Code::Decimal:
779+
{
780+
auto decCol = col->As<ColumnDecimal>();
781+
auto val = decCol->At(row);
782+
783+
// Convert the Int128 to a string.
784+
std::stringstream ss;
785+
ss << val;
786+
std::string str = ss.str();
787+
788+
// Start a destination string.
789+
std::stringstream res;
790+
auto scale = decCol->GetScale();
791+
792+
// Output a dash for negative values
793+
if (val < 0)
794+
{
795+
res << '-';
796+
str.erase(0, 1);
797+
}
798+
799+
if (str.length() < scale)
800+
{
801+
// Append the entire value prepended with zeros after the decimal.
802+
res << "0." << std::string(scale-1, '0') << str;
803+
}
804+
else
805+
{
806+
// There are digits before the decimal.
807+
auto decAt = str.length() - scale;
808+
res << str.substr(0, decAt);
809+
810+
// Append any digits after the decimal.
811+
if (decAt < str.length())
812+
{
813+
res << '.' << str.substr(decAt);
814+
}
815+
}
816+
817+
ret = DirectFunctionCall3(numeric_in,
818+
CStringGetDatum(res.str().c_str()),
819+
ObjectIdGetDatum(0),
820+
Int32GetDatum(-1));
821+
*valtype = NUMERICOID;
822+
}
823+
break;
748824
case Type::Code::FixedString: {
749825
auto s = std::string(col->As<ColumnFixedString>()->At(row));
750826
ret = CStringGetTextDatum(s.c_str());

‎src/pglink.c‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -377,12 +377,9 @@ extend_insert_query(ch_http_insert_state *state, TupleTableSlot *slot)
377377
break;
378378
case NUMERICOID:
379379
{
380-
Datum valueDatum;
381-
float8 f;
382-
383-
valueDatum = DirectFunctionCall1(numeric_float8, value);
384-
f = DatumGetFloat8(valueDatum);
385-
appendStringInfo(&state->sql, "%f", f);
380+
char *extval = DatumGetCString(DirectFunctionCall1(numeric_out, value));
381+
appendStringInfoString(&state->sql, extval);
382+
pfree(extval);
386383
}
387384
break;
388385
case BPCHAROID:

‎test/expected/decimal.out‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'binary');
3+
CREATE SERVER http_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS decimal_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE decimal_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE decimal_test.decimals (
20+
id Int32 NOT NULL,
21+
dec Decimal(8, 0) NOT NULL,
22+
dec32 Decimal32(4) NOT NULL,
23+
dec64 Decimal64(6) NOT NULL,
24+
dec128 Decimal128(8) NOT NULL
25+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
26+
$$);
27+
clickhouse_raw_query
28+
----------------------
29+
30+
(1 row)
31+
32+
CREATE SCHEMA dec_bin;
33+
CREATE SCHEMA dec_http;
34+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER binary_decimal_loopback INTO dec_bin;
35+
\d dec_bin.decimals
36+
Foreign table "dec_bin.decimals"
37+
Column | Type | Collation | Nullable | Default | FDW options
38+
--------+---------------+-----------+----------+---------+-------------
39+
id | integer | | not null | |
40+
dec | numeric(8,0) | | not null | |
41+
dec32 | numeric(9,4) | | not null | |
42+
dec64 | numeric(18,6) | | not null | |
43+
dec128 | numeric(38,8) | | not null | |
44+
Server: binary_decimal_loopback
45+
FDW options: (database 'decimal_test', table_name 'decimals', engine 'MergeTree')
46+
47+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER http_decimal_loopback INTO dec_http;
48+
\d dec_http.decimals
49+
Foreign table "dec_http.decimals"
50+
Column | Type | Collation | Nullable | Default | FDW options
51+
--------+---------------+-----------+----------+---------+-------------
52+
id | integer | | not null | |
53+
dec | numeric(8,0) | | not null | |
54+
dec32 | numeric(9,4) | | not null | |
55+
dec64 | numeric(18,6) | | not null | |
56+
dec128 | numeric(38,8) | | not null | |
57+
Server: http_decimal_loopback
58+
FDW options: (database 'decimal_test', table_name 'decimals', engine 'MergeTree')
59+
60+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
61+
INSERT INTO dec_bin.decimals (id, dec, dec32, dec64, dec128) VALUES
62+
(1, 42::NUMERIC, 98.6::NUMERIC, 102.4::NUMERIC, 1024.003::NUMERIC),
63+
(2, 9999, 9999.9999, 9999999.999999, 99999999999.99999999),
64+
(3, -9999, -9999.9999, -9999999.999999, -99999999999.99999999)
65+
;
66+
INSERT INTO dec_http.decimals VALUES
67+
(4, 1000000::NUMERIC, 10000::NUMERIC, 3000000000::NUMERIC, 400000000000::NUMERIC),
68+
(5, -1, -0.0001, -0.000001, -0.00000001),
69+
(6, 0, 0, 0, 0)
70+
;
71+
SELECT * FROM dec_bin.decimals ORDER BY id;
72+
id | dec | dec32 | dec64 | dec128
73+
----+---------+------------+-------------------+-----------------------
74+
1 | 42 | 98.6000 | 102.400000 | 1024.00300000
75+
2 | 9999 | 9999.9999 | 9999999.999999 | 99999999999.99999999
76+
3 | -9999 | -9999.9999 | -9999999.999999 | -99999999999.99999999
77+
4 | 1000000 | 10000.0000 | 3000000000.000000 | 400000000000.00000000
78+
5 | -1 | -0.0001 | -0.000001 | -0.00000001
79+
6 | 0 | 0.0000 | 0.000000 | 0.00000000
80+
(6 rows)
81+
82+
SELECT * FROM dec_http.decimals ORDER BY id;
83+
id | dec | dec32 | dec64 | dec128
84+
----+---------+------------+-------------------+-----------------------
85+
1 | 42 | 98.6000 | 102.400000 | 1024.00300000
86+
2 | 9999 | 9999.9999 | 9999999.999999 | 99999999999.99999999
87+
3 | -9999 | -9999.9999 | -9999999.999999 | -99999999999.99999999
88+
4 | 1000000 | 10000.0000 | 3000000000.000000 | 400000000000.00000000
89+
5 | -1 | -0.0001 | -0.000001 | -0.00000001
90+
6 | 0 | 0.0000 | 0.000000 | 0.00000000
91+
(6 rows)
92+
93+
SELECT clickhouse_raw_query('DROP DATABASE decimal_test');
94+
clickhouse_raw_query
95+
----------------------
96+
97+
(1 row)
98+
99+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
100+
DROP USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
101+
DROP SERVER binary_decimal_loopback CASCADE;
102+
NOTICE: drop cascades to foreign table dec_bin.decimals
103+
DROP SERVER http_decimal_loopback CASCADE;
104+
NOTICE: drop cascades to foreign table dec_http.decimals

‎test/sql/decimal.sql‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'binary');
3+
CREATE SERVER http_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
6+
7+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS decimal_test');
8+
SELECT clickhouse_raw_query('CREATE DATABASE decimal_test');
9+
SELECT clickhouse_raw_query($$
10+
CREATE TABLE decimal_test.decimals (
11+
id Int32 NOT NULL,
12+
dec Decimal(8, 0) NOT NULL,
13+
dec32 Decimal32(4) NOT NULL,
14+
dec64 Decimal64(6) NOT NULL,
15+
dec128 Decimal128(8) NOT NULL
16+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
17+
$$);
18+
19+
CREATE SCHEMA dec_bin;
20+
CREATE SCHEMA dec_http;
21+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER binary_decimal_loopback INTO dec_bin;
22+
\d dec_bin.decimals
23+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER http_decimal_loopback INTO dec_http;
24+
\d dec_http.decimals
25+
26+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
27+
INSERT INTO dec_bin.decimals (id, dec, dec32, dec64, dec128) VALUES
28+
(1, 42::NUMERIC, 98.6::NUMERIC, 102.4::NUMERIC, 1024.003::NUMERIC),
29+
(2, 9999, 9999.9999, 9999999.999999, 99999999999.99999999),
30+
(3, -9999, -9999.9999, -9999999.999999, -99999999999.99999999)
31+
;
32+
33+
INSERT INTO dec_http.decimals VALUES
34+
(4, 1000000::NUMERIC, 10000::NUMERIC, 3000000000::NUMERIC, 400000000000::NUMERIC),
35+
(5, -1, -0.0001, -0.000001, -0.00000001),
36+
(6, 0, 0, 0, 0)
37+
;
38+
39+
SELECT * FROM dec_bin.decimals ORDER BY id;
40+
SELECT * FROM dec_http.decimals ORDER BY id;
41+
42+
SELECT clickhouse_raw_query('DROP DATABASE decimal_test');
43+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
44+
DROP USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
45+
DROP SERVER binary_decimal_loopback CASCADE;
46+
DROP SERVER http_decimal_loopback CASCADE;

0 commit comments

Comments
 (0)