DBTOOLS-1895 added json rules for clickhouse

Author: Ямалиев Рашит Азатович yamaliev_ra

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- Improved parser rules for working with JSON formats in ClikHouse.
+
 ### Changed
 
 ### Fixed

CHANGELOG.ru.md

@@ -9,6 +9,8 @@
 
 ### Добавлено
 
+- Улучшены правела парсера для работы с форматами JSON в ClikHouse.
+
 ### Изменено
 
 ### Исправлено

src/main/antlr4/org/pgcodekeeper/core/database/ch/parser/generated/CHParser.g4

@@ -945,14 +945,25 @@ move_stmt
     ;
 
 json_pair
-    : json_value COLON json_value
+    : json_key COLON json_value
     ;
 
-json_value
+json_key
     : DOUBLE_QUOTED_IDENTIFIER
     | literal
     ;
 
+json_value
+    : LBRACE (json_pair (COMMA json_pair)*)? RBRACE
+    | LBRACKET (json_value (COMMA json_value)*)? RBRACKET
+    | DOUBLE_QUOTED_IDENTIFIER
+    | signed_number_literal
+    | STRING_LITERAL
+    | NULL
+    | BINARY_LITERAL
+    | IDENTIFIER
+    ;
+
 insert_stmt
     : with_clause? INSERT INTO TABLE? (qualified_name | FUNCTION table_function_expr) settings_clause? columns_clause? data_clause
     ;
@@ -964,7 +975,7 @@ columns_clause
 
 data_clause
     : FROM INFILE STRING_LITERAL (COMPRESSION identifier)? settings_clause?
-    | FORMAT identifier
+    | FORMAT identifier (json_value (COMMA? json_value)*)?
     | VALUES values_values (COMMA values_values)*
     | select_stmt
     ;

src/test/resources/org/pgcodekeeper/core/it/parser/ch/ch_insert.sql

@@ -358,4 +358,21 @@ INSERT INTO test_table FROM INFILE '/path/to/file.csv' FORMAT CSV;
 INSERT INTO test_table FROM INFILE '/path/to/file.csv' COMPRESSION 'gzip';
 INSERT INTO test_table FROM INFILE '/path/to/file.csv' COMPRESSION 'gzip' FORMAT CSV;
 INSERT INTO test_table FROM INFILE '/path/to/file.csv' SETTINGS max_threads = 4;
-INSERT INTO test_table FROM INFILE '/path/to/file.csv' SETTINGS max_threads = 4 FORMAT CSV;
+INSERT INTO test_table FROM INFILE '/path/to/file.csv' SETTINGS max_threads = 4 FORMAT CSV;
+INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}};
+INSERT INTO t_json_desc FORMAT JSONAsObject {"k1": 10};
+INSERT INTO t_json_desc FORMAT JSONAsObject
+[
+    {"k1": 10, "k2": "value1"},
+    {"k1": 20, "k2": "value2"},
+    {"k1": 30, "k2": "value3"}
+];
+INSERT INTO t_json_desc FORMAT JSONAsObject
+{"k1": 10, "k2": "value1"}
+{"k1": 20, "k2": "value2"}
+{"k1": 30, "k2": "value3"};
+INSERT INTO test FORMAT JSONEachRow {"b": true, "n": false, "z": null, "neg": -3.14, "exp": 1.5e3, "s": "txt", "empty_obj": {}, "empty_arr": []};
+INSERT INTO test FORMAT JSONEachRow {"nested": {"arr": [{"x": 1}, {"x": -2}], "flag": true, "deep": {"a": {"b": [null, "v"]}}}};
+INSERT INTO test FORMAT JSONEachRow {"a": 1}{"a": 2}{"a": 3};
+INSERT INTO test FORMAT JSONAsObject [{"k": 1}, {"k": 2}];
+INSERT INTO test FORMAT JSONEachRow {"mixed": [1, "two", true, null, [3, 4], {"k": "v"}]};

src/test/resources/org/pgcodekeeper/core/it/parser/ch/ch_insert_refs.txt

@@ -358,4 +358,13 @@ Reference: action = INSERT, line number = 357, charPositionInLine = 0
 Reference: action = INSERT, line number = 358, charPositionInLine = 0
 Reference: action = INSERT, line number = 359, charPositionInLine = 0
 Reference: action = INSERT, line number = 360, charPositionInLine = 0
-Reference: action = INSERT, line number = 361, charPositionInLine = 0
+Reference: action = INSERT, line number = 361, charPositionInLine = 0
+Reference: action = INSERT, line number = 362, charPositionInLine = 0
+Reference: action = INSERT, line number = 363, charPositionInLine = 0
+Reference: action = INSERT, line number = 364, charPositionInLine = 0
+Reference: action = INSERT, line number = 370, charPositionInLine = 0
+Reference: action = INSERT, line number = 374, charPositionInLine = 0
+Reference: action = INSERT, line number = 375, charPositionInLine = 0
+Reference: action = INSERT, line number = 376, charPositionInLine = 0
+Reference: action = INSERT, line number = 377, charPositionInLine = 0
+Reference: action = INSERT, line number = 378, charPositionInLine = 0

src/test/resources/org/pgcodekeeper/core/it/parser/ch/ch_select.sql

@@ -1408,4 +1408,10 @@ select arrayMap( x -> x % 3 ? 1 : null, range(3)) as k from system.numbers_mt li
 SELECT () as empty_tuple;
 SELECT ((), 1, 'test');
 SELECT * FROM t WHERE x = ();
+SELECT json, JSONAllPathsWithTypes(json) FROM test;
+SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(String, Array(Float64))') AS res;
+SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', -2);
+SELECT JSON_QUERY('{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}', '$.array[*][0 to 2, 4]');
+SELECT JSON_VALUE('{"hello":"world"}', '$.b') settings function_json_value_return_type_allow_nullable=true;
+SELECT toJSONString(tuple(1.25, NULL, NaN, +inf, -inf, [])) SETTINGS output_format_json_quote_denormals = 1;
 SELECT ifNull(x, ()) FROM t;

src/test/resources/org/pgcodekeeper/core/it/parser/ch/ch_select_refs.txt

@@ -1326,4 +1326,10 @@ Reference: action = SELECT, line number = 1407, charPositionInLine = 0
 Reference: action = SELECT, line number = 1408, charPositionInLine = 0
 Reference: action = SELECT, line number = 1409, charPositionInLine = 0
 Reference: action = SELECT, line number = 1410, charPositionInLine = 0
-Reference: action = SELECT, line number = 1411, charPositionInLine = 0
+Reference: action = SELECT, line number = 1411, charPositionInLine = 0
+Reference: action = SELECT, line number = 1412, charPositionInLine = 0
+Reference: action = SELECT, line number = 1413, charPositionInLine = 0
+Reference: action = SELECT, line number = 1414, charPositionInLine = 0
+Reference: action = SELECT, line number = 1415, charPositionInLine = 0
+Reference: action = SELECT, line number = 1416, charPositionInLine = 0
+Reference: action = SELECT, line number = 1417, charPositionInLine = 0

src/test/resources/org/pgcodekeeper/core/it/parser/ch/ch_table.sql

@@ -994,6 +994,10 @@ CREATE TABLE test (json JSON) ENGINE = Memory;
 CREATE TABLE test (json JSON(a.b UInt32)) ENGINE = Memory;
 CREATE TABLE test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory;
 CREATE TABLE test (id UInt64, json JSON(max_dynamic_paths=3)) ENGINE=Memory ORDER BY id;
+CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
+CREATE TABLE test (json JSON(max_dynamic_types=8)) ENGINE = Memory;
+CREATE TABLE test (json JSON(SKIP REGEXP 'path.to.skip.*')) ENGINE = Memory;
+CREATE TABLE test (json JSON(max_dynamic_paths=10, max_dynamic_types=8, a.b UInt32, a.b.c String, SKIP a.e, SKIP REGEXP 'paths.*')) ENGINE = Memory;
 CREATE TABLE test (id UInt32, vec QBit(Float32, 8)) ENGINE = Memory;
 
 --Buffer