Skip to content

Commit b2ed603

Browse files
authored
Support JSON functions (#261)
* Support JSON * add tests for mysql * refactor * fmt * organise * fix * cleanup * fmt * fix tests * fix * done * remove mysql 5.6 check from general * remove mysql 5.6 check from general * fix * fmt * fix * fix * fmt * fix
1 parent 2fc5ecd commit b2ed603

File tree

57 files changed

+4231
-87
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+4231
-87
lines changed

.github/workflows/rust.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ jobs:
6666

6767
- name: Run tests
6868
run: cargo test -- --test-threads=1
69+
env:
70+
MYSQL_VERSION: ${{ matrix.db.mysql }}
71+
PG_VERSION: ${{ matrix.db.postgres }}
6972

7073
lint:
7174
runs-on: ubuntu-latest

docker-compose.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ version: '3.1'
33
services:
44

55
postgres:
6+
platform: linux/amd64
67
image: postgres:${PG_VERSION:-13}
78
restart: always
89
environment:
@@ -13,6 +14,7 @@ services:
1314
- 54321:5432
1415

1516
mysql:
17+
platform: linux/amd64
1618
image: mysql:${MYSQL_VERSION:-8}
1719
restart: always
1820
volumes:

playpen/db/mysql_migration.sql

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,3 +172,54 @@ CREATE TABLE random (
172172
-- JSON types
173173
json1 JSON
174174
);
175+
176+
-- JSON Test Data Table
177+
-- This table contains various JSON structures for testing JSON operators and functions
178+
CREATE TABLE json_test_data (
179+
id INT AUTO_INCREMENT PRIMARY KEY,
180+
name VARCHAR(100) NOT NULL,
181+
data JSON NOT NULL,
182+
metadata JSON,
183+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
184+
);
185+
186+
INSERT INTO json_test_data (name, data, metadata) VALUES
187+
-- Simple object
188+
('user_profile',
189+
'{"userId": 1, "username": "john_doe", "email": "john@example.com", "age": 30, "active": true}',
190+
'{"source": "api", "version": "1.0"}'),
191+
192+
-- Nested object with address
193+
('user_with_address',
194+
'{"userId": 2, "username": "jane_smith", "email": "jane@example.com", "address": {"street": "123 Main St", "city": "Springfield", "state": "IL", "zipCode": "62701", "country": "USA"}}',
195+
'{"source": "import", "version": "1.0"}'),
196+
197+
-- Array of items
198+
('shopping_cart',
199+
'{"cartId": 101, "items": [{"productId": 1, "name": "Laptop", "quantity": 1, "price": 999.99}, {"productId": 2, "name": "Mouse", "quantity": 2, "price": 25.50}], "totalPrice": 1050.99}',
200+
'{"source": "web", "version": "2.0"}'),
201+
202+
-- Array of strings
203+
('tags',
204+
'{"postId": 42, "title": "MySQL JSON Functions", "tags": ["database", "mysql", "json", "tutorial"], "published": true}',
205+
'{"source": "cms", "version": "1.0"}'),
206+
207+
-- Nested arrays and objects
208+
('game_stats',
209+
'{"playerId": 123, "stats": {"level": 50, "experience": 125000, "inventory": [{"slot": 1, "item": "Sword of Fire", "rarity": "legendary"}, {"slot": 2, "item": "Shield of Light", "rarity": "epic"}], "achievements": ["First Kill", "Level 50", "Legendary Item"]}}',
210+
'{"source": "game_server", "version": "3.0"}'),
211+
212+
-- Deep nesting
213+
('nested_config',
214+
'{"app": {"name": "MyApp", "version": "1.0.0", "settings": {"database": {"host": "localhost", "port": 3306, "credentials": {"username": "admin", "encrypted": true}}, "features": {"darkMode": true, "notifications": {"email": true, "push": false}}}}}',
215+
'{"source": "config", "version": "1.0"}'),
216+
217+
-- Array of objects with nulls
218+
('product_reviews',
219+
'{"productId": 456, "reviews": [{"reviewId": 1, "rating": 5, "comment": "Excellent product!", "reviewer": "Alice"}, {"reviewId": 2, "rating": 4, "comment": null, "reviewer": "Bob"}, {"reviewId": 3, "rating": 3, "comment": "Average", "reviewer": null}]}',
220+
'{"source": "reviews", "version": "1.0"}'),
221+
222+
-- Mixed types
223+
('analytics',
224+
'{"date": "2024-01-15", "metrics": {"visitors": 1500, "pageViews": 4500, "bounceRate": 0.35, "sources": {"organic": 850, "direct": 400, "referral": 250}}}',
225+
'{"source": "analytics", "version": "1.0"}');

playpen/db/mysql_migration_5_6.sql

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,3 +158,54 @@ CREATE TABLE random (
158158

159159
json1 TEXT
160160
);
161+
162+
-- JSON Test Data Table
163+
-- This table contains various JSON structures for testing JSON operators and functions
164+
CREATE TABLE json_test_data (
165+
id INT AUTO_INCREMENT PRIMARY KEY,
166+
name VARCHAR(100) NOT NULL,
167+
data TEXT NOT NULL,
168+
metadata TEXT,
169+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
170+
);
171+
172+
INSERT INTO json_test_data (name, data, metadata) VALUES
173+
-- Simple object
174+
('user_profile',
175+
'{"userId": 1, "username": "john_doe", "email": "john@example.com", "age": 30, "active": true}',
176+
'{"source": "api", "version": "1.0"}'),
177+
178+
-- Nested object with address
179+
('user_with_address',
180+
'{"userId": 2, "username": "jane_smith", "email": "jane@example.com", "address": {"street": "123 Main St", "city": "Springfield", "state": "IL", "zipCode": "62701", "country": "USA"}}',
181+
'{"source": "import", "version": "1.0"}'),
182+
183+
-- Array of items
184+
('shopping_cart',
185+
'{"cartId": 101, "items": [{"productId": 1, "name": "Laptop", "quantity": 1, "price": 999.99}, {"productId": 2, "name": "Mouse", "quantity": 2, "price": 25.50}], "totalPrice": 1050.99}',
186+
'{"source": "web", "version": "2.0"}'),
187+
188+
-- Array of strings
189+
('tags',
190+
'{"postId": 42, "title": "MySQL JSON Functions", "tags": ["database", "mysql", "json", "tutorial"], "published": true}',
191+
'{"source": "cms", "version": "1.0"}'),
192+
193+
-- Nested arrays and objects
194+
('game_stats',
195+
'{"playerId": 123, "stats": {"level": 50, "experience": 125000, "inventory": [{"slot": 1, "item": "Sword of Fire", "rarity": "legendary"}, {"slot": 2, "item": "Shield of Light", "rarity": "epic"}], "achievements": ["First Kill", "Level 50", "Legendary Item"]}}',
196+
'{"source": "game_server", "version": "3.0"}'),
197+
198+
-- Deep nesting
199+
('nested_config',
200+
'{"app": {"name": "MyApp", "version": "1.0.0", "settings": {"database": {"host": "localhost", "port": 3306, "credentials": {"username": "admin", "encrypted": true}}, "features": {"darkMode": true, "notifications": {"email": true, "push": false}}}}}',
201+
'{"source": "config", "version": "1.0"}'),
202+
203+
-- Array of objects with nulls
204+
('product_reviews',
205+
'{"productId": 456, "reviews": [{"reviewId": 1, "rating": 5, "comment": "Excellent product!", "reviewer": "Alice"}, {"reviewId": 2, "rating": 4, "comment": null, "reviewer": "Bob"}, {"reviewId": 3, "rating": 3, "comment": "Average", "reviewer": null}]}',
206+
'{"source": "reviews", "version": "1.0"}'),
207+
208+
-- Mixed types
209+
('analytics',
210+
'{"date": "2024-01-15", "metrics": {"visitors": 1500, "pageViews": 4500, "bounceRate": 0.35, "sources": {"organic": 850, "direct": 400, "referral": 250}}}',
211+
'{"source": "analytics", "version": "1.0"}');

playpen/db/postgres_migration.sql

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,3 +218,54 @@ INSERT INTO classes (name, specialization) VALUES
218218
('druid', '{"role": "hybrid", "weapon": "staff", "abilities": ["shapeshift", "moonfire", "regrowth"]}'),
219219
('mage', '{"role": "ranged", "weapon": "wand", "abilities": ["fireball", "frostbolt", "arcane blast"]}'),
220220
('warlock', '{"role": "ranged", "weapon": "dagger", "abilities": ["summon demon", "shadowbolt", "curse of agony"]}');
221+
222+
-- JSON Test Data Table
223+
-- This table contains various JSON structures for testing JSON operators and functions
224+
CREATE TABLE json_test_data (
225+
id SERIAL PRIMARY KEY,
226+
name VARCHAR(100) NOT NULL,
227+
data JSONB NOT NULL,
228+
metadata JSON,
229+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
230+
);
231+
232+
INSERT INTO json_test_data (name, data, metadata) VALUES
233+
-- Simple object
234+
('user_profile',
235+
'{"userId": 1, "username": "john_doe", "email": "john@example.com", "age": 30, "active": true}',
236+
'{"source": "api", "version": "1.0"}'),
237+
238+
-- Nested object with address
239+
('user_with_address',
240+
'{"userId": 2, "username": "jane_smith", "email": "jane@example.com", "address": {"street": "123 Main St", "city": "Springfield", "state": "IL", "zipCode": "62701", "country": "USA"}}',
241+
'{"source": "import", "version": "1.0"}'),
242+
243+
-- Array of items
244+
('shopping_cart',
245+
'{"cartId": 101, "items": [{"productId": 1, "name": "Laptop", "quantity": 1, "price": 999.99}, {"productId": 2, "name": "Mouse", "quantity": 2, "price": 25.50}], "totalPrice": 1050.99}',
246+
'{"source": "web", "version": "2.0"}'),
247+
248+
-- Array of strings
249+
('tags',
250+
'{"postId": 42, "title": "PostgreSQL JSON Functions", "tags": ["database", "postgresql", "json", "tutorial"], "published": true}',
251+
'{"source": "cms", "version": "1.0"}'),
252+
253+
-- Nested arrays and objects
254+
('game_stats',
255+
'{"playerId": 123, "stats": {"level": 50, "experience": 125000, "inventory": [{"slot": 1, "item": "Sword of Fire", "rarity": "legendary"}, {"slot": 2, "item": "Shield of Light", "rarity": "epic"}], "achievements": ["First Kill", "Level 50", "Legendary Item"]}}',
256+
'{"source": "game_server", "version": "3.0"}'),
257+
258+
-- Deep nesting
259+
('nested_config',
260+
'{"app": {"name": "MyApp", "version": "1.0.0", "settings": {"database": {"host": "localhost", "port": 5432, "credentials": {"username": "admin", "encrypted": true}}, "features": {"darkMode": true, "notifications": {"email": true, "push": false}}}}}',
261+
'{"source": "config", "version": "1.0"}'),
262+
263+
-- Array of objects with nulls
264+
('product_reviews',
265+
'{"productId": 456, "reviews": [{"reviewId": 1, "rating": 5, "comment": "Excellent product!", "reviewer": "Alice"}, {"reviewId": 2, "rating": 4, "comment": null, "reviewer": "Bob"}, {"reviewId": 3, "rating": 3, "comment": "Average", "reviewer": null}]}',
266+
'{"source": "reviews", "version": "1.0"}'),
267+
268+
-- Mixed types
269+
('analytics',
270+
'{"date": "2024-01-15", "metrics": {"visitors": 1500, "pageViews": 4500, "bounceRate": 0.35, "sources": {"organic": 850, "direct": 400, "referral": 250}}}',
271+
'{"source": "analytics", "version": "1.0"}');
Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
use crate::common::lazy::DB_SCHEMA;
2+
use crate::core::connection::DBConn;
3+
use crate::ts_generator::errors::TsGeneratorError;
4+
use crate::ts_generator::sql_parser::expressions::function_handlers::FunctionHandlersContext;
5+
use crate::ts_generator::sql_parser::expressions::translate_data_type::translate_value;
6+
use crate::ts_generator::sql_parser::expressions::translate_table_with_joins::translate_table_from_expr;
7+
use crate::ts_generator::sql_parser::quoted_strings::DisplayIndent;
8+
use crate::ts_generator::types::ts_query::TsFieldType;
9+
use sqlparser::ast::{Expr, FunctionArg, FunctionArgExpr, TableWithJoins, Value};
10+
11+
/// Extract key name from a function argument (should be a string literal)
12+
fn extract_key_name(arg: &FunctionArg) -> Option<String> {
13+
match arg {
14+
FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(val))) => match &val.value {
15+
Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => Some(s.clone()),
16+
_ => None,
17+
},
18+
_ => None,
19+
}
20+
}
21+
22+
/// Extract expression from a function argument
23+
fn extract_expr_from_arg(arg: &FunctionArg) -> Option<&Expr> {
24+
match arg {
25+
FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => Some(expr),
26+
FunctionArg::Named {
27+
arg: FunctionArgExpr::Expr(expr),
28+
..
29+
} => Some(expr),
30+
_ => None,
31+
}
32+
}
33+
34+
/// Infer the TypeScript type from an SQL expression
35+
pub async fn infer_type_from_expr(
36+
expr: &Expr,
37+
single_table_name: &Option<&str>,
38+
table_with_joins: &Option<Vec<TableWithJoins>>,
39+
db_conn: &DBConn,
40+
) -> Option<(TsFieldType, bool)> {
41+
match expr {
42+
Expr::Identifier(ident) => {
43+
let column_name = DisplayIndent(ident).to_string();
44+
if let Some(table_name) = single_table_name {
45+
let table_details = DB_SCHEMA.lock().await.fetch_table(&vec![table_name], db_conn).await;
46+
47+
if let Some(table_details) = table_details {
48+
if let Some(field) = table_details.get(&column_name) {
49+
Some((field.field_type.to_owned(), field.is_nullable))
50+
} else {
51+
Some((TsFieldType::Any, false))
52+
}
53+
} else {
54+
Some((TsFieldType::Any, false))
55+
}
56+
} else {
57+
Some((TsFieldType::Any, false))
58+
}
59+
}
60+
Expr::CompoundIdentifier(idents) if idents.len() == 2 => {
61+
let column_name = DisplayIndent(&idents[1]).to_string();
62+
if let Ok(table_name) = translate_table_from_expr(table_with_joins, expr) {
63+
let table_details = DB_SCHEMA
64+
.lock()
65+
.await
66+
.fetch_table(&vec![table_name.as_str()], db_conn)
67+
.await;
68+
69+
if let Some(table_details) = table_details {
70+
if let Some(field) = table_details.get(&column_name) {
71+
Some((field.field_type.to_owned(), field.is_nullable))
72+
} else {
73+
Some((TsFieldType::Any, false))
74+
}
75+
} else {
76+
Some((TsFieldType::Any, false))
77+
}
78+
} else {
79+
Some((TsFieldType::Any, false))
80+
}
81+
}
82+
Expr::Value(val) => {
83+
if let Some(ts_field_type) = translate_value(&val.value) {
84+
Some((ts_field_type, false))
85+
} else {
86+
Some((TsFieldType::Any, false))
87+
}
88+
}
89+
_ => Some((TsFieldType::Any, false)),
90+
}
91+
}
92+
93+
/// Process key-value pairs from JSON build object arguments
94+
pub async fn process_json_build_object_args(
95+
args: &[FunctionArg],
96+
single_table_name: &Option<&str>,
97+
table_with_joins: &Option<Vec<TableWithJoins>>,
98+
db_conn: &DBConn,
99+
) -> Option<Vec<(String, TsFieldType, bool)>> {
100+
if !args.len().is_multiple_of(2) {
101+
// Invalid number of arguments
102+
return None;
103+
}
104+
105+
let mut object_fields = vec![];
106+
107+
// Process key-value pairs
108+
for i in (0..args.len()).step_by(2) {
109+
let key_arg = &args[i];
110+
let value_arg = &args[i + 1];
111+
112+
// Extract key name
113+
let key_name = extract_key_name(key_arg)?;
114+
115+
// Extract value expression
116+
let value_expr = extract_expr_from_arg(value_arg)?;
117+
118+
// Infer value type
119+
let (value_type, is_nullable) =
120+
infer_type_from_expr(value_expr, single_table_name, table_with_joins, db_conn).await?;
121+
122+
object_fields.push((key_name, value_type, is_nullable));
123+
}
124+
125+
Some(object_fields)
126+
}
127+
128+
/// Handle JSON build functions (jsonb_build_object, json_build_object, etc.)
129+
pub async fn handle_json_build_function(
130+
function_name: &str,
131+
args: &[FunctionArg],
132+
ctx: &mut FunctionHandlersContext<'_>,
133+
) -> Result<(), TsGeneratorError> {
134+
let expr_log = ctx.expr_for_logging.unwrap_or("");
135+
136+
// Handle jsonb_build_object / json_build_object / json_object (MySQL)
137+
let func_upper = function_name.to_uppercase();
138+
if func_upper == "JSONB_BUILD_OBJECT" || func_upper == "JSON_BUILD_OBJECT" || func_upper == "JSON_OBJECT" {
139+
if let Some(object_fields) =
140+
process_json_build_object_args(args, ctx.single_table_name, ctx.table_with_joins, ctx.db_conn).await
141+
{
142+
let object_type = TsFieldType::StructuredObject(object_fields);
143+
return ctx
144+
.ts_query
145+
.insert_result(Some(ctx.alias), &[object_type], ctx.is_selection, false, expr_log);
146+
}
147+
}
148+
149+
// For other build functions or on failure, return Any
150+
ctx
151+
.ts_query
152+
.insert_result(Some(ctx.alias), &[TsFieldType::Any], ctx.is_selection, false, expr_log)
153+
}
154+
155+
/// Handle JSON aggregation functions (jsonb_agg, json_agg, etc.)
156+
pub async fn handle_json_agg_function(
157+
args: &[FunctionArg],
158+
ctx: &mut FunctionHandlersContext<'_>,
159+
) -> Result<(), TsGeneratorError> {
160+
use super::super::functions::is_json_build_function;
161+
use sqlparser::ast::FunctionArguments;
162+
163+
let expr_log = ctx.expr_for_logging.unwrap_or("");
164+
165+
// jsonb_agg typically takes a single expression
166+
if args.len() != 1 {
167+
return ctx
168+
.ts_query
169+
.insert_result(Some(ctx.alias), &[TsFieldType::Any], ctx.is_selection, false, expr_log);
170+
}
171+
172+
let arg_expr = extract_expr_from_arg(&args[0]);
173+
174+
// Check if the argument is a jsonb_build_object function
175+
if let Some(Expr::Function(inner_func)) = arg_expr {
176+
let inner_func_name = inner_func.name.to_string();
177+
if is_json_build_function(inner_func_name.as_str()) {
178+
// Extract arguments from the inner function
179+
let inner_args = match &inner_func.args {
180+
FunctionArguments::List(arg_list) => &arg_list.args,
181+
_ => {
182+
return ctx
183+
.ts_query
184+
.insert_result(Some(ctx.alias), &[TsFieldType::Any], ctx.is_selection, false, expr_log);
185+
}
186+
};
187+
188+
// Process the inner jsonb_build_object
189+
if let Some(object_fields) =
190+
process_json_build_object_args(inner_args, ctx.single_table_name, ctx.table_with_joins, ctx.db_conn).await
191+
{
192+
let object_type = TsFieldType::StructuredObject(object_fields);
193+
let array_type = TsFieldType::Array(Box::new(object_type));
194+
return ctx
195+
.ts_query
196+
.insert_result(Some(ctx.alias), &[array_type], ctx.is_selection, false, expr_log);
197+
}
198+
}
199+
}
200+
201+
// If we can't infer the type, return Any
202+
ctx
203+
.ts_query
204+
.insert_result(Some(ctx.alias), &[TsFieldType::Any], ctx.is_selection, false, expr_log)
205+
}

0 commit comments

Comments
 (0)