Skip to content

Commit 6ebe5cc

Browse files
fivetran-MichaelLeeMichael Lee
andauthored
fix(duckdb,postgres,redshift,snowflake)!: ARRAY_CAT null propagation (#6829)
* change property ARRAY_APPEND_PROPAGATES_NULLS to ARRAY_FUNCS_PROPAGATES_NULLS * add ARRAY_FUNCS_PROPAGATES_NULLS = True for redshift * implement NULL propagation handling for ARRAY_CAT * update ARRAY_FUNCS_PROPAGATES_NULL comment * handle variable length arguments * fix mypy issues * remove excessive comments --------- Co-authored-by: Michael Lee <michael.lee@michael.lee-FMF6J19R7N>
1 parent b3b36ba commit 6ebe5cc

12 files changed

Lines changed: 227 additions & 30 deletions

File tree

sqlglot/dialects/dialect.py

Lines changed: 86 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -701,8 +701,8 @@ class Dialect(metaclass=_Dialect):
701701
ARRAY_AGG_INCLUDES_NULLS: t.Optional[bool] = True
702702
"""Whether ArrayAgg needs to filter NULL values."""
703703

704-
ARRAY_APPEND_PROPAGATES_NULLS = False
705-
"""Whether ArrayAppend returns NULL when the input array is NULL."""
704+
ARRAY_FUNCS_PROPAGATES_NULLS = False
705+
"""Whether Array update functions return NULL when the input array is NULL."""
706706

707707
PROMOTE_TO_INFERRED_DATETIME_TYPE = False
708708
"""
@@ -1366,7 +1366,7 @@ def array_append_sql(
13661366
13671367
Returns:
13681368
A callable that generates SQL with appropriate NULL handling for the target dialect.
1369-
Dialects that propagate NULLs need to set `ARRAY_APPEND_PROPAGATES_NULLS` to True.
1369+
Dialects that propagate NULLs need to set `ARRAY_FUNCS_PROPAGATES_NULLS` to True.
13701370
"""
13711371

13721372
def _array_append_sql(self: Generator, expression: exp.ArrayAppend | exp.ArrayPrepend) -> str:
@@ -1376,7 +1376,7 @@ def _array_append_sql(self: Generator, expression: exp.ArrayAppend | exp.ArrayPr
13761376
func_sql = self.func(name, *args)
13771377

13781378
source_null_propagation = bool(expression.args.get("null_propagation"))
1379-
target_null_propagation = self.dialect.ARRAY_APPEND_PROPAGATES_NULLS
1379+
target_null_propagation = self.dialect.ARRAY_FUNCS_PROPAGATES_NULLS
13801380

13811381
# No transpilation needed when source and target have matching NULL semantics
13821382
if source_null_propagation == target_null_propagation:
@@ -1400,6 +1400,88 @@ def _array_append_sql(self: Generator, expression: exp.ArrayAppend | exp.ArrayPr
14001400
return _array_append_sql
14011401

14021402

1403+
def array_concat_sql(
1404+
name: str,
1405+
) -> t.Callable[[Generator, exp.ArrayConcat], str]:
1406+
"""
1407+
Transpile ARRAY_CONCAT/ARRAY_CAT between dialects with different NULL propagation semantics.
1408+
1409+
Some dialects (Redshift, Snowflake, Spark) return NULL when ANY input array is NULL.
1410+
Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation.
1411+
1412+
Args:
1413+
name: Target dialect's function name (e.g., "ARRAY_CAT", "ARRAY_CONCAT", "LIST_CONCAT")
1414+
1415+
Returns:
1416+
A callable that generates SQL with appropriate NULL handling for the target dialect.
1417+
Dialects that propagate NULLs need to set `ARRAY_FUNCS_PROPAGATES_NULLS` to True.
1418+
"""
1419+
1420+
def _build_func_call(self: Generator, func_name: str, args: t.Sequence[exp.Expression]) -> str:
1421+
"""Build ARRAY_CONCAT call from a list of arguments, handling variadic vs binary nesting."""
1422+
if self.ARRAY_CONCAT_IS_VAR_LEN:
1423+
return self.func(func_name, *args)
1424+
elif len(args) == 1:
1425+
# Single arg gets empty array to preserve semantics
1426+
return self.func(func_name, args[0], exp.Array(expressions=[]))
1427+
else:
1428+
# Snowflake/PostgreSQL/Redshift require binary nesting: ARRAY_CAT(a, ARRAY_CAT(b, c))
1429+
# Build right-deep tree recursively to avoid creating new ArrayConcat expressions
1430+
result = self.func(func_name, args[-2], args[-1])
1431+
for arg in reversed(args[:-2]):
1432+
result = f"{func_name}({self.sql(arg)}, {result})"
1433+
return result
1434+
1435+
def _array_concat_sql(self: Generator, expression: exp.ArrayConcat) -> str:
1436+
this = expression.this
1437+
exprs = expression.expressions
1438+
all_args = [this] + exprs
1439+
1440+
source_null_propagation = bool(expression.args.get("null_propagation"))
1441+
target_null_propagation = self.dialect.ARRAY_FUNCS_PROPAGATES_NULLS
1442+
1443+
# Skip wrapper when source and target have matching NULL semantics,
1444+
# or when the first argument is an array literal (which can never be NULL),
1445+
# or when it's a single-argument call (empty array is added, preserving NULL semantics)
1446+
if (
1447+
source_null_propagation == target_null_propagation
1448+
or isinstance(this, exp.Array)
1449+
or len(exprs) == 0
1450+
):
1451+
return _build_func_call(self, name, all_args)
1452+
1453+
# Case 1: Source propagates NULLs, target doesn't (Snowflake → DuckDB)
1454+
# Check if ANY argument is NULL and return NULL explicitly
1455+
if source_null_propagation:
1456+
# Build OR-chain: a IS NULL OR b IS NULL OR c IS NULL
1457+
null_checks: t.List[exp.Expression] = [
1458+
exp.Is(this=arg.copy(), expression=exp.Null()) for arg in all_args
1459+
]
1460+
combined_check: exp.Expression = reduce(
1461+
lambda a, b: exp.Or(this=a, expression=b), null_checks
1462+
)
1463+
1464+
func_sql = _build_func_call(self, name, all_args)
1465+
1466+
return self.sql(
1467+
exp.If(
1468+
this=combined_check,
1469+
true=exp.Null(),
1470+
false=func_sql,
1471+
)
1472+
)
1473+
1474+
# Case 2: Source doesn't propagate NULLs, target does (DuckDB → Snowflake)
1475+
# Wrap ALL arguments in COALESCE to convert NULL → empty array
1476+
wrapped_args = [
1477+
exp.Coalesce(expressions=[arg.copy(), exp.Array(expressions=[])]) for arg in all_args
1478+
]
1479+
1480+
return _build_func_call(self, name, wrapped_args)
1481+
1482+
return _array_concat_sql
1483+
1484+
14031485
def var_map_sql(
14041486
self: Generator, expression: exp.Map | exp.VarMap, map_func_name: str = "MAP"
14051487
) -> str:

sqlglot/dialects/duckdb.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
NormalizationStrategy,
1515
approx_count_distinct_sql,
1616
array_append_sql,
17+
array_concat_sql,
1718
arrow_json_extract_sql,
1819
binary_from_function,
1920
bool_xor_sql,
@@ -1240,6 +1241,7 @@ class Parser(parser.Parser):
12401241
"JSON_EXTRACT_PATH": parser.build_extract_json_with_path(exp.JSONExtract),
12411242
"JSON_EXTRACT_STRING": parser.build_extract_json_with_path(exp.JSONExtractScalar),
12421243
"LIST_APPEND": exp.ArrayAppend.from_arg_list,
1244+
"LIST_CONCAT": parser.build_array_concat,
12431245
"LIST_CONTAINS": exp.ArrayContains.from_arg_list,
12441246
"LIST_COSINE_DISTANCE": exp.CosineDistance.from_arg_list,
12451247
"LIST_DISTANCE": exp.EuclideanDistance.from_arg_list,
@@ -1510,7 +1512,6 @@ class Generator(generator.Generator):
15101512
COPY_HAS_INTO_KEYWORD = False
15111513
STAR_EXCEPT = "EXCLUDE"
15121514
PAD_FILL_PATTERN_IS_REQUIRED = True
1513-
ARRAY_CONCAT_IS_VAR_LEN = False
15141515
ARRAY_SIZE_DIM_REQUIRED = False
15151516
NORMALIZE_EXTRACT_DATE_PARTS = True
15161517
SUPPORTS_LIKE_QUANTIFIERS = False
@@ -1526,6 +1527,7 @@ class Generator(generator.Generator):
15261527
generator=inline_array_unless_query,
15271528
),
15281529
exp.ArrayAppend: array_append_sql("LIST_APPEND"),
1530+
exp.ArrayConcat: array_concat_sql("LIST_CONCAT"),
15291531
exp.ArrayFilter: rename_func("LIST_FILTER"),
15301532
exp.ArrayRemove: remove_from_array_using_filter,
15311533
exp.ArraySort: _array_sort_sql,

sqlglot/dialects/postgres.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
JSON_EXTRACT_TYPE,
1010
any_value_to_max_sql,
1111
array_append_sql,
12+
array_concat_sql,
1213
binary_from_function,
1314
bool_xor_sql,
1415
datestrtodate_sql,
@@ -632,7 +633,7 @@ class Generator(generator.Generator):
632633
TRANSFORMS = {
633634
**generator.Generator.TRANSFORMS,
634635
exp.AnyValue: _versioned_anyvalue_sql,
635-
exp.ArrayConcat: lambda self, e: self.arrayconcat_sql(e, name="ARRAY_CAT"),
636+
exp.ArrayConcat: array_concat_sql("ARRAY_CAT"),
636637
exp.ArrayFilter: filter_array_using_unnest,
637638
exp.ArrayAppend: array_append_sql("ARRAY_APPEND"),
638639
exp.ArrayPrepend: array_append_sql("ARRAY_PREPEND", swap_params=True),

sqlglot/dialects/redshift.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from sqlglot import exp, transforms
66
from sqlglot.dialects.dialect import (
77
NormalizationStrategy,
8+
array_concat_sql,
89
concat_to_dpipe_sql,
910
concat_ws_to_dpipe_sql,
1011
date_delta_sql,
@@ -49,6 +50,7 @@ class Redshift(Postgres):
4950
HAS_DISTINCT_ARRAY_CONSTRUCTORS = True
5051
COALESCE_COMPARISON_NON_STANDARD = True
5152
REGEXP_EXTRACT_POSITION_OVERFLOW_RETURNS_NULL = False
53+
ARRAY_FUNCS_PROPAGATES_NULLS = True
5254

5355
# ref: https://docs.aws.amazon.com/redshift/latest/dg/r_FORMAT_strings.html
5456
TIME_FORMAT = "'YYYY-MM-DD HH24:MI:SS'"
@@ -190,7 +192,7 @@ class Generator(Postgres.Generator):
190192

191193
TRANSFORMS = {
192194
**Postgres.Generator.TRANSFORMS,
193-
exp.ArrayConcat: lambda self, e: self.arrayconcat_sql(e, name="ARRAY_CONCAT"),
195+
exp.ArrayConcat: array_concat_sql("ARRAY_CONCAT"),
194196
exp.Concat: concat_to_dpipe_sql,
195197
exp.ConcatWs: concat_ws_to_dpipe_sql,
196198
exp.ApproxDistinct: lambda self,

sqlglot/dialects/snowflake.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
Dialect,
88
NormalizationStrategy,
99
array_append_sql,
10+
array_concat_sql,
1011
build_timetostr_or_tochar,
1112
build_like,
1213
binary_from_function,
@@ -679,7 +680,7 @@ class Snowflake(Dialect):
679680
TABLESAMPLE_SIZE_IS_PERCENT = True
680681
COPY_PARAMS_ARE_CSV = False
681682
ARRAY_AGG_INCLUDES_NULLS = None
682-
ARRAY_APPEND_PROPAGATES_NULLS = True
683+
ARRAY_FUNCS_PROPAGATES_NULLS = True
683684
ALTER_TABLE_ADD_REQUIRED_FOR_EACH_COLUMN = False
684685
TRY_CAST_REQUIRES_STRING = True
685686
SUPPORTS_ALIAS_REFS_IN_JOIN_CONDITIONS = True
@@ -1564,7 +1565,7 @@ class Generator(generator.Generator):
15641565
exp.ArgMax: rename_func("MAX_BY"),
15651566
exp.ArgMin: rename_func("MIN_BY"),
15661567
exp.Array: transforms.preprocess([transforms.inherit_struct_field_names]),
1567-
exp.ArrayConcat: lambda self, e: self.arrayconcat_sql(e, name="ARRAY_CAT"),
1568+
exp.ArrayConcat: array_concat_sql("ARRAY_CAT"),
15681569
exp.ArrayAppend: array_append_sql("ARRAY_APPEND"),
15691570
exp.ArrayPrepend: array_append_sql("ARRAY_PREPEND"),
15701571
exp.ArrayContains: lambda self, e: self.func(

sqlglot/dialects/spark.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ def _groupconcat_sql(self: Spark.Generator, expression: exp.GroupConcat) -> str:
114114
class Spark(Spark2):
115115
SUPPORTS_ORDER_BY_ALL = True
116116
SUPPORTS_NULL_TYPE = True
117-
ARRAY_APPEND_PROPAGATES_NULLS = True
117+
ARRAY_FUNCS_PROPAGATES_NULLS = True
118118
EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
119119

120120
class Tokenizer(Spark2.Tokenizer):

sqlglot/expressions.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6094,7 +6094,7 @@ class ArrayPrepend(Func):
60946094

60956095
class ArrayConcat(Func):
60966096
_sql_names = ["ARRAY_CONCAT", "ARRAY_CAT"]
6097-
arg_types = {"this": True, "expressions": False}
6097+
arg_types = {"this": True, "expressions": False, "null_propagation": False}
60986098
is_var_len_args = True
60996099

61006100

sqlglot/generator.py

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4911,18 +4911,6 @@ def explodinggenerateseries_sql(self, expression: exp.ExplodingGenerateSeries) -
49114911

49124912
return self.sql(generate_series)
49134913

4914-
def arrayconcat_sql(self, expression: exp.ArrayConcat, name: str = "ARRAY_CONCAT") -> str:
4915-
exprs = expression.expressions
4916-
if not self.ARRAY_CONCAT_IS_VAR_LEN:
4917-
if len(exprs) == 0:
4918-
rhs: t.Union[str, exp.Expression] = exp.Array(expressions=[])
4919-
else:
4920-
rhs = reduce(lambda x, y: exp.ArrayConcat(this=x, expressions=[y]), exprs)
4921-
else:
4922-
rhs = self.expressions(expression) # type: ignore
4923-
4924-
return self.func(name, expression.this, rhs or None)
4925-
49264914
def converttimezone_sql(self, expression: exp.ConvertTimezone) -> str:
49274915
if self.SUPPORTS_CONVERT_TIMEZONE:
49284916
return self.function_fallback_sql(expression)

sqlglot/parser.py

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -187,15 +187,15 @@ def build_array_append(args: t.List, dialect: Dialect) -> exp.ArrayAppend:
187187
188188
Args:
189189
args: Function arguments [array, element]
190-
dialect: The dialect to read ARRAY_APPEND_PROPAGATES_NULLS from
190+
dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
191191
192192
Returns:
193193
ArrayAppend expression with appropriate null_propagation flag
194194
"""
195195
return exp.ArrayAppend(
196196
this=seq_get(args, 0),
197197
expression=seq_get(args, 1),
198-
null_propagation=dialect.ARRAY_APPEND_PROPAGATES_NULLS,
198+
null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS,
199199
)
200200

201201

@@ -208,15 +208,36 @@ def build_array_prepend(args: t.List, dialect: Dialect) -> exp.ArrayPrepend:
208208
209209
Args:
210210
args: Function arguments [array, element]
211-
dialect: The dialect to read ARRAY_APPEND_PROPAGATES_NULLS from
211+
dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
212212
213213
Returns:
214214
ArrayPrepend expression with appropriate null_propagation flag
215215
"""
216216
return exp.ArrayPrepend(
217217
this=seq_get(args, 0),
218218
expression=seq_get(args, 1),
219-
null_propagation=dialect.ARRAY_APPEND_PROPAGATES_NULLS,
219+
null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS,
220+
)
221+
222+
223+
def build_array_concat(args: t.List, dialect: Dialect) -> exp.ArrayConcat:
224+
"""
225+
Builds ArrayConcat with NULL propagation semantics based on the dialect configuration.
226+
227+
Some dialects (Redshift, Snowflake) return NULL when any input array is NULL.
228+
Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation.
229+
230+
Args:
231+
args: Function arguments [array1, array2, ...] (variadic)
232+
dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
233+
234+
Returns:
235+
ArrayConcat expression with appropriate null_propagation flag
236+
"""
237+
return exp.ArrayConcat(
238+
this=seq_get(args, 0),
239+
expressions=args[1:],
240+
null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS,
220241
)
221242

222243

@@ -256,6 +277,8 @@ class Parser(metaclass=_Parser):
256277
this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None
257278
),
258279
"ARRAY_APPEND": build_array_append,
280+
"ARRAY_CAT": build_array_concat,
281+
"ARRAY_CONCAT": build_array_concat,
259282
"ARRAY_PREPEND": build_array_prepend,
260283
"COUNT": lambda args: exp.Count(this=seq_get(args, 0), expressions=args[1:], big_int=True),
261284
"CONCAT": lambda args, dialect: exp.Concat(

tests/dialects/test_bigquery.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1685,7 +1685,7 @@ def test_bigquery(self):
16851685
"ARRAY_CONCAT([1, 2], [3, 4], [5, 6])",
16861686
write={
16871687
"bigquery": "ARRAY_CONCAT([1, 2], [3, 4], [5, 6])",
1688-
"duckdb": "ARRAY_CONCAT([1, 2], ARRAY_CONCAT([3, 4], [5, 6]))",
1688+
"duckdb": "LIST_CONCAT([1, 2], [3, 4], [5, 6])",
16891689
"postgres": "ARRAY_CAT(ARRAY[1, 2], ARRAY_CAT(ARRAY[3, 4], ARRAY[5, 6]))",
16901690
"redshift": "ARRAY_CONCAT(ARRAY(1, 2), ARRAY_CONCAT(ARRAY(3, 4), ARRAY(5, 6)))",
16911691
"snowflake": "ARRAY_CAT([1, 2], ARRAY_CAT([3, 4], [5, 6]))",

0 commit comments

Comments
 (0)