-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Expand file tree
/
Copy pathdialect.py
More file actions
2368 lines (1873 loc) · 80 KB
/
Copy pathdialect.py
File metadata and controls
2368 lines (1873 loc) · 80 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Contains code from https://github.com/tobymao/sqlglot/blob/v28.5.0/sqlglot/dialects/dialect.py
from __future__ import annotations
import importlib
import logging
import sys
import typing as t
from enum import Enum, auto
from functools import reduce
from bigframes_vendored.sqlglot import exp
from bigframes_vendored.sqlglot.dialects import DIALECT_MODULE_NAMES
from bigframes_vendored.sqlglot.errors import ParseError
from bigframes_vendored.sqlglot.generator import Generator, unsupported_args
from bigframes_vendored.sqlglot.helper import (
AutoName,
flatten,
is_int,
seq_get,
suggest_closest_match_and_fail,
to_bool,
)
from bigframes_vendored.sqlglot.jsonpath import JSONPathTokenizer
from bigframes_vendored.sqlglot.jsonpath import parse as parse_json_path
from bigframes_vendored.sqlglot.parser import Parser
from bigframes_vendored.sqlglot.time import TIMEZONES, format_time, subsecond_precision
from bigframes_vendored.sqlglot.tokens import Token, Tokenizer, TokenType
from bigframes_vendored.sqlglot.trie import new_trie
from bigframes_vendored.sqlglot.typing import EXPRESSION_METADATA
DATE_ADD_OR_DIFF = t.Union[
exp.DateAdd,
exp.DateDiff,
exp.DateSub,
exp.TsOrDsAdd,
exp.TsOrDsDiff,
]
DATE_ADD_OR_SUB = t.Union[exp.DateAdd, exp.TsOrDsAdd, exp.DateSub]
JSON_EXTRACT_TYPE = t.Union[
exp.JSONExtract, exp.JSONExtractScalar, exp.JSONBExtract, exp.JSONBExtractScalar
]
DATETIME_DELTA = t.Union[
exp.DateAdd,
exp.DatetimeAdd,
exp.DatetimeSub,
exp.TimeAdd,
exp.TimeSub,
exp.TimestampAdd,
exp.TimestampSub,
exp.TsOrDsAdd,
]
DATETIME_ADD = (
exp.DateAdd,
exp.TimeAdd,
exp.DatetimeAdd,
exp.TsOrDsAdd,
exp.TimestampAdd,
)
if t.TYPE_CHECKING:
from sqlglot._typing import B, E, F
logger = logging.getLogger("sqlglot")
UNESCAPED_SEQUENCES = {
"\\a": "\a",
"\\b": "\b",
"\\f": "\f",
"\\n": "\n",
"\\r": "\r",
"\\t": "\t",
"\\v": "\v",
"\\\\": "\\",
}
class Dialects(str, Enum):
"""Dialects supported by SQLGLot."""
DIALECT = ""
ATHENA = "athena"
BIGQUERY = "bigquery"
CLICKHOUSE = "clickhouse"
DATABRICKS = "databricks"
DORIS = "doris"
DREMIO = "dremio"
DRILL = "drill"
DRUID = "druid"
DUCKDB = "duckdb"
DUNE = "dune"
FABRIC = "fabric"
HIVE = "hive"
MATERIALIZE = "materialize"
MYSQL = "mysql"
ORACLE = "oracle"
POSTGRES = "postgres"
PRESTO = "presto"
PRQL = "prql"
REDSHIFT = "redshift"
RISINGWAVE = "risingwave"
SNOWFLAKE = "snowflake"
SOLR = "solr"
SPARK = "spark"
SPARK2 = "spark2"
SQLITE = "sqlite"
STARROCKS = "starrocks"
TABLEAU = "tableau"
TERADATA = "teradata"
TRINO = "trino"
TSQL = "tsql"
EXASOL = "exasol"
class NormalizationStrategy(str, AutoName):
"""Specifies the strategy according to which identifiers should be normalized."""
LOWERCASE = auto()
"""Unquoted identifiers are lowercased."""
UPPERCASE = auto()
"""Unquoted identifiers are uppercased."""
CASE_SENSITIVE = auto()
"""Always case-sensitive, regardless of quotes."""
CASE_INSENSITIVE = auto()
"""Always case-insensitive (lowercase), regardless of quotes."""
CASE_INSENSITIVE_UPPERCASE = auto()
"""Always case-insensitive (uppercase), regardless of quotes."""
class _Dialect(type):
_classes: t.Dict[str, t.Type[Dialect]] = {}
def __eq__(cls, other: t.Any) -> bool:
if cls is other:
return True
if isinstance(other, str):
return cls is cls.get(other)
if isinstance(other, Dialect):
return cls is type(other)
return False
def __hash__(cls) -> int:
return hash(cls.__name__.lower())
@property
def classes(cls):
if len(DIALECT_MODULE_NAMES) != len(cls._classes):
for key in DIALECT_MODULE_NAMES:
cls._try_load(key)
return cls._classes
@classmethod
def _try_load(cls, key: str | Dialects) -> None:
if isinstance(key, Dialects):
key = key.value
# This import will lead to a new dialect being loaded, and hence, registered.
# We check that the key is an actual sqlglot module to avoid blindly importing
# files. Custom user dialects need to be imported at the top-level package, in
# order for them to be registered as soon as possible.
if key in DIALECT_MODULE_NAMES:
importlib.import_module(f"bigframes_vendored.sqlglot.dialects.{key}")
@classmethod
def __getitem__(cls, key: str) -> t.Type[Dialect]:
if key not in cls._classes:
cls._try_load(key)
return cls._classes[key]
@classmethod
def get(
cls, key: str, default: t.Optional[t.Type[Dialect]] = None
) -> t.Optional[t.Type[Dialect]]:
if key not in cls._classes:
cls._try_load(key)
return cls._classes.get(key, default)
def __new__(cls, clsname, bases, attrs):
klass = super().__new__(cls, clsname, bases, attrs)
enum = Dialects.__members__.get(clsname.upper())
cls._classes[enum.value if enum is not None else clsname.lower()] = klass
klass.TIME_TRIE = new_trie(klass.TIME_MAPPING)
klass.FORMAT_TRIE = (
new_trie(klass.FORMAT_MAPPING) if klass.FORMAT_MAPPING else klass.TIME_TRIE
)
# Merge class-defined INVERSE_TIME_MAPPING with auto-generated mappings
# This allows dialects to define custom inverse mappings for roundtrip correctness
klass.INVERSE_TIME_MAPPING = {v: k for k, v in klass.TIME_MAPPING.items()} | (
klass.__dict__.get("INVERSE_TIME_MAPPING") or {}
)
klass.INVERSE_TIME_TRIE = new_trie(klass.INVERSE_TIME_MAPPING)
klass.INVERSE_FORMAT_MAPPING = {v: k for k, v in klass.FORMAT_MAPPING.items()}
klass.INVERSE_FORMAT_TRIE = new_trie(klass.INVERSE_FORMAT_MAPPING)
klass.INVERSE_CREATABLE_KIND_MAPPING = {
v: k for k, v in klass.CREATABLE_KIND_MAPPING.items()
}
base = seq_get(bases, 0)
base_tokenizer = (getattr(base, "tokenizer_class", Tokenizer),)
base_jsonpath_tokenizer = (
getattr(base, "jsonpath_tokenizer_class", JSONPathTokenizer),
)
base_parser = (getattr(base, "parser_class", Parser),)
base_generator = (getattr(base, "generator_class", Generator),)
klass.tokenizer_class = klass.__dict__.get(
"Tokenizer", type("Tokenizer", base_tokenizer, {})
)
klass.jsonpath_tokenizer_class = klass.__dict__.get(
"JSONPathTokenizer", type("JSONPathTokenizer", base_jsonpath_tokenizer, {})
)
klass.parser_class = klass.__dict__.get(
"Parser", type("Parser", base_parser, {})
)
klass.generator_class = klass.__dict__.get(
"Generator", type("Generator", base_generator, {})
)
klass.QUOTE_START, klass.QUOTE_END = list(
klass.tokenizer_class._QUOTES.items()
)[0]
klass.IDENTIFIER_START, klass.IDENTIFIER_END = list(
klass.tokenizer_class._IDENTIFIERS.items()
)[0]
def get_start_end(
token_type: TokenType,
) -> t.Tuple[t.Optional[str], t.Optional[str]]:
return next(
(
(s, e)
for s, (e, t) in klass.tokenizer_class._FORMAT_STRINGS.items()
if t == token_type
),
(None, None),
)
klass.BIT_START, klass.BIT_END = get_start_end(TokenType.BIT_STRING)
klass.HEX_START, klass.HEX_END = get_start_end(TokenType.HEX_STRING)
klass.BYTE_START, klass.BYTE_END = get_start_end(TokenType.BYTE_STRING)
klass.UNICODE_START, klass.UNICODE_END = get_start_end(TokenType.UNICODE_STRING)
if "\\" in klass.tokenizer_class.STRING_ESCAPES:
klass.UNESCAPED_SEQUENCES = {
**UNESCAPED_SEQUENCES,
**klass.UNESCAPED_SEQUENCES,
}
klass.ESCAPED_SEQUENCES = {v: k for k, v in klass.UNESCAPED_SEQUENCES.items()}
klass.SUPPORTS_COLUMN_JOIN_MARKS = "(+)" in klass.tokenizer_class.KEYWORDS
if enum not in ("", "bigquery", "snowflake"):
klass.INITCAP_SUPPORTS_CUSTOM_DELIMITERS = False
if enum not in ("", "bigquery"):
klass.generator_class.SELECT_KINDS = ()
if enum not in ("", "athena", "presto", "trino", "duckdb"):
klass.generator_class.TRY_SUPPORTED = False
klass.generator_class.SUPPORTS_UESCAPE = False
if enum not in ("", "databricks", "hive", "spark", "spark2"):
modifier_transforms = (
klass.generator_class.AFTER_HAVING_MODIFIER_TRANSFORMS.copy()
)
for modifier in ("cluster", "distribute", "sort"):
modifier_transforms.pop(modifier, None)
klass.generator_class.AFTER_HAVING_MODIFIER_TRANSFORMS = modifier_transforms
if enum not in ("", "doris", "mysql"):
klass.parser_class.ID_VAR_TOKENS = klass.parser_class.ID_VAR_TOKENS | {
TokenType.STRAIGHT_JOIN,
}
klass.parser_class.TABLE_ALIAS_TOKENS = (
klass.parser_class.TABLE_ALIAS_TOKENS
| {
TokenType.STRAIGHT_JOIN,
}
)
if enum not in ("", "databricks", "oracle", "redshift", "snowflake", "spark"):
klass.generator_class.SUPPORTS_DECODE_CASE = False
if not klass.SUPPORTS_SEMI_ANTI_JOIN:
klass.parser_class.TABLE_ALIAS_TOKENS = (
klass.parser_class.TABLE_ALIAS_TOKENS
| {
TokenType.ANTI,
TokenType.SEMI,
}
)
if enum not in (
"",
"postgres",
"duckdb",
"redshift",
"snowflake",
"presto",
"trino",
"mysql",
"singlestore",
):
no_paren_functions = klass.parser_class.NO_PAREN_FUNCTIONS.copy()
no_paren_functions.pop(TokenType.LOCALTIME, None)
if enum != "oracle":
no_paren_functions.pop(TokenType.LOCALTIMESTAMP, None)
klass.parser_class.NO_PAREN_FUNCTIONS = no_paren_functions
if enum in (
"",
"postgres",
"duckdb",
"trino",
):
no_paren_functions = klass.parser_class.NO_PAREN_FUNCTIONS.copy()
no_paren_functions[TokenType.CURRENT_CATALOG] = exp.CurrentCatalog
klass.parser_class.NO_PAREN_FUNCTIONS = no_paren_functions
else:
# For dialects that don't support this keyword, treat it as a regular identifier
# This fixes the "Unexpected token" error in BQ, Spark, etc.
klass.parser_class.ID_VAR_TOKENS = klass.parser_class.ID_VAR_TOKENS | {
TokenType.CURRENT_CATALOG,
}
if enum in (
"",
"duckdb",
"spark",
"postgres",
"tsql",
):
no_paren_functions = klass.parser_class.NO_PAREN_FUNCTIONS.copy()
no_paren_functions[TokenType.SESSION_USER] = exp.SessionUser
klass.parser_class.NO_PAREN_FUNCTIONS = no_paren_functions
else:
klass.parser_class.ID_VAR_TOKENS = klass.parser_class.ID_VAR_TOKENS | {
TokenType.SESSION_USER,
}
klass.VALID_INTERVAL_UNITS = {
*klass.VALID_INTERVAL_UNITS,
*klass.DATE_PART_MAPPING.keys(),
*klass.DATE_PART_MAPPING.values(),
}
return klass
class Dialect(metaclass=_Dialect):
INDEX_OFFSET = 0
"""The base index offset for arrays."""
WEEK_OFFSET = 0
"""First day of the week in DATE_TRUNC(week). Defaults to 0 (Monday). -1 would be Sunday."""
UNNEST_COLUMN_ONLY = False
"""Whether `UNNEST` table aliases are treated as column aliases."""
ALIAS_POST_TABLESAMPLE = False
"""Whether the table alias comes after tablesample."""
TABLESAMPLE_SIZE_IS_PERCENT = False
"""Whether a size in the table sample clause represents percentage."""
NORMALIZATION_STRATEGY = NormalizationStrategy.LOWERCASE
"""Specifies the strategy according to which identifiers should be normalized."""
IDENTIFIERS_CAN_START_WITH_DIGIT = False
"""Whether an unquoted identifier can start with a digit."""
DPIPE_IS_STRING_CONCAT = True
"""Whether the DPIPE token (`||`) is a string concatenation operator."""
STRICT_STRING_CONCAT = False
"""Whether `CONCAT`'s arguments must be strings."""
SUPPORTS_USER_DEFINED_TYPES = True
"""Whether user-defined data types are supported."""
SUPPORTS_SEMI_ANTI_JOIN = True
"""Whether `SEMI` or `ANTI` joins are supported."""
SUPPORTS_COLUMN_JOIN_MARKS = False
"""Whether the old-style outer join (+) syntax is supported."""
COPY_PARAMS_ARE_CSV = True
"""Separator of COPY statement parameters."""
NORMALIZE_FUNCTIONS: bool | str = "upper"
"""
Determines how function names are going to be normalized.
Possible values:
"upper" or True: Convert names to uppercase.
"lower": Convert names to lowercase.
False: Disables function name normalization.
"""
PRESERVE_ORIGINAL_NAMES: bool = False
"""
Whether the name of the function should be preserved inside the node's metadata,
can be useful for roundtripping deprecated vs new functions that share an AST node
e.g JSON_VALUE vs JSON_EXTRACT_SCALAR in BigQuery
"""
LOG_BASE_FIRST: t.Optional[bool] = True
"""
Whether the base comes first in the `LOG` function.
Possible values: `True`, `False`, `None` (two arguments are not supported by `LOG`)
"""
NULL_ORDERING = "nulls_are_small"
"""
Default `NULL` ordering method to use if not explicitly set.
Possible values: `"nulls_are_small"`, `"nulls_are_large"`, `"nulls_are_last"`
"""
TYPED_DIVISION = False
"""
Whether the behavior of `a / b` depends on the types of `a` and `b`.
False means `a / b` is always float division.
True means `a / b` is integer division if both `a` and `b` are integers.
"""
SAFE_DIVISION = False
"""Whether division by zero throws an error (`False`) or returns NULL (`True`)."""
CONCAT_COALESCE = False
"""A `NULL` arg in `CONCAT` yields `NULL` by default, but in some dialects it yields an empty string."""
HEX_LOWERCASE = False
"""Whether the `HEX` function returns a lowercase hexadecimal string."""
DATE_FORMAT = "'%Y-%m-%d'"
DATEINT_FORMAT = "'%Y%m%d'"
TIME_FORMAT = "'%Y-%m-%d %H:%M:%S'"
TIME_MAPPING: t.Dict[str, str] = {}
"""Associates this dialect's time formats with their equivalent Python `strftime` formats."""
# https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_model_rules_date_time
# https://docs.teradata.com/r/Teradata-Database-SQL-Functions-Operators-Expressions-and-Predicates/March-2017/Data-Type-Conversions/Character-to-DATE-Conversion/Forcing-a-FORMAT-on-CAST-for-Converting-Character-to-DATE
FORMAT_MAPPING: t.Dict[str, str] = {}
"""
Helper which is used for parsing the special syntax `CAST(x AS DATE FORMAT 'yyyy')`.
If empty, the corresponding trie will be constructed off of `TIME_MAPPING`.
"""
UNESCAPED_SEQUENCES: t.Dict[str, str] = {}
"""Mapping of an escaped sequence (`\\n`) to its unescaped version (`\n`)."""
PSEUDOCOLUMNS: t.Set[str] = set()
"""
Columns that are auto-generated by the engine corresponding to this dialect.
For example, such columns may be excluded from `SELECT *` queries.
"""
PREFER_CTE_ALIAS_COLUMN = False
"""
Some dialects, such as Snowflake, allow you to reference a CTE column alias in the
HAVING clause of the CTE. This flag will cause the CTE alias columns to override
any projection aliases in the subquery.
For example,
WITH y(c) AS (
SELECT SUM(a) FROM (SELECT 1 a) AS x HAVING c > 0
) SELECT c FROM y;
will be rewritten as
WITH y(c) AS (
SELECT SUM(a) AS c FROM (SELECT 1 AS a) AS x HAVING c > 0
) SELECT c FROM y;
"""
COPY_PARAMS_ARE_CSV = True
"""
Whether COPY statement parameters are separated by comma or whitespace
"""
FORCE_EARLY_ALIAS_REF_EXPANSION = False
"""
Whether alias reference expansion (_expand_alias_refs()) should run before column qualification (_qualify_columns()).
For example:
WITH data AS (
SELECT
1 AS id,
2 AS my_id
)
SELECT
id AS my_id
FROM
data
WHERE
my_id = 1
GROUP BY
my_id,
HAVING
my_id = 1
In most dialects, "my_id" would refer to "data.my_id" across the query, except:
- BigQuery, which will forward the alias to GROUP BY + HAVING clauses i.e
it resolves to "WHERE my_id = 1 GROUP BY id HAVING id = 1"
- Clickhouse, which will forward the alias across the query i.e it resolves
to "WHERE id = 1 GROUP BY id HAVING id = 1"
"""
EXPAND_ONLY_GROUP_ALIAS_REF = False
"""Whether alias reference expansion before qualification should only happen for the GROUP BY clause."""
ANNOTATE_ALL_SCOPES = False
"""Whether to annotate all scopes during optimization. Used by BigQuery for UNNEST support."""
DISABLES_ALIAS_REF_EXPANSION = False
"""
Whether alias reference expansion is disabled for this dialect.
Some dialects like Oracle do NOT support referencing aliases in projections or WHERE clauses.
The original expression must be repeated instead.
For example, in Oracle:
SELECT y.foo AS bar, bar * 2 AS baz FROM y -- INVALID
SELECT y.foo AS bar, y.foo * 2 AS baz FROM y -- VALID
"""
SUPPORTS_ALIAS_REFS_IN_JOIN_CONDITIONS = False
"""
Whether alias references are allowed in JOIN ... ON clauses.
Most dialects do not support this, but Snowflake allows alias expansion in the JOIN ... ON
clause (and almost everywhere else)
For example, in Snowflake:
SELECT a.id AS user_id FROM a JOIN b ON user_id = b.id -- VALID
Reference: https://docs.snowflake.com/en/sql-reference/sql/select#usage-notes
"""
SUPPORTS_ORDER_BY_ALL = False
"""
Whether ORDER BY ALL is supported (expands to all the selected columns) as in DuckDB, Spark3/Databricks
"""
PROJECTION_ALIASES_SHADOW_SOURCE_NAMES = False
"""
Whether projection alias names can shadow table/source names in GROUP BY and HAVING clauses.
In BigQuery, when a projection alias has the same name as a source table, the alias takes
precedence in GROUP BY and HAVING clauses, and the table becomes inaccessible by that name.
For example, in BigQuery:
SELECT id, ARRAY_AGG(col) AS custom_fields
FROM custom_fields
GROUP BY id
HAVING id >= 1
The "custom_fields" source is shadowed by the projection alias, so we cannot qualify "id"
with "custom_fields" in GROUP BY/HAVING.
"""
TABLES_REFERENCEABLE_AS_COLUMNS = False
"""
Whether table names can be referenced as columns (treated as structs).
BigQuery allows tables to be referenced as columns in queries, automatically treating
them as struct values containing all the table's columns.
For example, in BigQuery:
SELECT t FROM my_table AS t -- Returns entire row as a struct
"""
SUPPORTS_STRUCT_STAR_EXPANSION = False
"""
Whether the dialect supports expanding struct fields using star notation (e.g., struct_col.*).
BigQuery allows struct fields to be expanded with the star operator:
SELECT t.struct_col.* FROM table t
RisingWave also allows struct field expansion with the star operator using parentheses:
SELECT (t.struct_col).* FROM table t
This expands to all fields within the struct.
"""
EXCLUDES_PSEUDOCOLUMNS_FROM_STAR = False
"""
Whether pseudocolumns should be excluded from star expansion (SELECT *).
Pseudocolumns are special dialect-specific columns (e.g., Oracle's ROWNUM, ROWID, LEVEL,
or BigQuery's _PARTITIONTIME, _PARTITIONDATE) that are implicitly available but not part
of the table schema. When this is True, SELECT * will not include these pseudocolumns;
they must be explicitly selected.
"""
QUERY_RESULTS_ARE_STRUCTS = False
"""
Whether query results are typed as structs in metadata for type inference.
In BigQuery, subqueries store their column types as a STRUCT in metadata,
enabling special type inference for ARRAY(SELECT ...) expressions:
ARRAY(SELECT x, y FROM t) → ARRAY<STRUCT<...>>
For single column subqueries, BigQuery unwraps the struct:
ARRAY(SELECT x FROM t) → ARRAY<type_of_x>
This is metadata-only for type inference.
"""
REQUIRES_PARENTHESIZED_STRUCT_ACCESS = False
"""
Whether struct field access requires parentheses around the expression.
RisingWave requires parentheses for struct field access in certain contexts:
SELECT (col.field).subfield FROM table -- Parentheses required
Without parentheses, the parser may not correctly interpret nested struct access.
Reference: https://docs.risingwave.com/sql/data-types/struct#retrieve-data-in-a-struct
"""
SUPPORTS_NULL_TYPE = False
"""
Whether NULL/VOID is supported as a valid data type (not just a value).
Databricks and Spark v3+ support NULL as an actual type, allowing expressions like:
SELECT NULL AS col -- Has type NULL, not just value NULL
CAST(x AS VOID) -- Valid type cast
"""
COALESCE_COMPARISON_NON_STANDARD = False
"""
Whether COALESCE in comparisons has non-standard NULL semantics.
We can't convert `COALESCE(x, 1) = 2` into `NOT x IS NULL AND x = 2` for redshift,
because they are not always equivalent. For example, if `x` is `NULL` and it comes
from a table, then the result is `NULL`, despite `FALSE AND NULL` evaluating to `FALSE`.
In standard SQL and most dialects, these expressions are equivalent, but Redshift treats
table NULLs differently in this context.
"""
HAS_DISTINCT_ARRAY_CONSTRUCTORS = False
"""
Whether the ARRAY constructor is context-sensitive, i.e in Redshift ARRAY[1, 2, 3] != ARRAY(1, 2, 3)
as the former is of type INT[] vs the latter which is SUPER
"""
SUPPORTS_FIXED_SIZE_ARRAYS = False
"""
Whether expressions such as x::INT[5] should be parsed as fixed-size array defs/casts e.g.
in DuckDB. In dialects which don't support fixed size arrays such as Snowflake, this should
be interpreted as a subscript/index operator.
"""
STRICT_JSON_PATH_SYNTAX = True
"""Whether failing to parse a JSON path expression using the JSONPath dialect will log a warning."""
ON_CONDITION_EMPTY_BEFORE_ERROR = True
"""Whether "X ON EMPTY" should come before "X ON ERROR" (for dialects like T-SQL, MySQL, Oracle)."""
ARRAY_AGG_INCLUDES_NULLS: t.Optional[bool] = True
"""Whether ArrayAgg needs to filter NULL values."""
PROMOTE_TO_INFERRED_DATETIME_TYPE = False
"""
This flag is used in the optimizer's canonicalize rule and determines whether x will be promoted
to the literal's type in x::DATE < '2020-01-01 12:05:03' (i.e., DATETIME). When false, the literal
is cast to x's type to match it instead.
"""
SUPPORTS_VALUES_DEFAULT = True
"""Whether the DEFAULT keyword is supported in the VALUES clause."""
NUMBERS_CAN_BE_UNDERSCORE_SEPARATED = False
"""Whether number literals can include underscores for better readability"""
HEX_STRING_IS_INTEGER_TYPE: bool = False
"""Whether hex strings such as x'CC' evaluate to integer or binary/blob type"""
REGEXP_EXTRACT_DEFAULT_GROUP = 0
"""The default value for the capturing group."""
REGEXP_EXTRACT_POSITION_OVERFLOW_RETURNS_NULL = True
"""Whether REGEXP_EXTRACT returns NULL when the position arg exceeds the string length."""
SET_OP_DISTINCT_BY_DEFAULT: t.Dict[t.Type[exp.Expression], t.Optional[bool]] = {
exp.Except: True,
exp.Intersect: True,
exp.Union: True,
}
"""
Whether a set operation uses DISTINCT by default. This is `None` when either `DISTINCT` or `ALL`
must be explicitly specified.
"""
CREATABLE_KIND_MAPPING: dict[str, str] = {}
"""
Helper for dialects that use a different name for the same creatable kind. For example, the Clickhouse
equivalent of CREATE SCHEMA is CREATE DATABASE.
"""
ALTER_TABLE_SUPPORTS_CASCADE = False
"""
Hive by default does not update the schema of existing partitions when a column is changed.
the CASCADE clause is used to indicate that the change should be propagated to all existing partitions.
the Spark dialect, while derived from Hive, does not support the CASCADE clause.
"""
# Whether ADD is present for each column added by ALTER TABLE
ALTER_TABLE_ADD_REQUIRED_FOR_EACH_COLUMN = True
# Whether the value/LHS of the TRY_CAST(<value> AS <type>) should strictly be a
# STRING type (Snowflake's case) or can be of any type
TRY_CAST_REQUIRES_STRING: t.Optional[bool] = None
# Whether the double negation can be applied
# Not safe with MySQL and SQLite due to type coercion (may not return boolean)
SAFE_TO_ELIMINATE_DOUBLE_NEGATION = True
# Whether the INITCAP function supports custom delimiter characters as the second argument
# Default delimiter characters for INITCAP function: whitespace and non-alphanumeric characters
INITCAP_SUPPORTS_CUSTOM_DELIMITERS = True
INITCAP_DEFAULT_DELIMITER_CHARS = (
" \t\n\r\f\v!\"#$%&'()*+,\\-./:;<=>?@\\[\\]^_`{|}~"
)
BYTE_STRING_IS_BYTES_TYPE: bool = False
"""
Whether byte string literals (ex: BigQuery's b'...') are typed as BYTES/BINARY
"""
UUID_IS_STRING_TYPE: bool = False
"""
Whether a UUID is considered a string or a UUID type.
"""
JSON_EXTRACT_SCALAR_SCALAR_ONLY = False
"""
Whether JSON_EXTRACT_SCALAR returns null if a non-scalar value is selected.
"""
DEFAULT_FUNCTIONS_COLUMN_NAMES: t.Dict[
t.Type[exp.Func], t.Union[str, t.Tuple[str, ...]]
] = {}
"""
Maps function expressions to their default output column name(s).
For example, in Postgres, generate_series function outputs a column named "generate_series" by default,
so we map the ExplodingGenerateSeries expression to "generate_series" string.
"""
DEFAULT_NULL_TYPE = exp.DataType.Type.UNKNOWN
"""
The default type of NULL for producing the correct projection type.
For example, in BigQuery the default type of the NULL value is INT64.
"""
LEAST_GREATEST_IGNORES_NULLS = True
"""
Whether LEAST/GREATEST functions ignore NULL values, e.g:
- BigQuery, Snowflake, MySQL, Presto/Trino: LEAST(1, NULL, 2) -> NULL
- Spark, Postgres, DuckDB, TSQL: LEAST(1, NULL, 2) -> 1
"""
PRIORITIZE_NON_LITERAL_TYPES = False
"""
Whether to prioritize non-literal types over literals during type annotation.
"""
# --- Autofilled ---
tokenizer_class = Tokenizer
jsonpath_tokenizer_class = JSONPathTokenizer
parser_class = Parser
generator_class = Generator
# A trie of the time_mapping keys
TIME_TRIE: t.Dict = {}
FORMAT_TRIE: t.Dict = {}
INVERSE_TIME_MAPPING: t.Dict[str, str] = {}
INVERSE_TIME_TRIE: t.Dict = {}
INVERSE_FORMAT_MAPPING: t.Dict[str, str] = {}
INVERSE_FORMAT_TRIE: t.Dict = {}
INVERSE_CREATABLE_KIND_MAPPING: dict[str, str] = {}
ESCAPED_SEQUENCES: t.Dict[str, str] = {}
# Delimiters for string literals and identifiers
QUOTE_START = "'"
QUOTE_END = "'"
IDENTIFIER_START = '"'
IDENTIFIER_END = '"'
VALID_INTERVAL_UNITS: t.Set[str] = set()
# Delimiters for bit, hex, byte and unicode literals
BIT_START: t.Optional[str] = None
BIT_END: t.Optional[str] = None
HEX_START: t.Optional[str] = None
HEX_END: t.Optional[str] = None
BYTE_START: t.Optional[str] = None
BYTE_END: t.Optional[str] = None
UNICODE_START: t.Optional[str] = None
UNICODE_END: t.Optional[str] = None
DATE_PART_MAPPING = {
"Y": "YEAR",
"YY": "YEAR",
"YYY": "YEAR",
"YYYY": "YEAR",
"YR": "YEAR",
"YEARS": "YEAR",
"YRS": "YEAR",
"MM": "MONTH",
"MON": "MONTH",
"MONS": "MONTH",
"MONTHS": "MONTH",
"D": "DAY",
"DD": "DAY",
"DAYS": "DAY",
"DAYOFMONTH": "DAY",
"DAY OF WEEK": "DAYOFWEEK",
"WEEKDAY": "DAYOFWEEK",
"DOW": "DAYOFWEEK",
"DW": "DAYOFWEEK",
"WEEKDAY_ISO": "DAYOFWEEKISO",
"DOW_ISO": "DAYOFWEEKISO",
"DW_ISO": "DAYOFWEEKISO",
"DAYOFWEEK_ISO": "DAYOFWEEKISO",
"DAY OF YEAR": "DAYOFYEAR",
"DOY": "DAYOFYEAR",
"DY": "DAYOFYEAR",
"W": "WEEK",
"WK": "WEEK",
"WEEKOFYEAR": "WEEK",
"WOY": "WEEK",
"WY": "WEEK",
"WEEK_ISO": "WEEKISO",
"WEEKOFYEARISO": "WEEKISO",
"WEEKOFYEAR_ISO": "WEEKISO",
"Q": "QUARTER",
"QTR": "QUARTER",
"QTRS": "QUARTER",
"QUARTERS": "QUARTER",
"H": "HOUR",
"HH": "HOUR",
"HR": "HOUR",
"HOURS": "HOUR",
"HRS": "HOUR",
"M": "MINUTE",
"MI": "MINUTE",
"MIN": "MINUTE",
"MINUTES": "MINUTE",
"MINS": "MINUTE",
"S": "SECOND",
"SEC": "SECOND",
"SECONDS": "SECOND",
"SECS": "SECOND",
"MS": "MILLISECOND",
"MSEC": "MILLISECOND",
"MSECS": "MILLISECOND",
"MSECOND": "MILLISECOND",
"MSECONDS": "MILLISECOND",
"MILLISEC": "MILLISECOND",
"MILLISECS": "MILLISECOND",
"MILLISECON": "MILLISECOND",
"MILLISECONDS": "MILLISECOND",
"US": "MICROSECOND",
"USEC": "MICROSECOND",
"USECS": "MICROSECOND",
"MICROSEC": "MICROSECOND",
"MICROSECS": "MICROSECOND",
"USECOND": "MICROSECOND",
"USECONDS": "MICROSECOND",
"MICROSECONDS": "MICROSECOND",
"NS": "NANOSECOND",
"NSEC": "NANOSECOND",
"NANOSEC": "NANOSECOND",
"NSECOND": "NANOSECOND",
"NSECONDS": "NANOSECOND",
"NANOSECS": "NANOSECOND",
"EPOCH_SECOND": "EPOCH",
"EPOCH_SECONDS": "EPOCH",
"EPOCH_MILLISECONDS": "EPOCH_MILLISECOND",
"EPOCH_MICROSECONDS": "EPOCH_MICROSECOND",
"EPOCH_NANOSECONDS": "EPOCH_NANOSECOND",
"TZH": "TIMEZONE_HOUR",
"TZM": "TIMEZONE_MINUTE",
"DEC": "DECADE",
"DECS": "DECADE",
"DECADES": "DECADE",
"MIL": "MILLENNIUM",
"MILS": "MILLENNIUM",
"MILLENIA": "MILLENNIUM",
"C": "CENTURY",
"CENT": "CENTURY",
"CENTS": "CENTURY",
"CENTURIES": "CENTURY",
}
# Specifies what types a given type can be coerced into
COERCES_TO: t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]] = {}
# Specifies type inference & validation rules for expressions
EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
# Determines the supported Dialect instance settings
SUPPORTED_SETTINGS = {
"normalization_strategy",
"version",
}
@classmethod
def get_or_raise(cls, dialect: DialectType) -> Dialect:
"""
Look up a dialect in the global dialect registry and return it if it exists.
Args:
dialect: The target dialect. If this is a string, it can be optionally followed by
additional key-value pairs that are separated by commas and are used to specify
dialect settings, such as whether the dialect's identifiers are case-sensitive.
Example:
>>> dialect = dialect_class = get_or_raise("duckdb")
>>> dialect = get_or_raise("mysql, normalization_strategy = case_sensitive")
Returns:
The corresponding Dialect instance.
"""
if not dialect:
return cls()
if isinstance(dialect, _Dialect):
return dialect()
if isinstance(dialect, Dialect):
return dialect
if isinstance(dialect, str):
try:
dialect_name, *kv_strings = dialect.split(",")
kv_pairs = (kv.split("=") for kv in kv_strings)
kwargs = {}
for pair in kv_pairs:
key = pair[0].strip()
value: t.Union[bool | str | None] = None
if len(pair) == 1:
# Default initialize standalone settings to True
value = True
elif len(pair) == 2:
value = pair[1].strip()
kwargs[key] = to_bool(value)
except ValueError:
raise ValueError(
f"Invalid dialect format: '{dialect}'. "
"Please use the correct format: 'dialect [, k1 = v2 [, ...]]'."
)
result = cls.get(dialect_name.strip())
if not result:
suggest_closest_match_and_fail(
"dialect", dialect_name, list(DIALECT_MODULE_NAMES)
)
assert result is not None
return result(**kwargs)
raise ValueError(f"Invalid dialect type for '{dialect}': '{type(dialect)}'.")
@classmethod
def format_time(
cls, expression: t.Optional[str | exp.Expression]
) -> t.Optional[exp.Expression]:
"""Converts a time format in this dialect to its equivalent Python `strftime` format."""
if isinstance(expression, str):
return exp.Literal.string(
# the time formats are quoted
format_time(expression[1:-1], cls.TIME_MAPPING, cls.TIME_TRIE)
)
if expression and expression.is_string:
return exp.Literal.string(
format_time(expression.this, cls.TIME_MAPPING, cls.TIME_TRIE)