Add DataFusion specific dialect

Author: alamb

Cargo.lock

@@ -23,10 +23,11 @@ use std::{
 };
 
 use datafusion::sql::sqlparser::{
-    dialect::{Dialect, GenericDialect, dialect_from_str},
+    dialect::Dialect,
     keywords::Keyword,
     tokenizer::{Token, Tokenizer},
 };
+use datafusion_common::{DATAFUSION_DIALECT, datafusion_dialect_from_str};
 use datafusion_common::config;
 use rustyline::highlight::{CmdKind, Highlighter};
 
@@ -38,7 +39,8 @@ pub struct SyntaxHighlighter {
 
 impl SyntaxHighlighter {
     pub fn new(dialect: &config::Dialect) -> Self {
-        let dialect = dialect_from_str(dialect).unwrap_or(Box::new(GenericDialect {}));
+        let dialect =
+            datafusion_dialect_from_str(dialect).unwrap_or(Box::new(DATAFUSION_DIALECT));
         Self { dialect }
     }
 }

datafusion-cli/src/highlighter.rs

@@ -61,6 +61,7 @@ pub mod test_util;
 pub mod tree_node;
 pub mod types;
 pub mod utils;
+pub mod sql_dialect;
 /// Reexport arrow crate
 pub use arrow;
 pub use column::Column;
@@ -90,6 +91,9 @@ pub use nested_struct::cast_column;
 pub use null_equality::NullEquality;
 pub use param_value::ParamValues;
 pub use scalar::{ScalarType, ScalarValue};
+pub use sql_dialect::{
+    DATAFUSION_DIALECT, DataFusionDialect, datafusion_dialect_from_str,
+};
 pub use schema_reference::SchemaReference;
 pub use spans::{Location, Span, Spans};
 pub use stats::{ColumnStatistics, Statistics};

datafusion/common/src/lib.rs

@@ -0,0 +1,338 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::any::TypeId;
+
+use sqlparser::ast::{Expr, Map, MapEntry};
+use sqlparser::dialect::{Dialect, GenericDialect, Precedence};
+use sqlparser::keywords::Keyword;
+use sqlparser::parser::{Parser, ParserError};
+use sqlparser::tokenizer::Token;
+
+/// DataFusion's default SQL dialect.
+///
+/// This dialect is intentionally permissive and enables SQL features that
+/// DataFusion's parser and planner support across multiple upstream dialects.
+/// It is similar to sqlparser's `GenericDialect`, but is defined in DataFusion
+/// to decouple behavior from upstream defaults.
+#[derive(Debug, Default)]
+pub struct DataFusionDialect;
+
+pub const DATAFUSION_DIALECT: DataFusionDialect = DataFusionDialect;
+
+impl Dialect for DataFusionDialect {
+    fn dialect(&self) -> TypeId {
+        TypeId::of::<GenericDialect>()
+    }
+
+    fn is_delimited_identifier_start(&self, ch: char) -> bool {
+        ch == '"' || ch == '`'
+    }
+
+    fn is_identifier_start(&self, ch: char) -> bool {
+        ch.is_alphabetic() || ch == '_' || ch == '#' || ch == '@'
+    }
+
+    fn is_identifier_part(&self, ch: char) -> bool {
+        ch.is_alphabetic()
+            || ch.is_ascii_digit()
+            || ch == '@'
+            || ch == '$'
+            || ch == '#'
+            || ch == '_'
+    }
+
+    fn supports_unicode_string_literal(&self) -> bool {
+        true
+    }
+
+    fn supports_group_by_expr(&self) -> bool {
+        true
+    }
+
+    fn supports_group_by_with_modifier(&self) -> bool {
+        true
+    }
+
+    fn supports_left_associative_joins_without_parens(&self) -> bool {
+        true
+    }
+
+    fn supports_connect_by(&self) -> bool {
+        true
+    }
+
+    fn supports_match_recognize(&self) -> bool {
+        true
+    }
+
+    fn supports_pipe_operator(&self) -> bool {
+        true
+    }
+
+    fn supports_start_transaction_modifier(&self) -> bool {
+        true
+    }
+
+    fn supports_window_function_null_treatment_arg(&self) -> bool {
+        true
+    }
+
+    fn supports_dictionary_syntax(&self) -> bool {
+        true
+    }
+
+    fn supports_window_clause_named_window_reference(&self) -> bool {
+        true
+    }
+
+    fn supports_parenthesized_set_variables(&self) -> bool {
+        true
+    }
+
+    fn supports_select_wildcard_except(&self) -> bool {
+        true
+    }
+
+    fn support_map_literal_syntax(&self) -> bool {
+        true
+    }
+
+    fn allow_extract_custom(&self) -> bool {
+        true
+    }
+
+    fn allow_extract_single_quotes(&self) -> bool {
+        true
+    }
+
+    fn supports_extract_comma_syntax(&self) -> bool {
+        true
+    }
+
+    fn supports_create_view_comment_syntax(&self) -> bool {
+        true
+    }
+
+    fn supports_parens_around_table_factor(&self) -> bool {
+        true
+    }
+
+    fn supports_values_as_table_factor(&self) -> bool {
+        true
+    }
+
+    fn supports_create_index_with_clause(&self) -> bool {
+        true
+    }
+
+    fn supports_explain_with_utility_options(&self) -> bool {
+        true
+    }
+
+    fn supports_limit_comma(&self) -> bool {
+        true
+    }
+
+    fn supports_from_first_select(&self) -> bool {
+        true
+    }
+
+    fn supports_projection_trailing_commas(&self) -> bool {
+        true
+    }
+
+    fn supports_asc_desc_in_column_definition(&self) -> bool {
+        true
+    }
+
+    fn supports_try_convert(&self) -> bool {
+        true
+    }
+
+    fn supports_bitwise_shift_operators(&self) -> bool {
+        true
+    }
+
+    fn supports_comment_on(&self) -> bool {
+        true
+    }
+
+    fn supports_load_extension(&self) -> bool {
+        true
+    }
+
+    fn supports_named_fn_args_with_assignment_operator(&self) -> bool {
+        true
+    }
+
+    fn supports_struct_literal(&self) -> bool {
+        true
+    }
+
+    fn supports_empty_projections(&self) -> bool {
+        true
+    }
+
+    fn supports_nested_comments(&self) -> bool {
+        true
+    }
+
+    fn supports_user_host_grantee(&self) -> bool {
+        true
+    }
+
+    fn supports_string_escape_constant(&self) -> bool {
+        true
+    }
+
+    fn supports_array_typedef_with_brackets(&self) -> bool {
+        true
+    }
+
+    fn supports_match_against(&self) -> bool {
+        true
+    }
+
+    fn supports_set_names(&self) -> bool {
+        true
+    }
+
+    fn supports_comma_separated_set_assignments(&self) -> bool {
+        true
+    }
+
+    fn supports_filter_during_aggregation(&self) -> bool {
+        true
+    }
+
+    fn supports_select_wildcard_exclude(&self) -> bool {
+        true
+    }
+
+    fn supports_data_type_signed_suffix(&self) -> bool {
+        true
+    }
+
+    fn supports_interval_options(&self) -> bool {
+        true
+    }
+
+    fn supports_quote_delimited_string(&self) -> bool {
+        true
+    }
+
+    fn supports_lambda_functions(&self) -> bool {
+        true
+    }
+
+    fn supports_select_wildcard_replace(&self) -> bool {
+        true
+    }
+
+    fn supports_select_wildcard_ilike(&self) -> bool {
+        true
+    }
+
+    fn supports_select_wildcard_rename(&self) -> bool {
+        true
+    }
+
+    fn supports_optimize_table(&self) -> bool {
+        true
+    }
+
+    fn supports_install(&self) -> bool {
+        true
+    }
+
+    fn supports_detach(&self) -> bool {
+        true
+    }
+
+    fn supports_prewhere(&self) -> bool {
+        true
+    }
+
+    fn supports_with_fill(&self) -> bool {
+        true
+    }
+
+    fn supports_limit_by(&self) -> bool {
+        true
+    }
+
+    fn supports_interpolate(&self) -> bool {
+        true
+    }
+
+    fn supports_settings(&self) -> bool {
+        true
+    }
+
+    fn supports_select_format(&self) -> bool {
+        true
+    }
+
+    fn supports_comment_optimizer_hint(&self) -> bool {
+        true
+    }
+
+    fn parse_prefix(&self, parser: &mut Parser) -> Option<Result<Expr, ParserError>> {
+        let token = parser.peek_token_ref();
+        let is_map_literal = matches!(
+            token.token,
+            Token::Word(ref word) if word.keyword == Keyword::MAP
+        ) && matches!(parser.peek_tokens::<2>()[1], Token::LBrace);
+
+        if !is_map_literal {
+            return None;
+        }
+
+        Some(parse_map_literal(parser))
+    }
+}
+
+fn parse_map_literal(parser: &mut Parser) -> Result<Expr, ParserError> {
+    let _ = parser.parse_keyword(Keyword::MAP);
+    parser.expect_token(&Token::LBrace)?;
+    let entries = parser.parse_comma_separated0(parse_map_entry, Token::RBrace)?;
+    parser.expect_token(&Token::RBrace)?;
+    Ok(Expr::Map(Map { entries }))
+}
+
+fn parse_map_entry(parser: &mut Parser) -> Result<MapEntry, ParserError> {
+    let key = parser.parse_subexpr(DATAFUSION_DIALECT.prec_value(Precedence::Colon))?;
+    parser.expect_token(&Token::Colon)?;
+    let value = parser.parse_expr()?;
+    Ok(MapEntry {
+        key: Box::new(key),
+        value: Box::new(value),
+    })
+}
+
+/// Returns the DataFusion dialect for `generic` (and `datafusion`) and otherwise
+/// falls back to sqlparser's built-in dialect lookup.
+pub fn datafusion_dialect_from_str(
+    dialect_name: impl AsRef<str>,
+) -> Option<Box<dyn Dialect>> {
+    let dialect_name = dialect_name.as_ref();
+    match dialect_name.to_lowercase().as_str() {
+        "generic" | "datafusion" => Some(Box::new(DATAFUSION_DIALECT)),
+        _ => sqlparser::dialect::dialect_from_str(dialect_name),
+    }
+}