Changes from Embucket on top of 47.0.0

Make required funcs public (#1)

* Make required funcs public

* Make required funcs public

* Make required funcs public

Extend to_date UDF supported types (#3)

* Extend to_date supported types

* Remove TODO

Add "rlike" as an alias for regexp_like (#2)

* Add "rlike" as an alias for regexp_like

* Update docs

Extend references with new ShowSchemas type (#4)

* Extend references with new SHowSchemas type

* Make resolve_table_ref as pub

* Fix docs linter

Implement regexp_substr (#6)

* Extend references with new SHowSchemas type

Make resolve_table_ref as pub

Fix docs linter

Add DDL AlterTable

Add DDL AlterTable

Add "rlike" as an alias for regexp_like (#2)

* Add "rlike" as an alias for regexp_like

* Update docs

Extend references with new ShowSchemas type (#4)

* Extend references with new SHowSchemas type

* Make resolve_table_ref as pub

* Fix docs linter

Fix deps

Import alter table

Temp

Temp

Add regexp_substr udf

Add regexp_substr udf

Add regexp_substr udf

Add regexp_substr udf

* Add regexp_substr to docs

* Update comments and fix returned datatype

* Update docs

Make statements publci (#7)

Make context public (#9)

* Make conext public

* Fix clippy

DFParser should skip unsupported COPY INTO (#10)

Skip CREATE STAGE by DFParser (pass to dialect parser) (#11)

* DFParser should skip unsupported COPY INTO

* Add tests

* Add tests

* Add tests

* Add tests

* Fmt

* Fmt

* Fmt

Extend references with new Show* types (#14)

Add e flag support (#16)

* Add e flag support

* Add e flag support

* fix fmt

* clippy

return Int64 instead of UInt64 when returning count (#17)

Revert commit: 314a726 (#18)

* Revert "return Int64 instead of UInt64 when returning count (#17)"

This reverts commit 314a726.

* revert changes as of a bad approach use

Bump sqlparser version to 0.55. Fixes after rebase

named arguments for table functions (#20)

TableFunction Fixes (#23)

* TableFunction Fixes

* Also fix tests after rebase

* Taplo format

* Clippy

* Fix docs

PIVOT command (#22)

* Pivot initial changes

* Polishing

* Rebase fixes

* Add slt tests

* Add display for PIVOT

* Only check projections in case of subqueries

* Add builder test

* Add default on null. Cargo fmt

* Add default_on_null to proto and tests

* Fix protoc and clippy

* Cargo fmt

Update parser version (#26)

Fix incompatible type in IS NOT DISTINCT filter for PIVOT (#25)

* Fix incompattible type in IS NOT DISTINCT filter

* Cargo fmt

* Cargo clippy

UNPIVOT command (#24)

* UNPIVOT command implementation

* UNPIVOT command

* Add slt tests

* Taplo and cargo fmt

* Resolve comments

Update sql-parser

Update parser

update parser with fetch (#28)

Enhance string concat coercion to support castable types  (#27)

* Allow any stringle types for concatination

* Fix fmt

Added basic support for table functions in relations (#29)

* Add basic support for flatten join

* Fix outer

Update sqlparser-version

Get table function in ascii lowercase (#30)

* Get table function in ascii lowercase

* Get table function in ascii lowercase

Update parser version

Update parser

Allow boolean vs integer comparison (#31)

* Allow comparison between boolean and int values

* Allow comparison between boolean and int values

* Add integer types

* clippy+fmt+tests fix

* tests fix

* Fix incompatible types test

* Fix slt test

* Fix docs

Fix unpivot. Columns result always in uppercase (#32)

Update parser (#33)

* Update parser

* Update version

* Update parser and external table

* Fix external table

* Fix external table

`QUALIFY` clause implementation (#34)

* Implement  clause

* Remove comment

* Remove unimplemented

Support projection in subquery with IN clause (#35)

Update parser with SECURE keyword for views (#36)

* Update parser with SECURE keyword for views

* update rustc version for compatibility

Decorellate subqueries in IN inside JOIN filter and Aggregates (#37)

Author: osipovartem

Cargo.lock

@@ -184,7 +184,9 @@ regex = "1.12"
 rstest = "0.26.1"
 serde_json = "1"
 sha2 = "^0.10.9"
-sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor"] }
+sqlparser = { git = "https://github.com/Embucket/datafusion-sqlparser-rs.git", rev = "46c81692142afa6812d3b3bbe49d3daeb361e895", features = [
+    "visitor",
+] }
 strum = "0.28.0"
 strum_macros = "0.28.0"
 tempfile = "3"

Cargo.toml

@@ -326,9 +326,9 @@ fn fixed_len_byte_array_to_string(val: Option<&FixedLenByteArray>) -> Option<Str
 pub struct ParquetMetadataFunc {}
 
 impl TableFunctionImpl for ParquetMetadataFunc {
-    fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
+    fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
         let filename = match exprs.first() {
-            Some(Expr::Literal(ScalarValue::Utf8(Some(s)), _)) => s, // single quote: parquet_metadata('x.parquet')
+            Some((Expr::Literal(ScalarValue::Utf8(Some(s)), _), _)) => s, // single quote: parquet_metadata('x.parquet')
             Some(Expr::Column(Column { name, .. })) => name, // double quote: parquet_metadata("x.parquet")
             _ => {
                 return plan_err!(

FETCH_HEAD

@@ -69,8 +69,8 @@ pub async fn dataframe_example() -> Result<()> {
     write_out(&ctx).await?;
     register_cars_test_data("t1", &ctx).await?;
     register_cars_test_data("t2", &ctx).await?;
-    where_scalar_subquery(&ctx).await?;
-    where_in_subquery(&ctx).await?;
+    Box::pin(where_scalar_subquery(&ctx)).await?;
+    Box::pin(where_in_subquery(&ctx)).await?;
     where_exist_subquery(&ctx).await?;
     Ok(())
 }

datafusion-cli/src/functions.rs

@@ -135,14 +135,15 @@ impl TableProvider for LocalCsvTable {
 struct LocalCsvTableFunc {}
 
 impl TableFunctionImpl for LocalCsvTableFunc {
-    fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
-        let Some(Expr::Literal(ScalarValue::Utf8(Some(path)), _)) = exprs.first() else {
+    fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
+        let Some((Expr::Literal(ScalarValue::Utf8(Some(ref path)), _), _)) = exprs.first()
+        else {
             return plan_err!("read_csv requires at least one string argument");
         };
 
         let limit = exprs
             .get(1)
-            .map(|expr| {
+            .map(|(expr, _)| {
                 // try to simplify the expression, so 1+2 becomes 3, for example
                 let info = SimplifyContext::default();
                 let expr = ExprSimplifier::new(info).simplify(expr.clone())?;

datafusion-examples/examples/dataframe/dataframe.rs

@@ -488,7 +488,7 @@ pub trait TableProviderFactory: Debug + Sync + Send {
 /// A trait for table function implementations
 pub trait TableFunctionImpl: Debug + Sync + Send {
     /// Create a table provider
-    fn call(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>>;
+    fn call(&self, args: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>>;
 }
 
 /// A table that uses a function to generate data
@@ -517,7 +517,10 @@ impl TableFunction {
     }
 
     /// Get the function implementation and generate a table
-    pub fn create_table_provider(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>> {
+    pub fn create_table_provider(
+        &self,
+        args: &[(Expr, Option<String>)],
+    ) -> Result<Arc<dyn TableProvider>> {
         self.fun.call(args)
     }
 }

datafusion-examples/examples/udf/simple_udtf.rs

@@ -292,7 +292,7 @@ impl Session for SessionState {
 }
 
 impl SessionState {
-    pub(crate) fn resolve_table_ref(
+    pub fn resolve_table_ref(
         &self,
         table_ref: impl Into<TableReference>,
     ) -> ResolvedTableReference {
@@ -515,7 +515,8 @@ impl SessionState {
     }
 
     #[cfg(feature = "sql")]
-    fn get_parser_options(&self) -> ParserOptions {
+    /// Get the parser options
+    pub fn get_parser_options(&self) -> ParserOptions {
         let sql_parser_options = &self.config.options().sql_parser;
 
         ParserOptions {
@@ -1790,9 +1791,11 @@ impl From<SessionState> for SessionStateBuilder {
 /// This is used so the SQL planner can access the state of the session without
 /// having a direct dependency on the [`SessionState`] struct (and core crate)
 #[cfg(feature = "sql")]
-struct SessionContextProvider<'a> {
-    state: &'a SessionState,
-    tables: HashMap<ResolvedTableReference, Arc<dyn TableSource>>,
+pub struct SessionContextProvider<'a> {
+    /// The session state
+    pub state: &'a SessionState,
+    /// The tables available in the session
+    pub tables: HashMap<ResolvedTableReference, Arc<dyn TableSource>>,
 }
 
 #[cfg(feature = "sql")]
@@ -1827,12 +1830,13 @@ impl ContextProvider for SessionContextProvider<'_> {
     fn get_table_function_source(
         &self,
         name: &str,
-        args: Vec<Expr>,
+        args: Vec<(Expr, Option<String>)>,
     ) -> datafusion_common::Result<Arc<dyn TableSource>> {
+        let name = name.to_ascii_lowercase();
         let tbl_func = self
             .state
             .table_functions
-            .get(name)
+            .get(&name)
             .cloned()
             .ok_or_else(|| plan_datafusion_err!("table function '{name}' not found"))?;
         let simplify_context = SimplifyContext::default()
@@ -1844,10 +1848,11 @@ impl ContextProvider for SessionContextProvider<'_> {
         let schema = DFSchema::empty();
         let args = args
             .into_iter()
-            .map(|arg| {
+            .map(|(expr, named_param)| {
                 simplifier
-                    .coerce(arg, &schema)
+                    .coerce(expr, &schema)
                     .and_then(|e| simplifier.simplify(e))
+                    .map(|e| (e, named_param))
             })
             .collect::<datafusion_common::Result<Vec<_>>>()?;
         let provider = tbl_func.create_table_provider(&args)?;