Skip to content

Commit a77e5a5

Browse files
Jefffreyalamb
Jefffrey
and
alamb
authored
Further refactoring of type coercion function code (#19603)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Follow up to #19518 - Initial work before tackling #19004 ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> Found lots of code duplicated here, so unifying them. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> - Reduce code duplication around handling scalar/aggregate/window UDFs in type coercion related code, via usage of new `UDFCoercionExt` trait which unifies some of their behaviours (introduced by #19518) - Deprecate functions `can_coerce_from()` and `generate_signature_error_msg()` to work towards minimizing our public API surface - Fix some UDF signatures which nested `UserDefined` within `OneOf` - Fix bug where type coercion rewrites weren't being applied to arguments of non-aggregate window UDFs ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Existing tests. ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> Deprecated some functions. <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --------- Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
1 parent 568f19f commit a77e5a5

File tree

10 files changed

+183
-324
lines changed

10 files changed

+183
-324
lines changed

‎datafusion/core/tests/user_defined/user_defined_window_functions.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -536,7 +536,7 @@ impl OddCounter {
536536
impl SimpleWindowUDF {
537537
fn new(test_state: Arc<TestState>) -> Self {
538538
let signature =
539-
Signature::exact(vec![DataType::Float64], Volatility::Immutable);
539+
Signature::exact(vec![DataType::Int64], Volatility::Immutable);
540540
Self {
541541
signature,
542542
test_state: test_state.into(),

‎datafusion/expr/src/expr.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -955,7 +955,7 @@ impl AggregateFunction {
955955
pub enum WindowFunctionDefinition {
956956
/// A user defined aggregate function
957957
AggregateUDF(Arc<AggregateUDF>),
958-
/// A user defined aggregate function
958+
/// A user defined window function
959959
WindowUDF(Arc<WindowUDF>),
960960
}
961961

‎datafusion/expr/src/expr_schema.rs‎

Lines changed: 67 additions & 187 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use crate::expr::{
2121
InSubquery, Placeholder, ScalarFunction, TryCast, Unnest, WindowFunction,
2222
WindowFunctionParams,
2323
};
24-
use crate::type_coercion::functions::fields_with_udf;
24+
use crate::type_coercion::functions::{UDFCoercionExt, fields_with_udf};
2525
use crate::udf::ReturnFieldArgs;
2626
use crate::{LogicalPlan, Projection, Subquery, WindowFunctionDefinition, utils};
2727
use arrow::compute::can_cast_types;
@@ -152,44 +152,10 @@ impl ExprSchemable for Expr {
152152
}
153153
}
154154
}
155-
Expr::ScalarFunction(_func) => {
156-
let return_type = self.to_field(schema)?.1.data_type().clone();
157-
Ok(return_type)
158-
}
159-
Expr::WindowFunction(window_function) => Ok(self
160-
.window_function_field(schema, window_function)?
161-
.data_type()
162-
.clone()),
163-
Expr::AggregateFunction(AggregateFunction {
164-
func,
165-
params: AggregateFunctionParams { args, .. },
166-
}) => {
167-
let fields = args
168-
.iter()
169-
.map(|e| e.to_field(schema).map(|(_, f)| f))
170-
.collect::<Result<Vec<_>>>()?;
171-
let new_fields = fields_with_udf(&fields, func.as_ref())
172-
.map_err(|err| {
173-
let data_types = fields
174-
.iter()
175-
.map(|f| f.data_type().clone())
176-
.collect::<Vec<_>>();
177-
plan_datafusion_err!(
178-
"{} {}",
179-
match err {
180-
DataFusionError::Plan(msg) => msg,
181-
err => err.to_string(),
182-
},
183-
utils::generate_signature_error_msg(
184-
func.name(),
185-
func.signature().clone(),
186-
&data_types
187-
)
188-
)
189-
})?
190-
.into_iter()
191-
.collect::<Vec<_>>();
192-
Ok(func.return_field(&new_fields)?.data_type().clone())
155+
Expr::ScalarFunction(_)
156+
| Expr::WindowFunction(_)
157+
| Expr::AggregateFunction(_) => {
158+
Ok(self.to_field(schema)?.1.data_type().clone())
193159
}
194160
Expr::Not(_)
195161
| Expr::IsNull(_)
@@ -350,18 +316,9 @@ impl ExprSchemable for Expr {
350316
}
351317
}
352318
Expr::Cast(Cast { expr, .. }) => expr.nullable(input_schema),
353-
Expr::ScalarFunction(_func) => {
354-
let field = self.to_field(input_schema)?.1;
355-
356-
let nullable = field.is_nullable();
357-
Ok(nullable)
358-
}
359-
Expr::AggregateFunction(AggregateFunction { func, .. }) => {
360-
Ok(func.is_nullable())
361-
}
362-
Expr::WindowFunction(window_function) => Ok(self
363-
.window_function_field(input_schema, window_function)?
364-
.is_nullable()),
319+
Expr::ScalarFunction(_)
320+
| Expr::AggregateFunction(_)
321+
| Expr::WindowFunction(_) => Ok(self.to_field(input_schema)?.1.is_nullable()),
365322
Expr::ScalarVariable(field, _) => Ok(field.is_nullable()),
366323
Expr::TryCast { .. } | Expr::Unnest(_) | Expr::Placeholder(_) => Ok(true),
367324
Expr::IsNull(_)
@@ -534,69 +491,49 @@ impl ExprSchemable for Expr {
534491
)))
535492
}
536493
Expr::WindowFunction(window_function) => {
537-
self.window_function_field(schema, window_function)
538-
}
539-
Expr::AggregateFunction(aggregate_function) => {
540-
let AggregateFunction {
541-
func,
542-
params: AggregateFunctionParams { args, .. },
494+
let WindowFunction {
495+
fun,
496+
params: WindowFunctionParams { args, .. },
543497
..
544-
} = aggregate_function;
498+
} = window_function.as_ref();
545499

546500
let fields = args
547501
.iter()
548502
.map(|e| e.to_field(schema).map(|(_, f)| f))
549503
.collect::<Result<Vec<_>>>()?;
550-
// Verify that function is invoked with correct number and type of arguments as defined in `TypeSignature`
551-
let new_fields = fields_with_udf(&fields, func.as_ref())
552-
.map_err(|err| {
553-
let arg_types = fields
554-
.iter()
555-
.map(|f| f.data_type())
556-
.cloned()
557-
.collect::<Vec<_>>();
558-
plan_datafusion_err!(
559-
"{} {}",
560-
match err {
561-
DataFusionError::Plan(msg) => msg,
562-
err => err.to_string(),
563-
},
564-
utils::generate_signature_error_msg(
565-
func.name(),
566-
func.signature().clone(),
567-
&arg_types,
568-
)
569-
)
570-
})?
571-
.into_iter()
572-
.collect::<Vec<_>>();
573-
504+
match fun {
505+
WindowFunctionDefinition::AggregateUDF(udaf) => {
506+
let new_fields =
507+
verify_function_arguments(udaf.as_ref(), &fields)?;
508+
let return_field = udaf.return_field(&new_fields)?;
509+
Ok(return_field)
510+
}
511+
WindowFunctionDefinition::WindowUDF(udwf) => {
512+
let new_fields =
513+
verify_function_arguments(udwf.as_ref(), &fields)?;
514+
let return_field = udwf
515+
.field(WindowUDFFieldArgs::new(&new_fields, &schema_name))?;
516+
Ok(return_field)
517+
}
518+
}
519+
}
520+
Expr::AggregateFunction(AggregateFunction {
521+
func,
522+
params: AggregateFunctionParams { args, .. },
523+
}) => {
524+
let fields = args
525+
.iter()
526+
.map(|e| e.to_field(schema).map(|(_, f)| f))
527+
.collect::<Result<Vec<_>>>()?;
528+
let new_fields = verify_function_arguments(func.as_ref(), &fields)?;
574529
func.return_field(&new_fields)
575530
}
576531
Expr::ScalarFunction(ScalarFunction { func, args }) => {
577-
let (arg_types, fields): (Vec<DataType>, Vec<Arc<Field>>) = args
532+
let fields = args
578533
.iter()
579534
.map(|e| e.to_field(schema).map(|(_, f)| f))
580-
.collect::<Result<Vec<_>>>()?
581-
.into_iter()
582-
.map(|f| (f.data_type().clone(), f))
583-
.unzip();
584-
// Verify that function is invoked with correct number and type of arguments as defined in `TypeSignature`
585-
let new_fields =
586-
fields_with_udf(&fields, func.as_ref()).map_err(|err| {
587-
plan_datafusion_err!(
588-
"{} {}",
589-
match err {
590-
DataFusionError::Plan(msg) => msg,
591-
err => err.to_string(),
592-
},
593-
utils::generate_signature_error_msg(
594-
func.name(),
595-
func.signature().clone(),
596-
&arg_types,
597-
)
598-
)
599-
})?;
535+
.collect::<Result<Vec<_>>>()?;
536+
let new_fields = verify_function_arguments(func.as_ref(), &fields)?;
600537

601538
let arguments = args
602539
.iter()
@@ -684,6 +621,33 @@ impl ExprSchemable for Expr {
684621
}
685622
}
686623

624+
/// Verify that function is invoked with correct number and type of arguments as
625+
/// defined in `TypeSignature`.
626+
fn verify_function_arguments<F: UDFCoercionExt>(
627+
function: &F,
628+
input_fields: &[FieldRef],
629+
) -> Result<Vec<FieldRef>> {
630+
fields_with_udf(input_fields, function).map_err(|err| {
631+
let data_types = input_fields
632+
.iter()
633+
.map(|f| f.data_type())
634+
.cloned()
635+
.collect::<Vec<_>>();
636+
plan_datafusion_err!(
637+
"{} {}",
638+
match err {
639+
DataFusionError::Plan(msg) => msg,
640+
err => err.to_string(),
641+
},
642+
utils::generate_signature_error_message(
643+
function.name(),
644+
function.signature(),
645+
&data_types
646+
)
647+
)
648+
})
649+
}
650+
687651
/// Returns the innermost [Expr] that is provably null if `expr` is null.
688652
fn unwrap_certainly_null_expr(expr: &Expr) -> &Expr {
689653
match expr {
@@ -694,90 +658,6 @@ fn unwrap_certainly_null_expr(expr: &Expr) -> &Expr {
694658
}
695659
}
696660

697-
impl Expr {
698-
/// Common method for window functions that applies type coercion
699-
/// to all arguments of the window function to check if it matches
700-
/// its signature.
701-
///
702-
/// If successful, this method returns the data type and
703-
/// nullability of the window function's result.
704-
///
705-
/// Otherwise, returns an error if there's a type mismatch between
706-
/// the window function's signature and the provided arguments.
707-
fn window_function_field(
708-
&self,
709-
schema: &dyn ExprSchema,
710-
window_function: &WindowFunction,
711-
) -> Result<FieldRef> {
712-
let WindowFunction {
713-
fun,
714-
params: WindowFunctionParams { args, .. },
715-
..
716-
} = window_function;
717-
718-
let fields = args
719-
.iter()
720-
.map(|e| e.to_field(schema).map(|(_, f)| f))
721-
.collect::<Result<Vec<_>>>()?;
722-
match fun {
723-
WindowFunctionDefinition::AggregateUDF(udaf) => {
724-
let data_types = fields
725-
.iter()
726-
.map(|f| f.data_type())
727-
.cloned()
728-
.collect::<Vec<_>>();
729-
let new_fields = fields_with_udf(&fields, udaf.as_ref())
730-
.map_err(|err| {
731-
plan_datafusion_err!(
732-
"{} {}",
733-
match err {
734-
DataFusionError::Plan(msg) => msg,
735-
err => err.to_string(),
736-
},
737-
utils::generate_signature_error_msg(
738-
fun.name(),
739-
fun.signature(),
740-
&data_types
741-
)
742-
)
743-
})?
744-
.into_iter()
745-
.collect::<Vec<_>>();
746-
747-
udaf.return_field(&new_fields)
748-
}
749-
WindowFunctionDefinition::WindowUDF(udwf) => {
750-
let data_types = fields
751-
.iter()
752-
.map(|f| f.data_type())
753-
.cloned()
754-
.collect::<Vec<_>>();
755-
let new_fields = fields_with_udf(&fields, udwf.as_ref())
756-
.map_err(|err| {
757-
plan_datafusion_err!(
758-
"{} {}",
759-
match err {
760-
DataFusionError::Plan(msg) => msg,
761-
err => err.to_string(),
762-
},
763-
utils::generate_signature_error_msg(
764-
fun.name(),
765-
fun.signature(),
766-
&data_types
767-
)
768-
)
769-
})?
770-
.into_iter()
771-
.collect::<Vec<_>>();
772-
let (_, function_name) = self.qualified_name();
773-
let field_args = WindowUDFFieldArgs::new(&new_fields, &function_name);
774-
775-
udwf.field(field_args)
776-
}
777-
}
778-
}
779-
}
780-
781661
/// Cast subquery in InSubquery/ScalarSubquery to a given type.
782662
///
783663
/// 1. **Projection plan**: If the subquery is a projection (i.e. a SELECT statement with specific

0 commit comments

Comments
 (0)