Skip to content

Commit e210e54

Browse files
VedinJanKaul
Vedin
authored and
JanKaul
committed
Fixes after sync to 50.0.0
1 parent 0d69155 commit e210e54

18 files changed

Lines changed: 165 additions & 92 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion-cli/src/functions.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,9 +327,13 @@ pub struct ParquetMetadataFunc {}
327327

328328
impl TableFunctionImpl for ParquetMetadataFunc {
329329
fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
330+
if exprs.is_empty() {
331+
return plan_err!("parquet_metadata requires string argument as its input");
332+
}
333+
330334
let filename = match exprs.first() {
331335
Some((Expr::Literal(ScalarValue::Utf8(Some(s)), _), _)) => s, // single quote: parquet_metadata('x.parquet')
332-
Some(Expr::Column(Column { name, .. })) => name, // double quote: parquet_metadata("x.parquet")
336+
Some((Expr::Column(Column { name, .. }), _)) => name, // double quote: parquet_metadata("x.parquet")
333337
_ => {
334338
return plan_err!(
335339
"parquet_metadata requires string argument as its input"
@@ -517,7 +521,7 @@ impl MetadataCacheFunc {
517521
}
518522

519523
impl TableFunctionImpl for MetadataCacheFunc {
520-
fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
524+
fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
521525
if !exprs.is_empty() {
522526
return plan_err!("metadata_cache should have no arguments");
523527
}

‎datafusion-examples/examples/udf/simple_udtf.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,8 @@ struct LocalCsvTableFunc {}
136136

137137
impl TableFunctionImpl for LocalCsvTableFunc {
138138
fn call(&self, exprs: &[(Expr, Option<String>)]) -> Result<Arc<dyn TableProvider>> {
139-
let Some((Expr::Literal(ScalarValue::Utf8(Some(ref path)), _), _)) = exprs.first()
139+
let Some((Expr::Literal(ScalarValue::Utf8(Some(ref path)), _), _)) =
140+
exprs.first()
140141
else {
141142
return plan_err!("read_csv requires at least one string argument");
142143
};

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,8 @@ impl Session for SessionState {
292292
}
293293

294294
impl SessionState {
295+
/// Resolve a [`TableReference`] into a [`ResolvedTableReference`] using
296+
/// the session's configured default catalog and schema.
295297
pub fn resolve_table_ref(
296298
&self,
297299
table_ref: impl Into<TableReference>,

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,6 @@ use datafusion_physical_plan::recursive_query::RecursiveQueryExec;
105105
use datafusion_physical_plan::unnest::ListUnnest;
106106

107107
use async_trait::async_trait;
108-
use datafusion_datasource::file_groups::FileGroup;
109108
use datafusion_expr_common::operator::Operator;
110109
use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper};
111110
use futures::{StreamExt, TryStreamExt};
@@ -2556,8 +2555,6 @@ pub fn create_aggregate_expr_and_maybe_filter(
25562555
)
25572556
}
25582557

2559-
2560-
25612558
/// Transform a PIVOT operation into a more standard Aggregate + Projection plan
25622559
/// For known pivot values, we create a projection that includes "IS NOT DISTINCT FROM" conditions
25632560
///
@@ -2614,7 +2611,7 @@ pub fn transform_pivot_to_aggregate(
26142611
Box::new(Expr::Column(pivot_column.clone())),
26152612
Operator::IsNotDistinctFrom,
26162613
Box::new(Expr::Cast(Cast::new(
2617-
Box::new(Expr::Literal(value.clone())),
2614+
Box::new(Expr::Literal(value.clone(), None)),
26182615
pivot_col_type.clone(),
26192616
))),
26202617
));
@@ -3089,9 +3086,9 @@ impl DefaultPhysicalPlanner {
30893086
if let LogicalPlan::Pivot(pivot) = input.as_ref() {
30903087
if pivot.value_subquery.is_some()
30913088
&& input_exec
3092-
.as_any()
3093-
.downcast_ref::<AggregateExec>()
3094-
.is_some()
3089+
.as_any()
3090+
.downcast_ref::<AggregateExec>()
3091+
.is_some()
30953092
{
30963093
let agg_exec =
30973094
input_exec.as_any().downcast_ref::<AggregateExec>().unwrap();

‎datafusion/expr/src/logical_plan/display.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -743,8 +743,8 @@ impl<'n> TreeNodeVisitor<'n> for PgJsonVisitor<'_, '_> {
743743
mod tests {
744744
use crate::EmptyRelation;
745745
use arrow::datatypes::{DataType, Field, Schema};
746-
use insta::assert_snapshot;
747746
use datafusion_common::{Column, DFSchema, ScalarValue};
747+
use insta::assert_snapshot;
748748
use std::sync::Arc;
749749

750750
use super::*;

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2526,7 +2526,7 @@ fn pivot_schema(
25262526
}
25272527

25282528
for pivot_value in pivot_values {
2529-
let field_name = format!("{}", pivot_value);
2529+
let field_name = format!("{pivot_value}");
25302530
let data_type = aggregate_expr.get_type(input_schema)?;
25312531
fields.push(Arc::new(Field::new(field_name, data_type, true)));
25322532
}

‎datafusion/functions-table/src/generate_series.rs‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -510,10 +510,13 @@ impl TableFunctionImpl for GenerateSeriesFuncImpl {
510510
}
511511

512512
impl GenerateSeriesFuncImpl {
513-
fn call_int64(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
513+
fn call_int64(
514+
&self,
515+
exprs: &[(Expr, Option<String>)],
516+
) -> Result<Arc<dyn TableProvider>> {
514517
let mut normalize_args = Vec::new();
515-
for (expr_index, expr, _) in exprs.iter().enumerate() {
516-
match expr {
518+
for (expr_index, expr) in exprs.iter().enumerate() {
519+
match &expr.0 {
517520
Expr::Literal(ScalarValue::Null, _) => {}
518521
Expr::Literal(ScalarValue::Int64(Some(n)), _) => normalize_args.push(*n),
519522
other => {
@@ -565,7 +568,10 @@ impl GenerateSeriesFuncImpl {
565568
}))
566569
}
567570

568-
fn call_timestamp(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
571+
fn call_timestamp(
572+
&self,
573+
exprs: &[(Expr, Option<String>)],
574+
) -> Result<Arc<dyn TableProvider>> {
569575
if exprs.len() != 3 {
570576
return plan_err!(
571577
"{} function with timestamps requires exactly 3 arguments",
@@ -574,7 +580,7 @@ impl GenerateSeriesFuncImpl {
574580
}
575581

576582
// Parse start timestamp
577-
let (start_ts, tz) = match &exprs[0] {
583+
let (start_ts, tz) = match &exprs[0].0 {
578584
Expr::Literal(ScalarValue::TimestampNanosecond(ts, tz), _) => {
579585
(*ts, tz.clone())
580586
}
@@ -587,7 +593,7 @@ impl GenerateSeriesFuncImpl {
587593
};
588594

589595
// Parse end timestamp
590-
let end_ts = match &exprs[1] {
596+
let end_ts = match &exprs[1].0 {
591597
Expr::Literal(ScalarValue::Null, _) => None,
592598
Expr::Literal(ScalarValue::TimestampNanosecond(ts, _), _) => *ts,
593599
other => {
@@ -599,7 +605,7 @@ impl GenerateSeriesFuncImpl {
599605
};
600606

601607
// Parse step interval
602-
let step_interval = match &exprs[2] {
608+
let step_interval = match &exprs[2].0 {
603609
Expr::Literal(ScalarValue::Null, _) => None,
604610
Expr::Literal(ScalarValue::IntervalMonthDayNano(interval), _) => *interval,
605611
other => {
@@ -641,7 +647,10 @@ impl GenerateSeriesFuncImpl {
641647
}))
642648
}
643649

644-
fn call_date(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
650+
fn call_date(
651+
&self,
652+
exprs: &[(Expr, Option<String>)],
653+
) -> Result<Arc<dyn TableProvider>> {
645654
if exprs.len() != 3 {
646655
return plan_err!(
647656
"{} function with dates requires exactly 3 arguments",
@@ -656,7 +665,7 @@ impl GenerateSeriesFuncImpl {
656665
)]));
657666

658667
// Parse start date
659-
let start_date = match &exprs[0] {
668+
let start_date = match &exprs[0].0 {
660669
Expr::Literal(ScalarValue::Date32(Some(date)), _) => *date,
661670
Expr::Literal(ScalarValue::Date32(None), _)
662671
| Expr::Literal(ScalarValue::Null, _) => {
@@ -674,7 +683,7 @@ impl GenerateSeriesFuncImpl {
674683
};
675684

676685
// Parse end date
677-
let end_date = match &exprs[1] {
686+
let end_date = match &exprs[1].0 {
678687
Expr::Literal(ScalarValue::Date32(Some(date)), _) => *date,
679688
Expr::Literal(ScalarValue::Date32(None), _)
680689
| Expr::Literal(ScalarValue::Null, _) => {
@@ -692,7 +701,7 @@ impl GenerateSeriesFuncImpl {
692701
};
693702

694703
// Parse step interval
695-
let step_interval = match &exprs[2] {
704+
let step_interval = match &exprs[2].0 {
696705
Expr::Literal(ScalarValue::IntervalMonthDayNano(Some(interval)), _) => {
697706
*interval
698707
}

‎datafusion/functions/src/datetime/to_date.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -503,18 +503,14 @@ mod tests {
503503
];
504504
for scalar in test_cases {
505505
let timestamp_to_date_result =
506-
ToDateFunc::new().invoke_with_args(datafusion_expr::ScalarFunctionArgs {
507-
args: vec![ColumnarValue::Scalar(scalar.clone())],
508-
number_rows: 1,
509-
return_type: &DataType::Date32,
510-
});
506+
invoke_to_date_with_args(vec![ColumnarValue::Scalar(scalar.clone())], 1);
511507

512508
match timestamp_to_date_result {
513509
Ok(ColumnarValue::Scalar(ScalarValue::Date32(date_val))) => {
514510
let expected = Date32Type::parse_formatted("2025-01-13", "%Y-%m-%d");
515511
assert_eq!(date_val, expected, "to_date created wrong value");
516512
}
517-
_ => panic!("Conversion of {}", scalar),
513+
_ => panic!("Conversion of {scalar}"),
518514
}
519515
}
520516
}

‎datafusion/functions/src/regex/regexpsubstr.rs‎

Lines changed: 69 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use regex::Regex;
3333
use std::any::Any;
3434
use std::sync::{Arc, OnceLock};
3535

36-
#[derive(Debug)]
36+
#[derive(Debug, PartialEq, Eq, Hash)]
3737
pub struct RegexpSubstrFunc {
3838
signature: Signature,
3939
}
@@ -318,24 +318,23 @@ fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowError>
318318
if flags.is_empty() {
319319
regex.to_string()
320320
} else {
321-
format!("(?{}){}", flags, regex)
321+
format!("(?{flags}){regex}")
322322
}
323323
}
324324
};
325325

326326
Regex::new(&pattern).map_err(|_| {
327-
ArrowError::ComputeError(format!(
328-
"Regular expression did not compile: {}",
329-
pattern
330-
))
327+
ArrowError::ComputeError(
328+
format!("Regular expression did not compile: {pattern}",),
329+
)
331330
})
332331
}
333332

334333
#[cfg(test)]
335334
mod tests {
336335
use crate::regex::regexpsubstr::{regexp_substr, RegexpSubstrFunc};
337336
use arrow::array::{Array, ArrayRef, Int64Array, LargeStringArray, StringArray};
338-
use arrow::datatypes::DataType;
337+
use arrow::datatypes::{DataType, Field};
339338
use datafusion_common::ScalarValue;
340339
use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
341340
use datafusion_expr_common::columnar_value::ColumnarValue;
@@ -371,14 +370,30 @@ mod tests {
371370
ScalarValue::LargeUtf8 as fn(Option<String>) -> ScalarValue,
372371
),
373372
] {
373+
let args_vec = vec![
374+
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
375+
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
376+
];
377+
let arg_fields = args_vec
378+
.iter()
379+
.enumerate()
380+
.map(|(idx, arg)| {
381+
Field::new(format!("f_{idx}"), arg.data_type(), true).into()
382+
})
383+
.collect();
374384
let result =
375385
RegexpSubstrFunc::new().invoke_with_args(ScalarFunctionArgs {
376-
args: vec![
377-
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
378-
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
379-
],
386+
args: args_vec,
387+
arg_fields,
380388
number_rows: 1,
381-
return_type: data_type,
389+
return_field: Arc::new(Field::new(
390+
"f",
391+
data_type.clone(),
392+
true,
393+
)),
394+
config_options: Arc::new(
395+
datafusion_common::config::ConfigOptions::default(),
396+
),
382397
});
383398
match result {
384399
Ok(ColumnarValue::Scalar(
@@ -422,14 +437,26 @@ mod tests {
422437
),
423438
_ => unreachable!(),
424439
};
440+
let args_vec = vec![
441+
ColumnarValue::Array(Arc::new(array_values)),
442+
ColumnarValue::Scalar(regex),
443+
];
444+
let arg_fields = args_vec
445+
.iter()
446+
.enumerate()
447+
.map(|(idx, arg)| {
448+
Field::new(format!("f_{idx}"), arg.data_type(), true).into()
449+
})
450+
.collect();
425451
let result =
426452
RegexpSubstrFunc::new().invoke_with_args(ScalarFunctionArgs {
427-
args: vec![
428-
ColumnarValue::Array(Arc::new(array_values)),
429-
ColumnarValue::Scalar(regex),
430-
],
453+
args: args_vec,
454+
arg_fields,
431455
number_rows: 1,
432-
return_type: data_type,
456+
return_field: Arc::new(Field::new("f", data_type.clone(), true)),
457+
config_options: Arc::new(
458+
datafusion_common::config::ConfigOptions::default(),
459+
),
433460
});
434461
match result {
435462
Ok(ColumnarValue::Array(array)) => {
@@ -511,22 +538,34 @@ mod tests {
511538
ScalarValue::LargeUtf8 as fn(Option<String>) -> ScalarValue,
512539
),
513540
] {
541+
let args_vec = vec![
542+
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
543+
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
544+
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
545+
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
546+
ColumnarValue::Scalar(scalar(Some(flags[spos].to_string()))),
547+
ColumnarValue::Scalar(ScalarValue::Int64(Some(group_num[spos]))),
548+
];
549+
let arg_fields = args_vec
550+
.iter()
551+
.enumerate()
552+
.map(|(idx, arg)| {
553+
Field::new(format!("f_{idx}"), arg.data_type(), true).into()
554+
})
555+
.collect();
514556
let result =
515557
RegexpSubstrFunc::new().invoke_with_args(ScalarFunctionArgs {
516-
args: vec![
517-
ColumnarValue::Scalar(scalar(Some(value.to_string()))),
518-
ColumnarValue::Scalar(scalar(Some(regex.to_string()))),
519-
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
520-
ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
521-
ColumnarValue::Scalar(scalar(Some(
522-
flags[spos].to_string(),
523-
))),
524-
ColumnarValue::Scalar(ScalarValue::Int64(Some(
525-
group_num[spos],
526-
))),
527-
],
558+
args: args_vec,
559+
arg_fields,
528560
number_rows: 1,
529-
return_type: data_type,
561+
return_field: Arc::new(Field::new(
562+
"f",
563+
data_type.clone(),
564+
true,
565+
)),
566+
config_options: Arc::new(
567+
datafusion_common::config::ConfigOptions::default(),
568+
),
530569
});
531570
match result {
532571
Ok(ColumnarValue::Scalar(

0 commit comments

Comments
 (0)