Skip to content

Commit 981b5c3

Browse files
benbellickbenbellick
authored
chore(deps): bump substrait from 0.62 to 0.63.0 (apache#20876)
Bump `substrait` crate from 0.62.2 to 0.63.0 (v0.62.3 was yanked) This substrait bump included: - removing deprecated `extension_uri_reference` fields (use `extension_urn_reference`), removed deprecated `extension_uris` field on `Plan`/`ExtendedExpression` (use `extension_urns`) - Handle new `LambdaParameterReference`, `Lambda`, and `LambdaInvocation` enum variants - Update `UserDefined` literal handling for new `TypeAnchorType` oneof Supersedes apache#20750
1 parent f8fb5bd commit 981b5c3

File tree

13 files changed

+86
-65
lines changed

13 files changed

+86
-65
lines changed

‎Cargo.lock‎

Lines changed: 13 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4456,49 +4456,49 @@ mod tests {
44564456
[
44574457
{
44584458
"Plan": {
4459+
"Node Type": "Projection",
44594460
"Expressions": [
44604461
"employee_csv.id"
44614462
],
4462-
"Node Type": "Projection",
4463-
"Output": [
4464-
"id"
4465-
],
44664463
"Plans": [
44674464
{
4468-
"Condition": "employee_csv.state IN (<subquery>)",
44694465
"Node Type": "Filter",
4470-
"Output": [
4471-
"id",
4472-
"state"
4473-
],
4466+
"Condition": "employee_csv.state IN (<subquery>)",
44744467
"Plans": [
44754468
{
44764469
"Node Type": "Subquery",
4477-
"Output": [
4478-
"state"
4479-
],
44804470
"Plans": [
44814471
{
44824472
"Node Type": "TableScan",
4473+
"Relation Name": "employee_csv",
4474+
"Plans": [],
44834475
"Output": [
44844476
"state"
4485-
],
4486-
"Plans": [],
4487-
"Relation Name": "employee_csv"
4477+
]
44884478
}
4479+
],
4480+
"Output": [
4481+
"state"
44894482
]
44904483
},
44914484
{
44924485
"Node Type": "TableScan",
4486+
"Relation Name": "employee_csv",
4487+
"Plans": [],
44934488
"Output": [
44944489
"id",
44954490
"state"
4496-
],
4497-
"Plans": [],
4498-
"Relation Name": "employee_csv"
4491+
]
44994492
}
4493+
],
4494+
"Output": [
4495+
"id",
4496+
"state"
45004497
]
45014498
}
4499+
],
4500+
"Output": [
4501+
"id"
45024502
]
45034503
}
45044504
}

‎datafusion/sqllogictest/test_files/explain.slt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -646,11 +646,11 @@ logical_plan
646646
02)--{
647647
03)----"Plan": {
648648
04)------"Node Type": "Values",
649-
05)------"Output": [
650-
06)--------"column1"
651-
07)------],
652-
08)------"Plans": [],
653-
09)------"Values": "(Int64(1))"
649+
05)------"Values": "(Int64(1))",
650+
06)------"Plans": [],
651+
07)------"Output": [
652+
08)--------"column1"
653+
09)------]
654654
10)----}
655655
11)--}
656656
12)]

‎datafusion/substrait/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ object_store = { workspace = true }
4444
# We need to match the version in substrait, so we don't use the workspace version here
4545
pbjson-types = { version = "0.8.0" }
4646
prost = { workspace = true }
47-
substrait = { version = "0.62", features = ["serde"] }
47+
substrait = { version = "0.63.0", features = ["serde"] }
4848
url = { workspace = true }
4949
tokio = { workspace = true, features = ["fs"] }
5050

‎datafusion/substrait/src/extensions.rs‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -113,14 +113,10 @@ impl TryFrom<&Vec<SimpleExtensionDeclaration>> for Extensions {
113113
}
114114

115115
impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
116-
// Silence deprecation warnings for `extension_uri_reference` during the uri -> urn migration
117-
// See: https://github.com/substrait-io/substrait/issues/856
118-
#[expect(deprecated)]
119116
fn from(val: Extensions) -> Vec<SimpleExtensionDeclaration> {
120117
let mut extensions = vec![];
121118
for (f_anchor, f_name) in val.functions {
122119
let function_extension = ExtensionFunction {
123-
extension_uri_reference: u32::MAX,
124120
extension_urn_reference: u32::MAX,
125121
function_anchor: f_anchor,
126122
name: f_name,
@@ -133,7 +129,6 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
133129

134130
for (t_anchor, t_name) in val.types {
135131
let type_extension = ExtensionType {
136-
extension_uri_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545
137132
extension_urn_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545
138133
type_anchor: t_anchor,
139134
name: t_name,
@@ -146,8 +141,7 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
146141

147142
for (tv_anchor, tv_name) in val.type_variations {
148143
let type_variation_extension = ExtensionTypeVariation {
149-
extension_uri_reference: u32::MAX, // We don't register proper extension URIs yet
150-
extension_urn_reference: u32::MAX, // We don't register proper extension URIs yet
144+
extension_urn_reference: u32::MAX, // We don't register proper extension URNs yet
151145
type_variation_anchor: tv_anchor,
152146
name: tv_name,
153147
};

‎datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ pub(crate) fn from_substrait_field_reference(
5656
Some(RootType::Expression(_)) => not_impl_err!(
5757
"Expression root type in field reference is not supported"
5858
),
59+
Some(RootType::LambdaParameterReference(_)) => not_impl_err!(
60+
"Lambda parameter reference in field reference is not yet supported"
61+
),
5962
}
6063
}
6164
_ => not_impl_err!(

‎datafusion/substrait/src/logical_plan/consumer/expr/literal.rs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ use prost::Message;
4343
use std::sync::Arc;
4444
use substrait::proto;
4545
use substrait::proto::expression::Literal;
46-
use substrait::proto::expression::literal::user_defined::Val;
46+
use substrait::proto::expression::literal::user_defined::{TypeAnchorType, Val};
4747
use substrait::proto::expression::literal::{
4848
IntervalCompound, IntervalDayToSecond, IntervalYearToMonth, LiteralType,
4949
interval_day_to_second,
@@ -474,11 +474,17 @@ pub(crate) fn from_substrait_literal(
474474
)))
475475
};
476476

477-
if let Some(name) = consumer
478-
.get_extensions()
479-
.types
480-
.get(&user_defined.type_reference)
481-
{
477+
let type_ref = match user_defined.type_anchor_type {
478+
Some(TypeAnchorType::TypeReference(ref_val)) => ref_val,
479+
Some(TypeAnchorType::TypeAliasReference(_)) => {
480+
return not_impl_err!(
481+
"Type alias references in user-defined literals are not yet supported"
482+
);
483+
}
484+
None => 0,
485+
};
486+
487+
if let Some(name) = consumer.get_extensions().types.get(&type_ref) {
482488
match name.as_ref() {
483489
FLOAT_16_TYPE_NAME => {
484490
// Rules for encoding fp16 Substrait literals are defined as part of Arrow here:
@@ -518,14 +524,14 @@ pub(crate) fn from_substrait_literal(
518524
_ => {
519525
return not_impl_err!(
520526
"Unsupported Substrait user defined type with ref {} and name {}",
521-
user_defined.type_reference,
527+
type_ref,
522528
name
523529
);
524530
}
525531
}
526532
} else {
527533
#[expect(deprecated)]
528-
match user_defined.type_reference {
534+
match type_ref {
529535
// Kept for backwards compatibility, producers should useIntervalYearToMonth instead
530536
INTERVAL_YEAR_MONTH_TYPE_REF => {
531537
let Some(Val::Value(raw_val)) = user_defined.val.as_ref() else {
@@ -568,7 +574,7 @@ pub(crate) fn from_substrait_literal(
568574
_ => {
569575
return not_impl_err!(
570576
"Unsupported Substrait user defined type literal with ref {}",
571-
user_defined.type_reference
577+
type_ref
572578
);
573579
}
574580
}

‎datafusion/substrait/src/logical_plan/consumer/expr/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ pub async fn from_substrait_rex(
9393
RexType::DynamicParameter(expr) => {
9494
consumer.consume_dynamic_parameter(expr, input_schema).await
9595
}
96+
RexType::Lambda(_) | RexType::LambdaInvocation(_) => {
97+
not_impl_err!("Lambda expressions are not yet supported")
98+
}
9699
},
97100
None => substrait_err!("Expression must set rex_type: {expression:?}"),
98101
}

‎datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,15 @@ use substrait::proto::{
141141
///
142142
/// // and user-defined literals
143143
/// fn consume_user_defined_literal(&self, literal: &proto::expression::literal::UserDefined) -> Result<ScalarValue> {
144-
/// let type_string = self.extensions.types.get(&literal.type_reference).unwrap();
144+
/// // extract type_reference from the new TypeAnchorType oneof
145+
/// let type_ref = match literal.type_anchor_type {
146+
/// Some(proto::expression::literal::user_defined::TypeAnchorType::TypeReference(r)) => r,
147+
/// Some(proto::expression::literal::user_defined::TypeAnchorType::TypeAliasReference(_)) => {
148+
/// return not_impl_err!("Type alias references are not yet supported")
149+
/// }
150+
/// None => 0,
151+
/// };
152+
/// let type_string = self.extensions.types.get(&type_ref).unwrap();
145153
/// match type_string.as_str() {
146154
/// "u!foo" => not_impl_err!("handle foo conversion"),
147155
/// "u!bar" => not_impl_err!("handle bar conversion"),
@@ -444,10 +452,22 @@ pub trait SubstraitConsumer: Send + Sync + Sized {
444452
&self,
445453
user_defined_literal: &proto::expression::literal::UserDefined,
446454
) -> datafusion::common::Result<ScalarValue> {
447-
substrait_err!(
448-
"Missing handler for user-defined literals {}",
449-
user_defined_literal.type_reference
450-
)
455+
let type_ref = match user_defined_literal.type_anchor_type {
456+
Some(
457+
proto::expression::literal::user_defined::TypeAnchorType::TypeReference(
458+
ref_val,
459+
),
460+
) => ref_val,
461+
Some(
462+
proto::expression::literal::user_defined::TypeAnchorType::TypeAliasReference(_),
463+
) => {
464+
return not_impl_err!(
465+
"Type alias references in user-defined literals are not yet supported"
466+
)
467+
}
468+
None => 0,
469+
};
470+
substrait_err!("Missing handler for user-defined literals {}", type_ref)
451471
}
452472
}
453473

‎datafusion/substrait/src/logical_plan/producer/expr/literal.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ pub(crate) fn to_substrait_literal(
117117
(
118118
LiteralType::UserDefined(
119119
substrait::proto::expression::literal::UserDefined {
120-
type_reference: type_anchor,
120+
type_anchor_type: Some(substrait::proto::expression::literal::user_defined::TypeAnchorType::TypeReference(type_anchor)),
121121
type_parameters: vec![],
122122
val: Some(substrait::proto::expression::literal::user_defined::Val::Value(
123123
pbjson_types::Any {

0 commit comments

Comments
 (0)