From 906442974e5f582f79df2a2f279ea3e79ee80fe0 Mon Sep 17 00:00:00 2001 From: JoshuaTang <1240604020@qq.com> Date: Mon, 3 Nov 2025 14:38:20 -0800 Subject: [PATCH 1/2] feat: optimize the performance of datafusion planner --- rust/lance-graph/src/datafusion_planner.rs | 48 ++++++++++++++++------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/rust/lance-graph/src/datafusion_planner.rs b/rust/lance-graph/src/datafusion_planner.rs index 7cd5b41f..0be300cf 100644 --- a/rust/lance-graph/src/datafusion_planner.rs +++ b/rust/lance-graph/src/datafusion_planner.rs @@ -508,26 +508,50 @@ impl DataFusionPlanner { self.plan_error(&format!("Failed to scan node source '{}'", label), e) })?; - // Apply property filters using unqualified names (before aliasing) - for (k, v) in properties.iter() { - let lit_expr = - self.to_df_value_expr(&crate::ast::ValueExpression::Literal(v.clone())); - let filter_expr = Expr::BinaryExpr(BinaryExpr { - left: Box::new(col(k)), - op: Operator::Eq, - right: Box::new(lit_expr), - }); - builder = builder.filter(filter_expr).map_err(|e| { - self.plan_error(&format!("Failed to apply filter on property '{}'", k), e) + // Combine property filters into single predicate for efficiency + if !properties.is_empty() { + let filter_exprs: Vec = properties + .iter() + .map(|(k, v)| { + let lit_expr = self.to_df_value_expr( + &crate::ast::ValueExpression::Literal(v.clone()), + ); + Expr::BinaryExpr(BinaryExpr { + left: Box::new(col(k)), + op: Operator::Eq, + right: Box::new(lit_expr), + }) + }) + .collect(); + + // Combine with AND if multiple filters + let combined_filter = filter_exprs + .into_iter() + .reduce(|acc, expr| { + Expr::BinaryExpr(BinaryExpr { + left: Box::new(acc), + op: Operator::And, + right: Box::new(expr), + }) + }) + .unwrap(); + + builder = builder.filter(combined_filter).map_err(|e| { + self.plan_error("Failed to apply property filters", e) })?; } // Create qualified column aliases: variable__property + // Optimization: Pre-allocate string capacity to reduce allocations let qualified_exprs: Vec = schema .fields() .iter() .map(|field| { - let qualified_name = format!("{}__{}", variable, field.name()); + let mut qualified_name = + String::with_capacity(variable.len() + field.name().len() + 2); + qualified_name.push_str(variable); + qualified_name.push_str("__"); + qualified_name.push_str(field.name()); col(field.name()).alias(&qualified_name) }) .collect(); From 6e15531cebe10260a3c601d35c573246d8ad8460 Mon Sep 17 00:00:00 2001 From: JoshuaTang <1240604020@qq.com> Date: Mon, 3 Nov 2025 19:24:25 -0800 Subject: [PATCH 2/2] format code --- rust/lance-graph/src/datafusion_planner.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/lance-graph/src/datafusion_planner.rs b/rust/lance-graph/src/datafusion_planner.rs index 0be300cf..162b7c76 100644 --- a/rust/lance-graph/src/datafusion_planner.rs +++ b/rust/lance-graph/src/datafusion_planner.rs @@ -513,9 +513,8 @@ impl DataFusionPlanner { let filter_exprs: Vec = properties .iter() .map(|(k, v)| { - let lit_expr = self.to_df_value_expr( - &crate::ast::ValueExpression::Literal(v.clone()), - ); + let lit_expr = self + .to_df_value_expr(&crate::ast::ValueExpression::Literal(v.clone())); Expr::BinaryExpr(BinaryExpr { left: Box::new(col(k)), op: Operator::Eq, @@ -536,9 +535,9 @@ impl DataFusionPlanner { }) .unwrap(); - builder = builder.filter(combined_filter).map_err(|e| { - self.plan_error("Failed to apply property filters", e) - })?; + builder = builder + .filter(combined_filter) + .map_err(|e| self.plan_error("Failed to apply property filters", e))?; } // Create qualified column aliases: variable__property