From 43931b50eee0d282994c8d0114d0f92724181f50 Mon Sep 17 00:00:00 2001 From: likun Date: Sun, 5 Jul 2026 21:39:17 +0800 Subject: [PATCH] GH-50371: [C++][Gandiva] Fold common subexpressions --- cpp/src/gandiva/CMakeLists.txt | 1 + cpp/src/gandiva/engine.cc | 10 + cpp/src/gandiva/engine.h | 4 + cpp/src/gandiva/expr_cse.cc | 273 ++++++++++++++++++++++++ cpp/src/gandiva/expr_cse.h | 35 +++ cpp/src/gandiva/expr_decomposer.cc | 84 ++++++-- cpp/src/gandiva/expr_decomposer.h | 13 +- cpp/src/gandiva/filter.cc | 11 +- cpp/src/gandiva/llvm_generator.h | 1 + cpp/src/gandiva/projector.cc | 19 +- cpp/src/gandiva/projector.h | 2 + cpp/src/gandiva/tests/projector_test.cc | 224 +++++++++++++++++++ 12 files changed, 644 insertions(+), 33 deletions(-) create mode 100644 cpp/src/gandiva/expr_cse.cc create mode 100644 cpp/src/gandiva/expr_cse.h diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index aabe4ec8bf7..349e15e7799 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -65,6 +65,7 @@ set(SRC_FILES engine.cc date_utils.cc encrypt_utils.cc + expr_cse.cc expr_decomposer.cc expr_validator.cc expression.cc diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 901421c86cb..4cdc9238f67 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -543,6 +543,10 @@ Status Engine::FinalizeModule() { if (!cached_) { ARROW_RETURN_NOT_OK(RemoveUnusedFunctions()); + if (conf_->dump_ir()) { + unoptimized_module_ir_ = DumpModuleIR(*module_); + } + if (optimize_) { auto target_analysis = target_machine_->getTargetIRAnalysis(); // misc passes to allow for inlining, vectorization, .. @@ -615,4 +619,10 @@ const std::string& Engine::ir() { return module_ir_; } +const std::string& Engine::unoptimized_ir() { + DCHECK(!unoptimized_module_ir_.empty()) + << "dump_ir in Configuration must be set for dumping IR"; + return unoptimized_module_ir_; +} + } // namespace gandiva diff --git a/cpp/src/gandiva/engine.h b/cpp/src/gandiva/engine.h index 20165787cb6..2d4035db5b3 100644 --- a/cpp/src/gandiva/engine.h +++ b/cpp/src/gandiva/engine.h @@ -87,6 +87,9 @@ class GANDIVA_EXPORT Engine { /// Return the generated IR for the module. const std::string& ir(); + /// Return the generated IR before the optimizer pipeline runs. + const std::string& unoptimized_ir(); + /// Load the function IRs that can be accessed in the module. Status LoadFunctionIRs(); @@ -129,6 +132,7 @@ class GANDIVA_EXPORT Engine { bool cached_; bool functions_loaded_ = false; std::shared_ptr function_registry_; + std::string unoptimized_module_ir_; std::string module_ir_; // The lifetime of the TargetMachine is shared with LLJIT. This prevents unnecessary // duplication of this expensive object. diff --git a/cpp/src/gandiva/expr_cse.cc b/cpp/src/gandiva/expr_cse.cc new file mode 100644 index 00000000000..bf8524b0230 --- /dev/null +++ b/cpp/src/gandiva/expr_cse.cc @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/expr_cse.h" + +#include +#include +#include +#include +#include +#include + +#include "gandiva/condition.h" +#include "gandiva/function_registry.h" +#include "gandiva/function_signature.h" +#include "gandiva/node.h" + +namespace gandiva { + +namespace { + +struct FoldedNode { + NodePtr node; + std::string key; + bool can_eliminate; +}; + +class CommonSubexpressionFolder { + public: + explicit CommonSubexpressionFolder(const FunctionRegistry& registry) + : registry_(registry) {} + + ExpressionPtr FoldExpression(const ExpressionPtr& expression) { + auto folded = Fold(expression->root()); + if (folded.node == expression->root()) { + return expression; + } + return std::make_shared(folded.node, expression->result()); + } + + ConditionPtr FoldCondition(const ConditionPtr& condition) { + auto folded = Fold(condition->root()); + if (folded.node == condition->root()) { + return condition; + } + return std::make_shared(folded.node); + } + + private: + FoldedNode Fold(const NodePtr& node) { + if (node == nullptr) { + return {nullptr, "null", false}; + } + + if (auto field_node = std::dynamic_pointer_cast(node)) { + auto key = "field:" + field_node->field()->ToString(); + return {Intern(key, node), std::move(key), true}; + } + + if (auto literal_node = std::dynamic_pointer_cast(node)) { + auto key = "literal:" + literal_node->ToString(); + return {Intern(key, node), std::move(key), true}; + } + + if (auto function_node = std::dynamic_pointer_cast(node)) { + return FoldFunction(node, *function_node); + } + + if (auto boolean_node = std::dynamic_pointer_cast(node)) { + return FoldBoolean(node, *boolean_node); + } + + if (auto if_node = std::dynamic_pointer_cast(node)) { + return FoldIf(node, *if_node); + } + + // InExpressionNode stores constants in unordered_sets, so ToString() is not a + // stable structural key. Keep it opaque in this conservative pass. + std::stringstream ss; + ss << "opaque:" << node.get(); + return {node, ss.str(), false}; + } + + FoldedNode FoldFunction(const NodePtr& original, const FunctionNode& function_node) { + NodeVector children; + children.reserve(function_node.children().size()); + + std::vector child_keys; + child_keys.reserve(function_node.children().size()); + + bool children_unchanged = true; + bool children_can_eliminate = true; + for (const auto& child : function_node.children()) { + auto folded = Fold(child); + children_unchanged = children_unchanged && folded.node == child; + children_can_eliminate = children_can_eliminate && folded.can_eliminate; + children.push_back(folded.node); + child_keys.push_back(std::move(folded.key)); + } + + auto desc = function_node.descriptor(); + auto return_type = desc->return_type() == NULLPTR ? std::string("untyped") + : desc->return_type()->ToString(); + auto key = JoinKey("function", desc->name(), return_type, child_keys); + auto folded_node = + children_unchanged + ? original + : std::make_shared(desc->name(), children, desc->return_type()); + bool can_eliminate = children_can_eliminate && IsFunctionSafe(function_node); + return {can_eliminate ? Intern(key, folded_node) : folded_node, std::move(key), + can_eliminate}; + } + + FoldedNode FoldBoolean(const NodePtr& original, const BooleanNode& boolean_node) { + NodeVector folded_children; + std::vector folded_keys; + std::unordered_set seen_eliminable_children; + bool children_unchanged = true; + bool children_can_eliminate = true; + + for (const auto& child : boolean_node.children()) { + auto folded = Fold(child); + children_unchanged = children_unchanged && folded.node == child; + AppendBooleanChild(boolean_node.expr_type(), std::move(folded), + &seen_eliminable_children, &folded_children, &folded_keys, + &children_unchanged, &children_can_eliminate); + } + + if (boolean_node.children().size() > 1 && folded_children.size() == 1 && + children_can_eliminate) { + return {folded_children[0], folded_keys[0], true}; + } + + auto op = boolean_node.expr_type() == BooleanNode::AND ? "and" : "or"; + auto key = JoinKey("boolean", op, "bool", folded_keys); + auto folded_node = + children_unchanged && folded_children.size() == boolean_node.children().size() + ? original + : std::make_shared(boolean_node.expr_type(), folded_children); + return {children_can_eliminate ? Intern(key, folded_node) : folded_node, + std::move(key), children_can_eliminate}; + } + + void AppendBooleanChild(BooleanNode::ExprType expr_type, FoldedNode folded, + std::unordered_set* seen_eliminable_children, + NodeVector* folded_children, + std::vector* folded_keys, bool* children_unchanged, + bool* children_can_eliminate) { + auto nested_boolean = std::dynamic_pointer_cast(folded.node); + if (nested_boolean != nullptr && nested_boolean->expr_type() == expr_type && + folded.can_eliminate) { + *children_unchanged = false; + for (const auto& nested_child : nested_boolean->children()) { + auto nested_folded = Fold(nested_child); + AppendBooleanChild(expr_type, std::move(nested_folded), seen_eliminable_children, + folded_children, folded_keys, children_unchanged, + children_can_eliminate); + } + return; + } + + if (folded.can_eliminate) { + if (!seen_eliminable_children->insert(folded.key).second) { + *children_unchanged = false; + return; + } + } else { + *children_can_eliminate = false; + } + + folded_children->push_back(folded.node); + folded_keys->push_back(std::move(folded.key)); + } + + FoldedNode FoldIf(const NodePtr& original, const IfNode& if_node) { + auto condition = Fold(if_node.condition()); + auto then_node = Fold(if_node.then_node()); + auto else_node = Fold(if_node.else_node()); + + if (condition.can_eliminate && then_node.can_eliminate && else_node.can_eliminate && + then_node.key == else_node.key) { + return then_node; + } + + std::vector child_keys{condition.key, then_node.key, else_node.key}; + auto key = JoinKey("if", "", if_node.return_type()->ToString(), child_keys); + bool children_unchanged = condition.node == if_node.condition() && + then_node.node == if_node.then_node() && + else_node.node == if_node.else_node(); + auto folded_node = children_unchanged ? original + : std::make_shared( + condition.node, then_node.node, + else_node.node, if_node.return_type()); + + return {folded_node, std::move(key), false}; + } + + bool IsFunctionSafe(const FunctionNode& node) const { + auto desc = node.descriptor(); + FunctionSignature signature(desc->name(), desc->params(), desc->return_type()); + const NativeFunction* native_function = registry_.LookupSignature(signature); + if (native_function == nullptr) { + return false; + } + return native_function->result_nullable_type() != kResultNullInternal && + !native_function->NeedsContext() && !native_function->NeedsFunctionHolder() && + !native_function->CanReturnErrors(); + } + + NodePtr Intern(const std::string& key, const NodePtr& node) { + auto it = canonical_nodes_.find(key); + if (it != canonical_nodes_.end()) { + return it->second; + } + canonical_nodes_.emplace(key, node); + return node; + } + + std::string JoinKey(const std::string& kind, const std::string& name, + const std::string& return_type, + const std::vector& child_keys) const { + std::stringstream ss; + ss << kind << ":" << name << ":" << return_type << "("; + bool first = true; + for (const auto& child_key : child_keys) { + if (!first) { + ss << ","; + } + ss << child_key.size() << ":" << child_key; + first = false; + } + ss << ")"; + return ss.str(); + } + + const FunctionRegistry& registry_; + std::unordered_map canonical_nodes_; +}; + +} // namespace + +ExpressionVector FoldCommonSubexpressions(const FunctionRegistry& registry, + const ExpressionVector& expressions) { + CommonSubexpressionFolder folder(registry); + ExpressionVector folded_expressions; + folded_expressions.reserve(expressions.size()); + for (const auto& expression : expressions) { + folded_expressions.push_back(folder.FoldExpression(expression)); + } + return folded_expressions; +} + +ConditionPtr FoldCommonSubexpressions(const FunctionRegistry& registry, + const ConditionPtr& condition) { + CommonSubexpressionFolder folder(registry); + return folder.FoldCondition(condition); +} + +} // namespace gandiva diff --git a/cpp/src/gandiva/expr_cse.h b/cpp/src/gandiva/expr_cse.h new file mode 100644 index 00000000000..69664cfecdd --- /dev/null +++ b/cpp/src/gandiva/expr_cse.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "gandiva/expression.h" +#include "gandiva/gandiva_aliases.h" +#include "gandiva/visibility.h" + +namespace gandiva { + +class Condition; +class FunctionRegistry; + +GANDIVA_EXPORT ExpressionVector FoldCommonSubexpressions( + const FunctionRegistry& registry, const ExpressionVector& expressions); + +GANDIVA_EXPORT ConditionPtr FoldCommonSubexpressions(const FunctionRegistry& registry, + const ConditionPtr& condition); + +} // namespace gandiva diff --git a/cpp/src/gandiva/expr_decomposer.cc b/cpp/src/gandiva/expr_decomposer.cc index 921829db6a9..eaf44993416 100644 --- a/cpp/src/gandiva/expr_decomposer.cc +++ b/cpp/src/gandiva/expr_decomposer.cc @@ -61,6 +61,52 @@ const FunctionNode ExprDecomposer::TryOptimize(const FunctionNode& node) { } } +Status ExprDecomposer::DecomposeNode(const Node& node, ValueValidityPairPtr* out) { + bool can_reuse = CanReuseDecomposition(node); + if (can_reuse) { + auto it = decomposed_cache_.find(&node); + if (it != decomposed_cache_.end()) { + *out = it->second; + return Status::OK(); + } + } + + ARROW_RETURN_NOT_OK(node.Accept(*this)); + *out = std::move(result_); + if (can_reuse) { + decomposed_cache_.emplace(&node, *out); + } + return Status::OK(); +} + +bool ExprDecomposer::CanReuseDecomposition(const Node& node) { + auto cached = reusable_node_cache_.find(&node); + if (cached != reusable_node_cache_.end()) { + return cached->second; + } + + bool reusable = false; + if (dynamic_cast(&node) != nullptr || + dynamic_cast(&node) != nullptr) { + reusable = true; + } else if (auto function_node = dynamic_cast(&node)) { + auto desc = function_node->descriptor(); + FunctionSignature signature(desc->name(), desc->params(), desc->return_type()); + const NativeFunction* native_function = registry_.LookupSignature(signature); + reusable = native_function != nullptr && + native_function->result_nullable_type() != kResultNullInternal && + !native_function->NeedsContext() && + !native_function->NeedsFunctionHolder() && + !native_function->CanReturnErrors(); + for (const auto& child : function_node->children()) { + reusable = reusable && CanReuseDecomposition(*child); + } + } + + reusable_node_cache_.emplace(&node, reusable); + return reusable; +} + // Decompose a field node - wherever possible, merge the validity vectors of the // child nodes. Status ExprDecomposer::Visit(const FunctionNode& in_node) { @@ -73,10 +119,9 @@ Status ExprDecomposer::Visit(const FunctionNode& in_node) { // decompose the children. std::vector args; for (auto& child : node.children()) { - auto status = child->Accept(*this); - ARROW_RETURN_NOT_OK(status); - - args.push_back(result()); + ValueValidityPairPtr child_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*child, &child_vv)); + args.push_back(child_vv); } // Make a function holder, if required. @@ -130,24 +175,24 @@ Status ExprDecomposer::Visit(const IfNode& node) { nested_if_else_ = false; PushConditionEntry(node); - auto status = node.condition()->Accept(*this); + ValueValidityPairPtr condition_vv; + auto status = DecomposeNode(*node.condition(), &condition_vv); ARROW_RETURN_NOT_OK(status); - auto condition_vv = result(); PopConditionEntry(node); // Add a local bitmap to track the output validity. int local_bitmap_idx = PushThenEntry(node, svd_nested_if_else); - status = node.then_node()->Accept(*this); + ValueValidityPairPtr then_vv; + status = DecomposeNode(*node.then_node(), &then_vv); ARROW_RETURN_NOT_OK(status); - auto then_vv = result(); PopThenEntry(node); PushElseEntry(node, local_bitmap_idx); nested_if_else_ = (dynamic_cast(node.else_node().get()) != nullptr); - status = node.else_node()->Accept(*this); + ValueValidityPairPtr else_vv; + status = DecomposeNode(*node.else_node(), &else_vv); ARROW_RETURN_NOT_OK(status); - auto else_vv = result(); bool is_terminal_else = PopElseEntry(node); auto validity_dex = std::make_shared(local_bitmap_idx); @@ -164,10 +209,9 @@ Status ExprDecomposer::Visit(const BooleanNode& node) { // decompose the children. std::vector args; for (auto& child : node.children()) { - auto status = child->Accept(*this); - ARROW_RETURN_NOT_OK(status); - - args.push_back(result()); + ValueValidityPairPtr child_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*child, &child_vv)); + args.push_back(child_vv); } // Add a local bitmap to track the output validity. @@ -190,9 +234,9 @@ Status ExprDecomposer::Visit(const BooleanNode& node) { Status ExprDecomposer::Visit(const InExpressionNode& node) { /* decompose the children. */ std::vector args; - auto status = node.eval_expr()->Accept(*this); - ARROW_RETURN_NOT_OK(status); - args.push_back(result()); + ValueValidityPairPtr eval_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*node.eval_expr(), &eval_vv)); + args.push_back(eval_vv); /* In always outputs valid results, so no validity dex */ auto value_dex = std::make_shared>( args, node.values(), node.get_precision(), node.get_scale()); @@ -206,9 +250,9 @@ template Status ExprDecomposer::VisitInGeneric(const InExpressionNode& node) { /* decompose the children. */ std::vector args; - auto status = node.eval_expr()->Accept(*this); - ARROW_RETURN_NOT_OK(status); - args.push_back(result()); + ValueValidityPairPtr eval_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*node.eval_expr(), &eval_vv)); + args.push_back(eval_vv); /* In always outputs valid results, so no validity dex */ auto value_dex = std::make_shared>(args, node.values()); int holder_idx = annotator_.AddHolderPointer(value_dex->in_holder().get()); diff --git a/cpp/src/gandiva/expr_decomposer.h b/cpp/src/gandiva/expr_decomposer.h index 90a27744b36..e1d39970905 100644 --- a/cpp/src/gandiva/expr_decomposer.h +++ b/cpp/src/gandiva/expr_decomposer.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "gandiva/arrow.h" @@ -42,11 +43,7 @@ class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor { : registry_(registry), annotator_(annotator), nested_if_else_(false) {} Status Decompose(const Node& root, ValueValidityPairPtr* out) { - auto status = root.Accept(*this); - if (status.ok()) { - *out = std::move(result_); - } - return status; + return DecomposeNode(root, out); } private: @@ -75,6 +72,10 @@ class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor { template Status VisitInGeneric(const InExpressionNode& node); + Status DecomposeNode(const Node& node, ValueValidityPairPtr* out); + + bool CanReuseDecomposition(const Node& node); + // Optimize a function node, if possible. const FunctionNode TryOptimize(const FunctionNode& node); @@ -125,6 +126,8 @@ class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor { Annotator& annotator_; std::stack> if_entries_stack_; ValueValidityPairPtr result_; + std::unordered_map decomposed_cache_; + std::unordered_map reusable_node_cache_; bool nested_if_else_; }; diff --git a/cpp/src/gandiva/filter.cc b/cpp/src/gandiva/filter.cc index 8a270cfdc06..ed49ec37acf 100644 --- a/cpp/src/gandiva/filter.cc +++ b/cpp/src/gandiva/filter.cc @@ -23,6 +23,7 @@ #include "gandiva/bitmap_accumulator.h" #include "gandiva/cache.h" #include "gandiva/condition.h" +#include "gandiva/expr_cse.h" #include "gandiva/expr_validator.h" #include "gandiva/llvm_generator.h" #include "gandiva/selection_vector_impl.h" @@ -45,10 +46,13 @@ Status Filter::Make(SchemaPtr schema, ConditionPtr condition, ARROW_RETURN_IF(configuration == nullptr, Status::Invalid("Configuration cannot be null")); + auto folded_condition = + FoldCommonSubexpressions(*configuration->function_registry(), condition); + std::shared_ptr>> cache = LLVMGenerator::GetCache(); - Condition conditionToKey = *(condition.get()); + Condition conditionToKey = *(folded_condition.get()); ExpressionCacheKey cache_key(schema, configuration, conditionToKey); @@ -73,13 +77,14 @@ Status Filter::Make(SchemaPtr schema, ConditionPtr condition, // Return if the expression is invalid since we will not be able to process further. ExprValidator expr_validator(llvm_gen->types(), schema, configuration->function_registry()); - ARROW_RETURN_NOT_OK(expr_validator.Validate(condition)); + ARROW_RETURN_NOT_OK(expr_validator.Validate(folded_condition)); } // Set the object cache for LLVM ARROW_RETURN_NOT_OK(llvm_gen->SetLLVMObjectCache(obj_cache)); - ARROW_RETURN_NOT_OK(llvm_gen->Build({condition}, SelectionVector::Mode::MODE_NONE)); + ARROW_RETURN_NOT_OK( + llvm_gen->Build({folded_condition}, SelectionVector::Mode::MODE_NONE)); // Instantiate the filter with the completely built llvm generator *filter = std::make_shared(std::move(llvm_gen), schema, configuration); diff --git a/cpp/src/gandiva/llvm_generator.h b/cpp/src/gandiva/llvm_generator.h index a60e2bf6b29..485b2466e49 100644 --- a/cpp/src/gandiva/llvm_generator.h +++ b/cpp/src/gandiva/llvm_generator.h @@ -84,6 +84,7 @@ class GANDIVA_EXPORT LLVMGenerator { LLVMTypes* types() { return engine_->types(); } llvm::Module* module() { return engine_->module(); } const std::string& ir() { return engine_->ir(); } + const std::string& unoptimized_ir() { return engine_->unoptimized_ir(); } private: explicit LLVMGenerator(bool cached, diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc index ec0302146ff..5f5419852ea 100644 --- a/cpp/src/gandiva/projector.cc +++ b/cpp/src/gandiva/projector.cc @@ -24,6 +24,7 @@ #include "arrow/util/logging.h" #include "gandiva/cache.h" +#include "gandiva/expr_cse.h" #include "gandiva/expr_validator.h" #include "gandiva/llvm_generator.h" @@ -61,11 +62,15 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, ARROW_RETURN_IF(configuration == nullptr, Status::Invalid("Configuration cannot be null")); + auto folded_exprs = + FoldCommonSubexpressions(*configuration->function_registry(), exprs); + // see if equivalent projector was already built std::shared_ptr>> cache = LLVMGenerator::GetCache(); - ExpressionCacheKey cache_key(schema, configuration, exprs, selection_vector_mode); + ExpressionCacheKey cache_key(schema, configuration, folded_exprs, + selection_vector_mode); bool is_cached = false; @@ -89,7 +94,7 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, if (!is_cached) { ExprValidator expr_validator(llvm_gen->types(), schema, configuration->function_registry()); - for (auto& expr : exprs) { + for (auto& expr : folded_exprs) { ARROW_RETURN_NOT_OK(expr_validator.Validate(expr)); } } @@ -97,12 +102,12 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, // Set the object cache for LLVM ARROW_RETURN_NOT_OK(llvm_gen->SetLLVMObjectCache(obj_cache)); - ARROW_RETURN_NOT_OK(llvm_gen->Build(exprs, selection_vector_mode)); + ARROW_RETURN_NOT_OK(llvm_gen->Build(folded_exprs, selection_vector_mode)); // save the output field types. Used for validation at Evaluate() time. std::vector output_fields; - output_fields.reserve(exprs.size()); - for (auto& expr : exprs) { + output_fields.reserve(folded_exprs.size()); + for (auto& expr : folded_exprs) { output_fields.push_back(expr->result()); } @@ -283,6 +288,10 @@ Status Projector::ValidateArrayDataCapacity(const arrow::ArrayData& array_data, const std::string& Projector::DumpIR() { return llvm_generator_->ir(); } +const std::string& Projector::DumpUnoptimizedIR() { + return llvm_generator_->unoptimized_ir(); +} + void Projector::SetBuiltFromCache(bool flag) { built_from_cache_ = flag; } bool Projector::GetBuiltFromCache() { return built_from_cache_; } diff --git a/cpp/src/gandiva/projector.h b/cpp/src/gandiva/projector.h index f1ae7e4dc8c..2fa1e56b35a 100644 --- a/cpp/src/gandiva/projector.h +++ b/cpp/src/gandiva/projector.h @@ -120,6 +120,8 @@ class GANDIVA_EXPORT Projector { const std::string& DumpIR(); + const std::string& DumpUnoptimizedIR(); + void SetBuiltFromCache(bool flag); bool GetBuiltFromCache(); diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 268cb55a642..c92b50950b5 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -24,6 +24,7 @@ #include #include +#include #include "arrow/memory_pool.h" #include "gandiva/function_registry.h" @@ -39,6 +40,40 @@ using arrow::float32; using arrow::int32; using arrow::int64; +namespace { + +int CountOccurrences(const std::string& text, const std::string& needle) { + int count = 0; + std::string::size_type pos = 0; + while ((pos = text.find(needle, pos)) != std::string::npos) { + ++count; + pos += needle.size(); + } + return count; +} + +int CountInt32AddInstructions(const std::string& ir) { + return CountOccurrences(ir, " add i32 ") + CountOccurrences(ir, " add nsw i32 ") + + CountOccurrences(ir, " add nuw i32 ") + + CountOccurrences(ir, " add nuw nsw i32 "); +} + +std::string ExtractFunctionIR(const std::string& ir, const std::string& function_name) { + const auto name_pos = ir.find("@" + function_name + "("); + if (name_pos == std::string::npos) { + return ""; + } + const auto function_start = ir.rfind("\ndefine ", name_pos); + const auto start = function_start == std::string::npos ? 0 : function_start + 1; + const auto function_end = ir.find("\n}\n", name_pos); + if (function_end == std::string::npos) { + return ir.substr(start); + } + return ir.substr(start, function_end + 3 - start); +} + +} // namespace + class TestProjector : public ::testing::Test { public: void SetUp() { @@ -388,6 +423,195 @@ TEST_F(TestProjector, TestAllIntTypes) { TestArithmeticOpsForType(pool_); } +TEST_F(TestProjector, TestCommonSubexpressionEliminationIR) { + auto field0 = arrow::field("cse_f0", arrow::int32()); + auto field1 = arrow::field("cse_f1", arrow::int32()); + auto schema = arrow::schema({field0, field1}); + + auto left_sum = TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}, + arrow::int32()); + auto right_sum = TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}, + arrow::int32()); + auto square = + TreeExprBuilder::MakeFunction("multiply", {left_sum, right_sum}, arrow::int32()); + auto expr = + TreeExprBuilder::MakeExpression(square, arrow::field("cse_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); + EXPECT_EQ(1, CountInt32AddInstructions(optimized_expr_ir)); +} + +TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { + auto field0 = arrow::field("nested_cse_f0", arrow::int32()); + auto field1 = arrow::field("nested_cse_f1", arrow::int32()); + auto schema = arrow::schema({field0, field1}); + + auto make_sum = [&]() { + return TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}, + arrow::int32()); + }; + auto make_square = [&]() { + return TreeExprBuilder::MakeFunction("multiply", {make_sum(), make_sum()}, + arrow::int32()); + }; + auto nested = TreeExprBuilder::MakeFunction("add", {make_square(), make_square()}, + arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + nested, arrow::field("nested_cse_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_EQ(5, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); +} + +TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { + auto condition_field = arrow::field("generated_if_cond", arrow::boolean()); + auto value_field = arrow::field("generated_if_value", arrow::int32()); + auto schema = arrow::schema({condition_field, value_field}); + + auto condition = TreeExprBuilder::MakeField(condition_field); + auto then_node = TreeExprBuilder::MakeField(value_field); + auto else_node = TreeExprBuilder::MakeField(value_field); + auto if_node = TreeExprBuilder::MakeIf(condition, then_node, else_node, arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + if_node, arrow::field("generated_if_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("generated_if_value")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("generated_if_cond")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("then:")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("else:")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("validAndMatch")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("res_value = phi")); + EXPECT_NE(std::string::npos, optimized_expr_ir.find("generated_if_value")); + EXPECT_EQ(std::string::npos, optimized_expr_ir.find("generated_if_cond")); + EXPECT_EQ(std::string::npos, optimized_expr_ir.find("validAndMatch")); + EXPECT_EQ(std::string::npos, optimized_expr_ir.find("res_value = phi")); +} + +TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { + auto field0 = arrow::field("generated_bool_f0", arrow::boolean()); + auto field1 = arrow::field("generated_bool_f1", arrow::boolean()); + auto schema = arrow::schema({field0, field1}); + + auto make_and = [&]() { + return TreeExprBuilder::MakeAnd( + {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}); + }; + auto bool_expr = TreeExprBuilder::MakeOr({make_and(), make_and()}); + auto expr = TreeExprBuilder::MakeExpression( + bool_expr, arrow::field("generated_bool_out", arrow::boolean())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "short_circuit"), 0); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "non_short_circuit"), 0); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "res_value = phi"), 0); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); + EXPECT_GT(CountOccurrences(optimized_expr_ir, "generated_bool_f0"), 0); + EXPECT_GT(CountOccurrences(optimized_expr_ir, "generated_bool_f1"), 0); + EXPECT_GT(CountOccurrences(optimized_expr_ir, "\"0_lbmap\""), 0); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"1_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"2_lbmap\"")); +} + +TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { + auto value = arrow::field("nested_between_value", arrow::int32()); + auto lower = arrow::field("nested_between_lower", arrow::int32()); + auto upper = arrow::field("nested_between_upper", arrow::int32()); + auto schema = arrow::schema({value, lower, upper}); + + auto make_between = [&]() { + auto ge_lower = TreeExprBuilder::MakeFunction( + "greater_than_or_equal_to", + {TreeExprBuilder::MakeField(value), TreeExprBuilder::MakeField(lower)}, + arrow::boolean()); + auto le_upper = TreeExprBuilder::MakeFunction( + "less_than_or_equal_to", + {TreeExprBuilder::MakeField(value), TreeExprBuilder::MakeField(upper)}, + arrow::boolean()); + return TreeExprBuilder::MakeAnd({ge_lower, le_upper}); + }; + + auto nested_between = TreeExprBuilder::MakeAnd( + {make_between(), TreeExprBuilder::MakeOr({make_between(), make_between()})}); + auto expr = TreeExprBuilder::MakeExpression( + nested_between, arrow::field("nested_between_out", arrow::boolean())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, + "call i1 @greater_than_or_equal_to_int32_int32")); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, + "call i1 @less_than_or_equal_to_int32_int32")); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, + "call i1 @greater_than_or_equal_to_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, + "call i1 @less_than_or_equal_to_int32_int32")); +} + TEST_F(TestProjector, TestExtendedMath) { #ifdef __aarch64__ GTEST_SKIP() << "Failed on aarch64 with 'JIT session error: Symbols not found: [ "