Skip to content

Commit 013238f

Browse files
seawindeseawinde
authored
branch-4.0: [fix](fe) Backport null-reject MV rewrite fixes (#62492, #63268) (#63585)
pr: #62492, #63268
1 parent 901376f commit 013238f

15 files changed

Lines changed: 750 additions & 85 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java‎

Lines changed: 143 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.nereids.rules.exploration.mv;
1919

2020
import org.apache.doris.catalog.MTMV;
21+
import org.apache.doris.catalog.TableIf;
2122
import org.apache.doris.common.AnalysisException;
2223
import org.apache.doris.common.Id;
2324
import org.apache.doris.common.Pair;
@@ -29,6 +30,7 @@
2930
import org.apache.doris.nereids.CascadesContext;
3031
import org.apache.doris.nereids.StatementContext;
3132
import org.apache.doris.nereids.jobs.executor.Rewriter;
33+
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
3234
import org.apache.doris.nereids.properties.LogicalProperties;
3335
import org.apache.doris.nereids.properties.OrderKey;
3436
import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory;
@@ -43,6 +45,7 @@
4345
import org.apache.doris.nereids.rules.rewrite.MergeProjectable;
4446
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
4547
import org.apache.doris.nereids.trees.expressions.Expression;
48+
import org.apache.doris.nereids.trees.expressions.IsNull;
4649
import org.apache.doris.nereids.trees.expressions.NamedExpression;
4750
import org.apache.doris.nereids.trees.expressions.Not;
4851
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -821,21 +824,28 @@ protected SplitPredicate predicatesCompensate(
821824
Set<Set<Slot>> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot();
822825
// check query is use the null reject slot which view comparison need
823826
if (!requireNoNullableViewSlot.isEmpty()) {
827+
// Required null-reject slots are recorded on the view side. Map query slots to view slots
828+
// before checking whether query predicates or INNER JoinEdges can reject those null rows.
824829
SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
825-
// try to use
826-
boolean valid = containsNullRejectSlot(requireNoNullableViewSlot,
827-
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping, queryStructInfo,
828-
viewStructInfo, cascadesContext);
829-
if (!valid) {
830+
Optional<Set<Expression>> queryBasedNullRejectCompensationPredicates =
831+
getQueryBasedNullRejectCompensationPredicates(
832+
requireNoNullableViewSlot,
833+
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping,
834+
queryStructInfo, viewStructInfo, viewToQuerySlotMapping, cascadesContext);
835+
if (!queryBasedNullRejectCompensationPredicates.isPresent()) {
830836
queryStructInfo = queryStructInfo.withPredicates(queryStructInfo.getPredicates()
831837
.mergePulledUpPredicates(comparisonResult.getQueryAllPulledUpExpressions()));
832-
valid = containsNullRejectSlot(requireNoNullableViewSlot,
833-
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping,
834-
queryStructInfo, viewStructInfo, cascadesContext);
838+
queryBasedNullRejectCompensationPredicates = getQueryBasedNullRejectCompensationPredicates(
839+
requireNoNullableViewSlot, queryStructInfo.getPredicates().getPulledUpPredicates(),
840+
queryToViewMapping, queryStructInfo, viewStructInfo, viewToQuerySlotMapping, cascadesContext);
835841
}
836-
if (!valid) {
842+
if (!queryBasedNullRejectCompensationPredicates.isPresent()) {
837843
return SplitPredicate.INVALID_INSTANCE;
838844
}
845+
if (!queryBasedNullRejectCompensationPredicates.get().isEmpty()) {
846+
queryStructInfo = queryStructInfo.withPredicates(queryStructInfo.getPredicates()
847+
.mergePulledUpPredicates(queryBasedNullRejectCompensationPredicates.get()));
848+
}
839849
}
840850
// compensate couldNot PulledUp Conjunctions
841851
Map<Expression, ExpressionInfo> couldNotPulledUpCompensateConjunctions =
@@ -863,56 +873,148 @@ protected SplitPredicate predicatesCompensate(
863873
}
864874

865875
/**
866-
* Check the queryPredicates contains the required nullable slot
876+
* Check whether query-side null-reject evidence covers each required view-side slot set.
877+
*
878+
* <p>The check is view-based because the required null-reject slots come from the MV join graph.
879+
* The returned compensation predicates are query-based because they will be merged into queryStructInfo.
880+
*
881+
* <p>Return meanings:
882+
* Optional.empty(): no valid proof, or no safe output slot can carry the compensation predicate.
883+
* Optional.of(emptySet()): existing query predicates already provide the required null-reject.
884+
* Optional.of(nonEmptySet): INNER JoinEdge proof must be materialized as these IS NOT NULL predicates.
867885
*/
868-
private boolean containsNullRejectSlot(Set<Set<Slot>> requireNoNullableViewSlot,
886+
private Optional<Set<Expression>> getQueryBasedNullRejectCompensationPredicates(
887+
Set<Set<Slot>> requireNoNullableViewSlot,
869888
Set<Expression> queryPredicates,
870889
SlotMapping queryToViewMapping,
871890
StructInfo queryStructInfo,
872891
StructInfo viewStructInfo,
892+
SlotMapping viewToQueryMapping,
873893
CascadesContext cascadesContext) {
874-
Set<Expression> queryPulledUpPredicates = queryPredicates.stream()
875-
.flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream())
876-
.map(expr -> {
877-
// NOTICE inferNotNull generate Not with isGeneratedIsNotNull = false,
878-
// so, we need set this flag to false before comparison.
879-
if (expr instanceof Not) {
880-
return ((Not) expr).withGeneratedIsNotNull(false);
881-
}
882-
return expr;
883-
})
894+
Set<Slot> predicateNullRejectViewSlots = getViewBasedNullRejectSlots(
895+
getPredicateNullRejectSlots(queryPredicates, cascadesContext), queryToViewMapping, queryStructInfo);
896+
Set<Slot> innerJoinNullRejectViewSlots = getViewBasedNullRejectSlots(
897+
getInnerJoinNullRejectSlots(queryStructInfo, cascadesContext), queryToViewMapping, queryStructInfo);
898+
Set<Slot> allNullRejectViewSlots = new HashSet<>(predicateNullRejectViewSlots);
899+
allNullRejectViewSlots.addAll(innerJoinNullRejectViewSlots);
900+
if (allNullRejectViewSlots.isEmpty()) {
901+
return Optional.empty();
902+
}
903+
Set<Slot> viewOutputSlots = viewStructInfo.getPlanOutputShuttledExpressions().stream()
904+
.filter(Slot.class::isInstance)
905+
.map(Slot.class::cast)
884906
.collect(Collectors.toSet());
885-
Set<Expression> queryNullRejectPredicates =
886-
ExpressionUtils.inferNotNull(queryPulledUpPredicates, cascadesContext);
887-
if (queryPulledUpPredicates.containsAll(queryNullRejectPredicates)) {
888-
// Query has no null reject predicates, return
889-
return false;
907+
Map<SlotReference, SlotReference> viewToQuerySlotReferenceMap = viewToQueryMapping.toSlotReferenceMap();
908+
Set<Expression> compensationPredicates = new HashSet<>();
909+
for (Set<Slot> requiredViewSlots : getShuttledRequireNoNullableViewSlots(
910+
requireNoNullableViewSlot, viewStructInfo)) {
911+
if (Sets.intersection(requiredViewSlots, allNullRejectViewSlots).isEmpty()) {
912+
return Optional.empty();
913+
}
914+
if (!Sets.intersection(requiredViewSlots, predicateNullRejectViewSlots).isEmpty()) {
915+
continue;
916+
}
917+
Optional<Slot> compensationViewSlot = findCompensationViewSlot(
918+
requiredViewSlots, viewOutputSlots, innerJoinNullRejectViewSlots);
919+
if (!compensationViewSlot.isPresent()) {
920+
return Optional.empty();
921+
}
922+
Slot querySlot = viewToQuerySlotReferenceMap.get(compensationViewSlot.get());
923+
if (querySlot == null) {
924+
return Optional.empty();
925+
}
926+
compensationPredicates.add(new Not(new IsNull(querySlot), false));
927+
}
928+
return Optional.of(compensationPredicates);
929+
}
930+
931+
private Set<Slot> getPredicateNullRejectSlots(Set<Expression> queryPredicates, CascadesContext cascadesContext) {
932+
Set<Slot> nullRejectSlots = new HashSet<>();
933+
for (Expression queryPredicate : queryPredicates) {
934+
TypeUtils.isNotNull(queryPredicate).ifPresent(nullRejectSlots::add);
935+
}
936+
for (Expression inferredNotNull : ExpressionUtils.inferNotNull(queryPredicates, cascadesContext)) {
937+
TypeUtils.isNotNull(inferredNotNull).ifPresent(nullRejectSlots::add);
890938
}
891-
// Get query null reject predicate slots
892-
Set<Expression> queryNullRejectSlotSet = new HashSet<>();
893-
for (Expression queryNullRejectPredicate : queryNullRejectPredicates) {
894-
Optional<Slot> notNullSlot = TypeUtils.isNotNull(queryNullRejectPredicate);
895-
if (!notNullSlot.isPresent()) {
939+
return nullRejectSlots;
940+
}
941+
942+
private Set<Slot> getInnerJoinNullRejectSlots(StructInfo queryStructInfo, CascadesContext cascadesContext) {
943+
Set<Slot> nullRejectSlots = new HashSet<>();
944+
// INNER JOIN conditions guarantee NOT NULL on join-key slots.
945+
// After EliminateOuterJoin converts LEFT to INNER, the JoinEdge objects in the HyperGraph
946+
// retain the INNER type even though EliminateNotNull removes filter-level NOT NULL predicates.
947+
for (JoinEdge joinEdge : queryStructInfo.getHyperGraph().getJoinEdges()) {
948+
if (joinEdge.getJoinType().isInnerJoin()) {
949+
nullRejectSlots.addAll(ExpressionUtils.inferNotNullSlots(
950+
ImmutableSet.copyOf(joinEdge.getExpressions()), cascadesContext));
951+
}
952+
}
953+
return nullRejectSlots;
954+
}
955+
956+
private Set<Slot> getViewBasedNullRejectSlots(Set<Slot> queryNullRejectSlots,
957+
SlotMapping queryToViewMapping, StructInfo queryStructInfo) {
958+
Set<Slot> viewBasedSlots = new HashSet<>();
959+
for (Slot queryNullRejectSlot : queryNullRejectSlots) {
960+
Expression shuttledQuerySlot = ExpressionUtils.shuttleExpressionWithLineage(
961+
queryNullRejectSlot, queryStructInfo.getTopPlan());
962+
if (!(shuttledQuerySlot instanceof Slot)) {
896963
continue;
897964
}
898-
queryNullRejectSlotSet.add(notNullSlot.get());
965+
Expression viewSlot = ExpressionUtils.replace(shuttledQuerySlot,
966+
queryToViewMapping.toSlotReferenceMap());
967+
if (viewSlot instanceof Slot) {
968+
viewBasedSlots.add((Slot) viewSlot);
969+
}
899970
}
900-
// query slot need shuttle to use table slot, avoid alias influence
901-
Set<Expression> queryUsedNeedRejectNullSlotsViewBased = ExpressionUtils.shuttleExpressionWithLineage(
902-
new ArrayList<>(queryNullRejectSlotSet), queryStructInfo.getTopPlan()).stream()
903-
.map(expr -> ExpressionUtils.replace(expr, queryToViewMapping.toSlotReferenceMap()))
904-
.collect(Collectors.toSet());
905-
// view slot need shuttle to use table slot, avoid alias influence
971+
return viewBasedSlots;
972+
}
973+
974+
private Set<Set<Slot>> getShuttledRequireNoNullableViewSlots(Set<Set<Slot>> requireNoNullableViewSlot,
975+
StructInfo viewStructInfo) {
906976
Set<Set<Slot>> shuttledRequireNoNullableViewSlot = new HashSet<>();
907977
for (Set<Slot> requireNullableSlots : requireNoNullableViewSlot) {
908978
shuttledRequireNoNullableViewSlot.add(
909979
ExpressionUtils.shuttleExpressionWithLineage(new ArrayList<>(requireNullableSlots),
910980
viewStructInfo.getTopPlan()).stream().map(Slot.class::cast)
911981
.collect(Collectors.toSet()));
912982
}
913-
// query pulledUp predicates should have null reject predicates and contains any require noNullable slot
914-
return shuttledRequireNoNullableViewSlot.stream().noneMatch(viewRequiredNullSlotSet ->
915-
Sets.intersection(viewRequiredNullSlotSet, queryUsedNeedRejectNullSlotsViewBased).isEmpty());
983+
return shuttledRequireNoNullableViewSlot;
984+
}
985+
986+
private Optional<Slot> findCompensationViewSlot(Set<Slot> requiredViewSlots, Set<Slot> viewOutputSlots,
987+
Set<Slot> innerJoinNullRejectViewSlots) {
988+
Set<Slot> outputRequiredSlots = Sets.intersection(requiredViewSlots, viewOutputSlots);
989+
Optional<Slot> compensationViewSlot = outputRequiredSlots.stream()
990+
.filter(innerJoinNullRejectViewSlots::contains)
991+
.findFirst();
992+
if (compensationViewSlot.isPresent()) {
993+
return compensationViewSlot;
994+
}
995+
return outputRequiredSlots.stream()
996+
.filter(slot -> isOriginalNonNullableSlotOnInnerJoinProofTable(slot, innerJoinNullRejectViewSlots))
997+
.findFirst();
998+
}
999+
1000+
private boolean isOriginalNonNullableSlotOnInnerJoinProofTable(Slot slot, Set<Slot> innerJoinNullRejectViewSlots) {
1001+
if (!(slot instanceof SlotReference)) {
1002+
return false;
1003+
}
1004+
SlotReference slotReference = (SlotReference) slot;
1005+
if (!slotReference.getOriginalColumn().map(column -> !column.isAllowNull()).orElse(!slot.nullable())) {
1006+
return false;
1007+
}
1008+
Optional<TableIf> originalTable = slotReference.getOriginalTable();
1009+
if (!originalTable.isPresent()) {
1010+
return false;
1011+
}
1012+
return innerJoinNullRejectViewSlots.stream()
1013+
.filter(SlotReference.class::isInstance)
1014+
.map(SlotReference.class::cast)
1015+
.map(SlotReference::getOriginalTable)
1016+
.anyMatch(referenceTable -> referenceTable.isPresent()
1017+
&& referenceTable.get().equals(originalTable.get()));
9161018
}
9171019

9181020
/**

0 commit comments

Comments
 (0)