Skip to content

Commit

Permalink
[refactor](nereids) refactor analyze view (apache#37106)
Browse files Browse the repository at this point in the history 
The Analyzer of NereidsPlanner use different rules to analyze normal plan and view, to prevent the plans in the views analyze multiple times, because some rules can not apply multiple times, say, decimal type coercion, if this rule applied multiple times, it will generate a wrong result.

But this design is trick. Normally, after process the LogicalView, the whole plan tree in the LogicalView should not contains unbound plan, but the current situation is not like this. And this problem block development of some rules, so I refactor it:
1. the Analyzer will not traverse the children of the LogicalView
2. After link the LogicalView to the outer plan tree, the whole plan tree of the LogicalView will not contains unbound plan
3. analyze view and table use the same rules, keep it simple

(cherry picked from commit 5858bee)
  • Loading branch information
@924060929
924060929 committed Jul 2, 2024
1 parent 42f4271 commit 8ae486e
Show file tree
Hide file tree
Showing 9 changed files with 446 additions and 399 deletions.
12 changes: 2 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,19 +240,11 @@ public void toMemo() {
}

public Analyzer newAnalyzer() {
return newAnalyzer(false);
}

public Analyzer newAnalyzer(boolean analyzeView) {
return new Analyzer(this, analyzeView);
}

public Analyzer newAnalyzer(boolean analyzeView, Optional<CustomTableResolver> customTableResolver) {
return new Analyzer(this, analyzeView, customTableResolver);
return newAnalyzer(Optional.empty());
}

public Analyzer newAnalyzer(Optional<CustomTableResolver> customTableResolver) {
return newAnalyzer(false, customTableResolver);
return new Analyzer(this, customTableResolver);
}

@Override
Expand Down
40 changes: 38 additions & 2 deletions ...e-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,16 @@
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;

import com.google.common.collect.ImmutableList;

import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand All @@ -46,6 +49,8 @@
* Each batch of rules will be uniformly executed.
*/
public abstract class AbstractBatchJobExecutor {
private static final ThreadLocal<Set<Class<Plan>>> NOT_TRAVERSE_CHILDREN = new ThreadLocal();
private static final Predicate<Plan> TRAVERSE_ALL_PLANS = plan -> true;

protected CascadesContext cascadesContext;

Expand All @@ -65,6 +70,17 @@ public static List<RewriteJob> jobs(RewriteJob... jobs) {
).collect(ImmutableList.toImmutableList());
}

/** notTraverseChildrenOf */
public static <T> T notTraverseChildrenOf(
Set<Class<? extends Plan>> notTraverseClasses, Supplier<T> action) {
try {
NOT_TRAVERSE_CHILDREN.set((Set) notTraverseClasses);
return action.get();
} finally {
NOT_TRAVERSE_CHILDREN.remove();
}
}

public static TopicRewriteJob topic(String topicName, RewriteJob... jobs) {
return new TopicRewriteJob(topicName, Arrays.asList(jobs));
}
Expand All @@ -82,7 +98,7 @@ public static RewriteJob bottomUp(List<RuleFactory> ruleFactories) {
.map(RuleFactory::buildRules)
.flatMap(List::stream)
.collect(ImmutableList.toImmutableList());
return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteBottomUpJob::new, true);
return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteBottomUpJob::new, getTraversePredicate(), true);
}

public static RewriteJob topDown(RuleFactory... ruleFactories) {
Expand All @@ -98,7 +114,7 @@ public static RewriteJob topDown(List<RuleFactory> ruleFactories, boolean once)
.map(RuleFactory::buildRules)
.flatMap(List::stream)
.collect(ImmutableList.toImmutableList());
return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteTopDownJob::new, once);
return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteTopDownJob::new, getTraversePredicate(), once);
}

public static RewriteJob custom(RuleType ruleType, Supplier<CustomRewriter> planRewriter) {
Expand Down Expand Up @@ -126,4 +142,24 @@ public void execute() {
}

public abstract List<RewriteJob> getJobs();

private static Predicate<Plan> getTraversePredicate() {
Set<Class<Plan>> notTraverseChildren = NOT_TRAVERSE_CHILDREN.get();
return notTraverseChildren == null
? TRAVERSE_ALL_PLANS
: new NotTraverseChildren(notTraverseChildren);
}

private static class NotTraverseChildren implements Predicate<Plan> {
private final Set<Class<Plan>> notTraverseChildren;

public NotTraverseChildren(Set<Class<Plan>> notTraverseChildren) {
this.notTraverseChildren = Objects.requireNonNull(notTraverseChildren, "notTraversePlans can not be null");
}

@Override
public boolean test(Plan plan) {
return !notTraverseChildren.contains(plan.getClass());
}
}
}
80 changes: 33 additions & 47 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute;
import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.logical.LogicalView;

import com.google.common.collect.ImmutableSet;

import java.util.List;
import java.util.Objects;
Expand All @@ -59,8 +63,7 @@
*/
public class Analyzer extends AbstractBatchJobExecutor {

public static final List<RewriteJob> DEFAULT_ANALYZE_JOBS = buildAnalyzeJobs(Optional.empty());
public static final List<RewriteJob> DEFAULT_ANALYZE_VIEW_JOBS = buildAnalyzeViewJobs(Optional.empty());
public static final List<RewriteJob> ANALYZE_JOBS = buildAnalyzeJobs(Optional.empty());

private final List<RewriteJob> jobs;

Expand All @@ -69,36 +72,23 @@ public class Analyzer extends AbstractBatchJobExecutor {
* @param cascadesContext planner context for execute job
*/
public Analyzer(CascadesContext cascadesContext) {
this(cascadesContext, false);
}

public Analyzer(CascadesContext cascadesContext, boolean analyzeView) {
this(cascadesContext, analyzeView, Optional.empty());
this(cascadesContext, Optional.empty());
}

/**
* constructor of Analyzer. For view, we only do bind relation since other analyze step will do by outer Analyzer.
*
* @param cascadesContext current context for analyzer
* @param analyzeView analyze view or user sql. If true, analyzer is used for view.
* @param customTableResolver custom resolver for outer catalog.
*/
public Analyzer(CascadesContext cascadesContext, boolean analyzeView,
Optional<CustomTableResolver> customTableResolver) {
public Analyzer(CascadesContext cascadesContext, Optional<CustomTableResolver> customTableResolver) {
super(cascadesContext);
Objects.requireNonNull(customTableResolver, "customTableResolver cannot be null");
if (analyzeView) {
if (customTableResolver.isPresent()) {
this.jobs = buildAnalyzeViewJobs(customTableResolver);
} else {
this.jobs = DEFAULT_ANALYZE_VIEW_JOBS;
}

if (customTableResolver.isPresent()) {
this.jobs = buildAnalyzeJobs(customTableResolver);
} else {
if (customTableResolver.isPresent()) {
this.jobs = buildAnalyzeJobs(customTableResolver);
} else {
this.jobs = DEFAULT_ANALYZE_JOBS;
}
this.jobs = ANALYZE_JOBS;
}
}

Expand All @@ -114,47 +104,43 @@ public void analyze() {
execute();
}

private static List<RewriteJob> buildAnalyzeViewJobs(Optional<CustomTableResolver> customTableResolver) {
return jobs(
topDown(new AnalyzeCTE()),
topDown(new EliminateLogicalSelectHint()),
bottomUp(
new BindRelation(customTableResolver),
new CheckPolicy()
)
private static List<RewriteJob> buildAnalyzeJobs(Optional<CustomTableResolver> customTableResolver) {
return notTraverseChildrenOf(
ImmutableSet.of(LogicalView.class, LogicalCTEAnchor.class),
() -> buildAnalyzerJobs(customTableResolver)
);
}

private static List<RewriteJob> buildAnalyzeJobs(Optional<CustomTableResolver> customTableResolver) {
private static List<RewriteJob> buildAnalyzerJobs(Optional<CustomTableResolver> customTableResolver) {
return jobs(
// we should eliminate hint before "Subquery unnesting".
topDown(new AnalyzeCTE()),
topDown(new EliminateLogicalSelectHint()),
bottomUp(
new BindRelation(customTableResolver),
new CheckPolicy()
new BindRelation(customTableResolver),
new CheckPolicy()
),
bottomUp(new BindExpression()),
topDown(new BindSink()),
bottomUp(new CheckAfterBind()),
bottomUp(
new ProjectToGlobalAggregate(),
// this rule check's the logicalProject node's isDistinct property
// and replace the logicalProject node with a LogicalAggregate node
// so any rule before this, if create a new logicalProject node
// should make sure isDistinct property is correctly passed around.
// please see rule BindSlotReference or BindFunction for example
new EliminateDistinctConstant(),
new ProjectWithDistinctToAggregate(),
new ReplaceExpressionByChildOutput(),
new OneRowRelationExtractAggregate()
new ProjectToGlobalAggregate(),
// this rule check's the logicalProject node's isDistinct property
// and replace the logicalProject node with a LogicalAggregate node
// so any rule before this, if create a new logicalProject node
// should make sure isDistinct property is correctly passed around.
// please see rule BindSlotReference or BindFunction for example
new EliminateDistinctConstant(),
new ProjectWithDistinctToAggregate(),
new ReplaceExpressionByChildOutput(),
new OneRowRelationExtractAggregate()
),
topDown(
new FillUpMissingSlots(),
// We should use NormalizeRepeat to compute nullable properties for LogicalRepeat in the analysis
// stage. NormalizeRepeat will compute nullable property, add virtual slot, LogicalAggregate and
// LogicalProject for normalize. This rule depends on FillUpMissingSlots to fill up slots.
new NormalizeRepeat()
new FillUpMissingSlots(),
// We should use NormalizeRepeat to compute nullable properties for LogicalRepeat in the analysis
// stage. NormalizeRepeat will compute nullable property, add virtual slot, LogicalAggregate and
// LogicalProject for normalize. This rule depends on FillUpMissingSlots to fill up slots.
new NormalizeRepeat()
),
bottomUp(new AdjustAggregateNullableForEmptySet()),
// consider sql with user defined var @t_zone
Expand Down
Loading

0 comments on commit 8ae486e

Please sign in to comment.