Skip to content

Commit

Permalink
[enhancement](Nereids) New optimizer support check column privileges (a…
Browse files Browse the repository at this point in the history 
…pache#31494)

(cherry picked from commit accfcfc)
  • Loading branch information
@924060929
924060929 committed Mar 3, 2024
1 parent de9b5f7 commit cd2dc36
Show file tree
Hide file tree
Showing 20 changed files with 850 additions and 69 deletions.
43 changes: 22 additions & 21 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,6 @@ public class CascadesContext implements ScheduleContext {
private boolean isLeadingDisableJoinReorder = false;

private final Map<String, Hint> hintMap = Maps.newLinkedHashMap();
private final boolean shouldCheckRelationAuthentication;
private final ThreadLocal<Boolean> showPlanProcess = new ThreadLocal<>();

// This list is used to listen the change event of the plan which
Expand All @@ -133,12 +132,12 @@ public class CascadesContext implements ScheduleContext {
/**
* Constructor of OptimizerContext.
*
* @param memo {@link Memo} reference
* @param statementContext {@link StatementContext} reference
* @param memo {@link Memo} reference
*/
private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> currentTree,
StatementContext statementContext, Plan plan, Memo memo,
CTEContext cteContext, PhysicalProperties requireProperties, boolean shouldCheckRelationAuthentication) {
CTEContext cteContext, PhysicalProperties requireProperties) {
this.parent = Objects.requireNonNull(parent, "parent should not null");
this.currentTree = Objects.requireNonNull(currentTree, "currentTree should not null");
this.statementContext = Objects.requireNonNull(statementContext, "statementContext should not null");
Expand All @@ -152,7 +151,6 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
this.subqueryExprIsAnalyzed = new HashMap<>();
this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable());
this.materializationContexts = new ArrayList<>();
this.shouldCheckRelationAuthentication = shouldCheckRelationAuthentication;
}

/**
Expand All @@ -161,13 +159,7 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
public static CascadesContext initContext(StatementContext statementContext,
Plan initPlan, PhysicalProperties requireProperties) {
return newContext(Optional.empty(), Optional.empty(), statementContext,
initPlan, new CTEContext(), requireProperties, true);
}

public static CascadesContext initViewContext(StatementContext statementContext,
Plan initPlan, PhysicalProperties requireProperties) {
return newContext(Optional.empty(), Optional.empty(), statementContext,
initPlan, new CTEContext(), requireProperties, false);
initPlan, new CTEContext(), requireProperties);
}

/**
Expand All @@ -176,14 +168,14 @@ public static CascadesContext initViewContext(StatementContext statementContext,
public static CascadesContext newContextWithCteContext(CascadesContext cascadesContext,
Plan initPlan, CTEContext cteContext) {
return newContext(Optional.of(cascadesContext), Optional.empty(),
cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY,
cascadesContext.shouldCheckRelationAuthentication);
cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY
);
}

public static CascadesContext newCurrentTreeContext(CascadesContext context) {
return CascadesContext.newContext(context.getParent(), context.getCurrentTree(), context.getStatementContext(),
context.getRewritePlan(), context.getCteContext(),
context.getCurrentJobContext().getRequiredProperties(), context.shouldCheckRelationAuthentication);
context.getCurrentJobContext().getRequiredProperties());
}

/**
Expand All @@ -192,14 +184,14 @@ public static CascadesContext newCurrentTreeContext(CascadesContext context) {
public static CascadesContext newSubtreeContext(Optional<CTEId> subtree, CascadesContext context,
Plan plan, PhysicalProperties requireProperties) {
return CascadesContext.newContext(Optional.of(context), subtree, context.getStatementContext(),
plan, context.getCteContext(), requireProperties, context.shouldCheckRelationAuthentication);
plan, context.getCteContext(), requireProperties);
}

private static CascadesContext newContext(Optional<CascadesContext> parent, Optional<CTEId> subtree,
StatementContext statementContext, Plan initPlan, CTEContext cteContext,
PhysicalProperties requireProperties, boolean shouldCheckRelationAuthentication) {
PhysicalProperties requireProperties) {
return new CascadesContext(parent, subtree, statementContext, initPlan, null,
cteContext, requireProperties, shouldCheckRelationAuthentication);
cteContext, requireProperties);
}

public CascadesContext getRoot() {
Expand Down Expand Up @@ -655,10 +647,6 @@ public void setLeadingJoin(boolean leadingJoin) {
isLeadingJoin = leadingJoin;
}

public boolean shouldCheckRelationAuthentication() {
return shouldCheckRelationAuthentication;
}

public boolean isLeadingDisableJoinReorder() {
return isLeadingDisableJoinReorder;
}
Expand All @@ -675,6 +663,10 @@ public void addPlanProcess(PlanProcess planProcess) {
planProcesses.add(planProcess);
}

public void addPlanProcesses(List<PlanProcess> planProcesses) {
this.planProcesses.addAll(planProcesses);
}

public List<PlanProcess> getPlanProcesses() {
return planProcesses;
}
Expand Down Expand Up @@ -706,4 +698,13 @@ public void withPlanProcess(boolean showPlanProcess, Runnable task) {
}
}
}

/** keepOrShowPlanProcess */
public void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) {
if (showPlanProcess) {
withPlanProcess(showPlanProcess, task);
} else {
task.run();
}
}
}
17 changes: 15 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.doris.nereids.rules.rewrite.CheckDataTypes;
import org.apache.doris.nereids.rules.rewrite.CheckMatchExpression;
import org.apache.doris.nereids.rules.rewrite.CheckMultiDistinct;
import org.apache.doris.nereids.rules.rewrite.CheckPrivileges;
import org.apache.doris.nereids.rules.rewrite.CollectFilterAboveConsumer;
import org.apache.doris.nereids.rules.rewrite.CollectProjectAboveConsumer;
import org.apache.doris.nereids.rules.rewrite.ColumnPruning;
Expand All @@ -66,7 +67,7 @@
import org.apache.doris.nereids.rules.rewrite.EliminateOrderByConstant;
import org.apache.doris.nereids.rules.rewrite.EliminateSemiJoin;
import org.apache.doris.nereids.rules.rewrite.EliminateSort;
import org.apache.doris.nereids.rules.rewrite.EliminateSortUnderSubquery;
import org.apache.doris.nereids.rules.rewrite.EliminateSortUnderSubqueryOrView;
import org.apache.doris.nereids.rules.rewrite.EliminateUnnecessaryProject;
import org.apache.doris.nereids.rules.rewrite.EnsureProjectOnTopJoin;
import org.apache.doris.nereids.rules.rewrite.ExtractAndNormalizeWindowExpression;
Expand All @@ -78,6 +79,7 @@
import org.apache.doris.nereids.rules.rewrite.InferJoinNotNull;
import org.apache.doris.nereids.rules.rewrite.InferPredicates;
import org.apache.doris.nereids.rules.rewrite.InferSetOperatorDistinct;
import org.apache.doris.nereids.rules.rewrite.InlineLogicalView;
import org.apache.doris.nereids.rules.rewrite.LimitSortToTopN;
import org.apache.doris.nereids.rules.rewrite.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion;
Expand Down Expand Up @@ -142,7 +144,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
topic("Plan Normalization",
topDown(
new EliminateOrderByConstant(),
new EliminateSortUnderSubquery(),
new EliminateSortUnderSubqueryOrView(),
new EliminateGroupByConstant(),
// MergeProjects depends on this rule
new LogicalSubQueryAliasToLogicalProject(),
Expand Down Expand Up @@ -231,6 +233,17 @@ public class Rewriter extends AbstractBatchJobExecutor {
new ApplyToJoin()
)
),
// before `Subquery unnesting` topic, some correlate slots should have appeared at LogicalApply.left,
// but it appeared at LogicalApply.right. After the `Subquery unnesting` topic, all slots is placed in a
// normal position, then we can check column privileges by these steps
//
// 1. use ColumnPruning rule to derive the used slots in LogicalView
// 2. and then check the column privileges
// 3. finally, we can eliminate the LogicalView
topic("Inline view and check column privileges",
custom(RuleType.CHECK_PRIVILEGES, CheckPrivileges::new),
bottomUp(new InlineLogicalView())
),
topic("Eliminate optimization",
bottomUp(
new EliminateLimit(),
Expand Down
5 changes: 4 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ public enum RuleType {

ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
ELIMINATE_SUBQUERY_ORDER_BY(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),
ELIMINATE_HINT(RuleTypeClass.REWRITE),
ELIMINATE_JOIN_ON_EMPTYRELATION(RuleTypeClass.REWRITE),
ELIMINATE_FILTER_ON_EMPTYRELATION(RuleTypeClass.REWRITE),
Expand Down Expand Up @@ -288,6 +289,8 @@ public enum RuleType {
CTE_INLINE(RuleTypeClass.REWRITE),
REWRITE_CTE_CHILDREN(RuleTypeClass.REWRITE),
COLLECT_FILTER_ON_CONSUMER(RuleTypeClass.REWRITE),
INLINE_VIEW(RuleTypeClass.REWRITE),
CHECK_PRIVILEGES(RuleTypeClass.REWRITE),

COLLECT_FILTER(RuleTypeClass.REWRITE),
COLLECT_JOIN_CONSTRAINT(RuleTypeClass.REWRITE),
Expand Down
34 changes: 17 additions & 17 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalView;
import org.apache.doris.nereids.util.RelationUtil;
import org.apache.doris.qe.ConnectContext;

Expand Down Expand Up @@ -157,7 +159,7 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe
}

// TODO: should generate different Scan sub class according to table's type
LogicalPlan scan = getAndCheckLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
LogicalPlan scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
if (cascadesContext.isLeadingJoin()) {
LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
leading.putRelationIdAndTableName(Pair.of(unboundRelation.getRelationId(), tableName));
Expand All @@ -178,7 +180,7 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
if (table == null) {
table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv());
}
return getAndCheckLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
}

private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> tableQualifier) {
Expand Down Expand Up @@ -234,25 +236,18 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
return scan;
}

private LogicalPlan getAndCheckLogicalPlan(TableIf table, UnboundRelation unboundRelation,
private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation,
List<String> tableQualifier, CascadesContext cascadesContext) {
// if current context is in the view, we can skip check authentication because
// the view already checked authentication
if (cascadesContext.shouldCheckRelationAuthentication()) {
UserAuthentication.checkPermission(table, cascadesContext.getConnectContext());
}
return doGetLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
}

private LogicalPlan doGetLogicalPlan(TableIf table, UnboundRelation unboundRelation, List<String> tableQualifier,
CascadesContext cascadesContext) {
switch (table.getType()) {
case OLAP:
case MATERIALIZED_VIEW:
return makeOlapScan(table, unboundRelation, tableQualifier);
case VIEW:
Plan viewPlan = parseAndAnalyzeView(((View) table).getInlineViewDef(), cascadesContext);
return new LogicalSubQueryAlias<>(tableQualifier, viewPlan);
View view = (View) table;
String inlineViewDef = view.getInlineViewDef();
Plan viewBody = parseAndAnalyzeView(inlineViewDef, cascadesContext);
LogicalView<Plan> logicalView = new LogicalView<>(view, viewBody);
return new LogicalSubQueryAlias<>(tableQualifier, logicalView);
case HMS_EXTERNAL_TABLE:
if (Config.enable_query_hive_views && ((HMSExternalTable) table).isView()) {
String hiveCatalog = ((HMSExternalTable) table).getCatalog().getName();
Expand All @@ -276,6 +271,8 @@ private LogicalPlan doGetLogicalPlan(TableIf table, UnboundRelation unboundRelat
return new LogicalOdbcScan(unboundRelation.getRelationId(), table, tableQualifier);
case ES_EXTERNAL_TABLE:
return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, tableQualifier);
case TEST_EXTERNAL_TABLE:
return new LogicalTestScan(unboundRelation.getRelationId(), table, tableQualifier);
default:
throw new AnalysisException("Unsupported tableType " + table.getType());
}
Expand All @@ -299,9 +296,12 @@ private Plan parseAndAnalyzeView(String ddlSql, CascadesContext parentContext) {
if (parsedViewPlan instanceof UnboundResultSink) {
parsedViewPlan = (LogicalPlan) ((UnboundResultSink<?>) parsedViewPlan).child();
}
CascadesContext viewContext = CascadesContext.initViewContext(
CascadesContext viewContext = CascadesContext.initContext(
parentContext.getStatementContext(), parsedViewPlan, PhysicalProperties.ANY);
viewContext.newAnalyzer(true, customTableResolver).analyze();
viewContext.keepOrShowPlanProcess(parentContext.showPlanProcess(), () -> {
viewContext.newAnalyzer(true, customTableResolver).analyze();
});
parentContext.addPlanProcesses(viewContext.getPlanProcesses());
// we should remove all group expression of the plan which in other memo, so the groupId would not conflict
return viewContext.getRewritePlan();
}
Expand Down
1 change: 1 addition & 0 deletions ...ore/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ private void setVar(SelectHintSetVar selectHint, StatementContext context) {
}
throw new AnalysisException("The nereids is disabled in this sql, fallback to original planner");
}
context.invalidCache(selectHint.getHintName());
}

private void extractLeading(SelectHintLeading selectHint, CascadesContext context,
Expand Down
20 changes: 8 additions & 12 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,20 @@

import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.qe.ConnectContext;

import java.util.Set;

/**
* Check whether a user is permitted to scan specific tables.
*/
public class UserAuthentication {
/** checkPermission. */
public static void checkPermission(TableIf table, ConnectContext connectContext) {
public static void checkPermission(TableIf table, ConnectContext connectContext, Set<String> columns)
throws UserException {
if (table == null) {
return;
}
Expand All @@ -40,7 +42,7 @@ public static void checkPermission(TableIf table, ConnectContext connectContext)
}
String tableName = table.getName();
DatabaseIf db = table.getDatabase();
// when table inatanceof FunctionGenTable,db will be null
// when table instanceof FunctionGenTable,db will be null
if (db == null) {
return;
}
Expand All @@ -50,13 +52,7 @@ public static void checkPermission(TableIf table, ConnectContext connectContext)
return;
}
String ctlName = catalog.getName();
// TODO: 2023/7/19 checkColumnsPriv
if (!connectContext.getEnv().getAccessManager().checkTblPriv(connectContext, ctlName, dbName,
tableName, PrivPredicate.SELECT)) {
String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SELECT",
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
ctlName + ": " + dbName + ": " + tableName);
throw new AnalysisException(message);
}
connectContext.getEnv().getAccessManager().checkColumnsPriv(
connectContext.getCurrentUserIdentity(), ctlName, dbName, tableName, columns, PrivPredicate.SELECT);
}
}
Loading

0 comments on commit cd2dc36

Please sign in to comment.