Skip to content

Commit

Permalink
[enhancement](Nereids) Enable parse sql from sql cache (apache#33262)
Browse files Browse the repository at this point in the history
Before this pr, the query must pass through parser, analyzer, rewriter, optimizer and translator, then we can check whether this query can use sql cache, if the query is too long, or the number of join tables too big, the plan time usually >= 500ms.

This pr reduce this time by skip the fashion plan path, because we can reuse the previous physical plan and query result if no any changed. In some cases we should not parse sql from sql cache, e.g. table structure changed, data changed, user policies changed, privileges changed, contains non-deterministic functions, and user variables changed.

In my test case: query a view which has lots of join and union, and the tables has empty partition, the query latency is about 3ms. if not parse sql from sql cache, the plan time is about 550ms

## Features
1. use Config.sql_cache_manage_num to control how many sql cache be reused in on fe
2. if explain plan appear some plans contains `LogicalSqlCache` or `PhysicalSqlCache`, it means the query can use sql cache, like this:
```sql
mysql> set enable_sql_cache=true;
Query OK, 0 rows affected (0.00 sec)

mysql> explain physical plan select * from test.t;
+----------------------------------------------------------------------------------+
| Explain String(Nereids Planner)                                                  |
+----------------------------------------------------------------------------------+
| cost = 3.135                                                                     |
| PhysicalResultSink[53] ( outputExprs=[c1#0, c2#1] )                              |
| +--PhysicalDistribute[50]@0 ( stats=3, distributionSpec=DistributionSpecGather ) |
|    +--PhysicalOlapScan[t]@0 ( stats=3 )                                          |
+----------------------------------------------------------------------------------+
4 rows in set (0.02 sec)

mysql> select * from test.t;
+------+------+
| c1   | c2   |
+------+------+
|    1 |    2 |
|   -2 |   -2 |
| NULL |   30 |
+------+------+
3 rows in set (0.05 sec)

mysql> explain physical plan select * from test.t;
+-------------------------------------------------------------------------------------------+
| Explain String(Nereids Planner)                                                           |
+-------------------------------------------------------------------------------------------+
| cost = 0.0                                                                                |
| PhysicalSqlCache[2] ( queryId=78511f515cda466b-95385d892d6c68d0, backend=127.0.0.1:9050 ) |
| +--PhysicalResultSink[52] ( outputExprs=[c1#0, c2#1] )                                    |
|    +--PhysicalDistribute[49]@0 ( stats=3, distributionSpec=DistributionSpecGather )       |
|       +--PhysicalOlapScan[t]@0 ( stats=3 )                                                |
+-------------------------------------------------------------------------------------------+
5 rows in set (0.01 sec)
```
  • Loading branch information
924060929 authored Apr 12, 2024
1 parent c5a81e9 commit 03bd2a3
Show file tree
Hide file tree
Showing 54 changed files with 3,025 additions and 427 deletions.
10 changes: 8 additions & 2 deletions fe/fe-common/src/main/java/org/apache/doris/common/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

package org.apache.doris.common;

import org.apache.doris.common.ConfigBase.ConfField;

public class Config extends ConfigBase {

@ConfField(description = {"用户自定义配置文件的路径,用于存放 fe_custom.conf。该文件中的配置会覆盖 fe.conf 中的配置",
Expand Down Expand Up @@ -2004,6 +2002,14 @@ public class Config extends ConfigBase {
+ "the old load statement will be degraded."})
public static boolean enable_nereids_load = false;

/**
* the plan cache num which can be reused for the next query
*/
@ConfField(mutable = false, varType = VariableAnnotation.EXPERIMENTAL, description = {
"当前默认设置为 100,用来控制控制NereidsSqlCacheManager管理的sql cache数量。",
"Now default set to 100, this config is used to control the number of "
+ "sql cache managed by NereidsSqlCacheManager"})
public static int sql_cache_manage_num = 100;

/**
* Maximum number of events to poll in each RPC.
Expand Down
4 changes: 4 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/common/Pair.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ private Pair(F first, S second) {
this.second = second;
}

public static <P, K extends P> Pair<K, K> ofSame(K same) {
return new Pair<>(same, same);
}

public static <F, S> Pair<F, S> of(F first, S second) {
return new Pair<>(first, second);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,10 @@ public void checkLimitations(Long partitionNum, Long tabletNum, Long cardinality
return;
}
// match global rule
List<SqlBlockRule> globalRules =
nameToSqlBlockRuleMap.values().stream().filter(SqlBlockRule::getGlobal).collect(Collectors.toList());
for (SqlBlockRule rule : globalRules) {
checkLimitations(rule, partitionNum, tabletNum, cardinality);
for (SqlBlockRule rule : nameToSqlBlockRuleMap.values()) {
if (rule.getGlobal()) {
checkLimitations(rule, partitionNum, tabletNum, cardinality);
}
}
// match user rule
String[] bindSqlBlockRules = Env.getCurrentEnv().getAuth().getSqlBlockRules(user);
Expand Down
10 changes: 10 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.NereidsSqlCacheManager;
import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.UserException;
Expand Down Expand Up @@ -530,6 +531,8 @@ public class Env {

private DNSCache dnsCache;

private final NereidsSqlCacheManager sqlCacheManager;

public List<TFrontendInfo> getFrontendInfos() {
List<TFrontendInfo> res = new ArrayList<>();

Expand Down Expand Up @@ -766,6 +769,9 @@ public Env(boolean isCheckpointCatalog) {
this.mtmvService = new MTMVService();
this.insertOverwriteManager = new InsertOverwriteManager();
this.dnsCache = new DNSCache();
this.sqlCacheManager = new NereidsSqlCacheManager(
Config.sql_cache_manage_num, Config.cache_last_version_interval_second
);
}

public static void destroyCheckpoint() {
Expand Down Expand Up @@ -6092,6 +6098,10 @@ public MasterDaemon getTabletStatMgr() {
return tabletStatMgr;
}

public NereidsSqlCacheManager getSqlCacheManager() {
return sqlCacheManager;
}

public void alterMTMVRefreshInfo(AlterMTMVRefreshInfo info) {
AlterMTMV alter = new AlterMTMV(info.getMvName(), info.getRefreshInfo(), MTMVAlterOpType.ALTER_REFRESH_INFO);
this.alter.processAlterMTMV(alter, false);
Expand Down
Loading

0 comments on commit 03bd2a3

Please sign in to comment.