diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index bb34edba48d1a3..8472ec78ad36c1 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1303,13 +1303,6 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = false) public static boolean cache_enable_sql_mode = true; - /** - * If set to true, fe will get data from be cache, - * This option is suitable for real-time updating of partial partitions. - */ - @ConfField(mutable = true, masterOnly = false) - public static boolean cache_enable_partition_mode = true; - /** * Minimum interval between last version when caching results, * This parameter distinguishes between offline and real-time updates diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index af2b8f971484f4..cb9b9dafd89eec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -139,7 +139,6 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_REWRITE_ELEMENT_AT_TO_SLOT = "enable_rewrite_element_at_to_slot"; public static final String ENABLE_ODBC_TRANSCATION = "enable_odbc_transcation"; public static final String ENABLE_SQL_CACHE = "enable_sql_cache"; - public static final String ENABLE_PARTITION_CACHE = "enable_partition_cache"; public static final String ENABLE_COST_BASED_JOIN_REORDER = "enable_cost_based_join_reorder"; @@ -858,9 +857,6 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_SQL_CACHE) public boolean enableSqlCache = false; - @VariableMgr.VarAttr(name = ENABLE_PARTITION_CACHE) - public boolean enablePartitionCache = false; - @VariableMgr.VarAttr(name = FORWARD_TO_MASTER) public boolean forwardToMaster = true; @@ -2475,14 +2471,6 @@ public void setEnableSqlCache(boolean enableSqlCache) { this.enableSqlCache = enableSqlCache; } - public boolean isEnablePartitionCache() { - return enablePartitionCache; - } - - public void setEnablePartitionCache(boolean enablePartitionCache) { - this.enablePartitionCache = enablePartitionCache; - } - public int getPartitionedHashJoinRowsThreshold() { return partitionedHashJoinRowsThreshold; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 47fccfcd37a9c9..00c197b27da305 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -142,11 +142,12 @@ private void checkCacheConfig() { enableSqlCache = true; } } - if (Config.cache_enable_partition_mode) { - if (context.getSessionVariable().isEnablePartitionCache()) { - enablePartitionCache = true; - } - } + enablePartitionCache = false; + // if (Config.cache_enable_partition_mode) { + // if (context.getSessionVariable().isEnablePartitionCache()) { + // enablePartitionCache = true; + // } + // } } public TUniqueId getQueryId() { @@ -206,8 +207,9 @@ public boolean enablePartitionCache() { } public static boolean canUseCache(SessionVariable sessionVariable) { - return (sessionVariable.isEnableSqlCache() || sessionVariable.isEnablePartitionCache()) - && commonCacheCondition(sessionVariable); + // return (sessionVariable.isEnableSqlCache() || sessionVariable.isEnablePartitionCache()) + // && commonCacheCondition(sessionVariable); + return (sessionVariable.isEnableSqlCache()) && commonCacheCondition(sessionVariable); } public static boolean canUseSqlCache(SessionVariable sessionVariable) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index f342ca697b4c52..e1d5d3f1d9865f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -82,7 +82,7 @@ protected void runBeforeAll() throws Exception { FeConstants.runningUnitTest = true; Config.enable_query_hive_views = true; Config.cache_enable_sql_mode = true; - Config.cache_enable_partition_mode = true; + // Config.cache_enable_partition_mode = true; connectContext.getSessionVariable().setEnableSqlCache(true); env = Env.getCurrentEnv(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java index 86a1bc17834f59..b152bdc3909647 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -70,7 +70,6 @@ import org.apache.doris.qe.cache.CacheAnalyzer.CacheMode; import org.apache.doris.qe.cache.CacheCoordinator; import org.apache.doris.qe.cache.CacheProxy; -import org.apache.doris.qe.cache.PartitionCache; import org.apache.doris.qe.cache.PartitionRange; import org.apache.doris.qe.cache.RowBatchBuilder; import org.apache.doris.qe.cache.SqlCache; @@ -125,9 +124,7 @@ public static void start() { FrontendOptions.init(); context = new ConnectContext(); Config.cache_enable_sql_mode = true; - Config.cache_enable_partition_mode = true; context.getSessionVariable().setEnableSqlCache(true); - context.getSessionVariable().setEnablePartitionCache(true); Config.cache_last_version_interval_second = 7200; } catch (UnknownHostException e) { @@ -610,20 +607,20 @@ public void testWithinMinTime() throws Exception { Assert.assertEquals(ca.getCacheMode(), CacheMode.None); } - @Test - public void testPartitionModel() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, COUNT(DISTINCT userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " - + "eventdate<=\"2020-01-15\" GROUP BY eventdate" - ); - - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); - } + // @Test + // public void testPartitionModel() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, COUNT(DISTINCT userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + // + "eventdate<=\"2020-01-15\" GROUP BY eventdate" + // ); + // + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); + // } @Test public void testParseByte() throws Exception { @@ -637,90 +634,90 @@ public void testParseByte() throws Exception { Assert.assertEquals(key2.realValue(), 3); } - @Test - public void testPartitionIntTypeSql() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200112 and `date`<=20200115 GROUP BY date" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createOrderScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - LOG.warn("Rewrite partition range size={}", size); - Assert.assertEquals(size, 4); - - String sql; - range.setCacheFlag(20200112L); //get data from cache - range.setCacheFlag(20200113L); //get data from cache - - hitRange = range.buildDiskPartitionRange(newRangeList); - Assert.assertEquals(hitRange, Cache.HitRange.Left); - Assert.assertEquals(newRangeList.size(), 2); - Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200114); - Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200115); - - cache.rewriteSelectStmt(newRangeList); - sql = ca.getRewriteStmt().getWhereClause().toSql(); - Assert.assertEquals(sql, "(`date` >= 20200114) AND (`date` <= 20200115)"); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testSimpleCacheSql() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " - + "eventdate<=\"2020-01-15\" GROUP BY eventdate" - ); - - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - LOG.warn("Rewrite partition range size={}", size); - Assert.assertEquals(size, 4); - - String sql; - range.setCacheFlag(20200112L); //get data from cache - range.setCacheFlag(20200113L); //get data from cache - - hitRange = range.buildDiskPartitionRange(newRangeList); - cache.rewriteSelectStmt(newRangeList); - sql = ca.getRewriteStmt().getWhereClause().toSql(); - Assert.assertEquals(sql, "(`eventdate` >= '2020-01-14') AND (`eventdate` <= '2020-01-15')"); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } + // @Test + // public void testPartitionIntTypeSql() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200112 and `date`<=20200115 GROUP BY date" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createOrderScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // LOG.warn("Rewrite partition range size={}", size); + // Assert.assertEquals(size, 4); + // + // String sql; + // range.setCacheFlag(20200112L); //get data from cache + // range.setCacheFlag(20200113L); //get data from cache + // + // hitRange = range.buildDiskPartitionRange(newRangeList); + // Assert.assertEquals(hitRange, Cache.HitRange.Left); + // Assert.assertEquals(newRangeList.size(), 2); + // Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200114); + // Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200115); + // + // cache.rewriteSelectStmt(newRangeList); + // sql = ca.getRewriteStmt().getWhereClause().toSql(); + // Assert.assertEquals(sql, "(`date` >= 20200114) AND (`date` <= 20200115)"); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testSimpleCacheSql() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + // + "eventdate<=\"2020-01-15\" GROUP BY eventdate" + // ); + // + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // LOG.warn("Rewrite partition range size={}", size); + // Assert.assertEquals(size, 4); + // + // String sql; + // range.setCacheFlag(20200112L); //get data from cache + // range.setCacheFlag(20200113L); //get data from cache + // + // hitRange = range.buildDiskPartitionRange(newRangeList); + // cache.rewriteSelectStmt(newRangeList); + // sql = ca.getRewriteStmt().getWhereClause().toSql(); + // Assert.assertEquals(sql, "(`eventdate` >= '2020-01-14') AND (`eventdate` <= '2020-01-15')"); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } @Test public void testHitSqlCache() throws Exception { @@ -736,306 +733,305 @@ public void testHitSqlCache() throws Exception { Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); } - @Test - public void testHitPartPartition() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " - + "eventdate<=\"2020-01-14\" GROUP BY eventdate" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - LOG.warn("Rewrite partition range size={}", size); - Assert.assertEquals(size, 3); - - range.setCacheFlag(20200113); - range.setCacheFlag(20200114); - - hitRange = range.buildDiskPartitionRange(newRangeList); - Assert.assertEquals(hitRange, Cache.HitRange.Right); - Assert.assertEquals(newRangeList.size(), 2); - Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200112); - Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200112); - - List updateRangeList = range.buildUpdatePartitionRange(); - Assert.assertEquals(updateRangeList.size(), 1); - Assert.assertEquals(updateRangeList.get(0).getCacheKey().realValue(), 20200112); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testNoUpdatePartition() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " - + "eventdate<=\"2020-01-14\" GROUP BY eventdate" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - LOG.warn("Rewrite partition range size={}", size); - Assert.assertEquals(size, 3); - - range.setCacheFlag(20200112); //get data from cache - range.setCacheFlag(20200113); - range.setCacheFlag(20200114); - - hitRange = range.buildDiskPartitionRange(newRangeList); - Assert.assertEquals(hitRange, Cache.HitRange.Full); - Assert.assertEquals(newRangeList.size(), 0); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } - - - @Test - public void testUpdatePartition() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " - + "eventdate<=\"2020-01-15\" GROUP BY eventdate" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - LOG.warn("Rewrite partition range size={}", size); - Assert.assertEquals(size, 4); - - String sql; - range.setCacheFlag(20200112L); //get data from cache - range.setTooNewByKey(20200115); - - range.buildDiskPartitionRange(newRangeList); - Assert.assertEquals(newRangeList.size(), 2); - cache.rewriteSelectStmt(newRangeList); - - sql = ca.getRewriteStmt().getWhereClause().toSql(); - Assert.assertEquals(sql, "(`eventdate` >= '2020-01-13') AND (`eventdate` <= '2020-01-15')"); - - List updateRangeList = range.buildUpdatePartitionRange(); - Assert.assertEquals(updateRangeList.size(), 2); - Assert.assertEquals(updateRangeList.get(0).getCacheKey().realValue(), 20200113); - Assert.assertEquals(updateRangeList.get(1).getCacheKey().realValue(), 20200114); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testRewriteMultiPredicate1() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>\"2020-01-11\" and " - + "eventdate<\"2020-01-16\"" - + " and eventid=1 GROUP BY eventdate" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - - cache.rewriteSelectStmt(null); - LOG.warn("Nokey multi={}", cache.getNokeyStmt().getWhereClause().toSql()); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause().toSql(), "(`eventid` = 1)"); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - Assert.assertEquals(size, 4); - - String sql; - range.setCacheFlag(20200112L); //get data from cache - range.setCacheFlag(20200113L); //get data from cache - - range.buildDiskPartitionRange(newRangeList); - - cache.rewriteSelectStmt(newRangeList); - sql = ca.getRewriteStmt().getWhereClause().toSql(); - LOG.warn("MultiPredicate={}", sql); - Assert.assertEquals(sql, "(`eventdate` > '2020-01-13') AND (`eventdate` < '2020-01-16') AND (`eventid` = 1)"); - } catch (Exception e) { - LOG.warn("multi ex={}", e); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testRewriteJoin() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT appevent.eventdate, country, COUNT(appevent.userid) FROM appevent" - + " INNER JOIN userprofile ON appevent.userid = userprofile.userid" - + " WHERE appevent.eventdate>=\"2020-01-12\" and appevent.eventdate<=\"2020-01-15\"" - + " and eventid=1 GROUP BY appevent.eventdate, country" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - cache.rewriteSelectStmt(null); - LOG.warn("Join nokey={}", cache.getNokeyStmt().getWhereClause().toSql()); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause().toSql(), "(`eventid` = 1)"); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - Assert.assertEquals(size, 4); - - String sql; - range.setCacheFlag(20200112L); //get data from cache - range.setCacheFlag(20200113L); //get data from cache - - range.buildDiskPartitionRange(newRangeList); - - cache.rewriteSelectStmt(newRangeList); - sql = ca.getRewriteStmt().getWhereClause().toSql(); - LOG.warn("Join rewrite={}", sql); - Assert.assertEquals(sql, "(`appevent`.`eventdate` >= '2020-01-14')" - + " AND (`appevent`.`eventdate` <= '2020-01-15') AND (`eventid` = 1)"); - } catch (Exception e) { - LOG.warn("Join ex={}", e); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testSubSelect() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, sum(pv) FROM (SELECT eventdate, COUNT(userid) AS pv FROM appevent WHERE " - + "eventdate>\"2020-01-11\" AND eventdate<\"2020-01-16\"" - + " AND eventid=1 GROUP BY eventdate) tbl GROUP BY eventdate" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - - cache.rewriteSelectStmt(null); - LOG.warn("Sub nokey={}", cache.getNokeyStmt().toSql()); - Assert.assertEquals(cache.getNokeyStmt().toSql(), - "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " - + "FROM (SELECT `eventdate`, count(`userid`) `pv` " - + "FROM `testDb`.`appevent` WHERE (`eventid` = 1) GROUP BY `eventdate`) tbl " - + "GROUP BY `eventdate`"); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - Assert.assertEquals(size, 4); - - String sql; - range.setCacheFlag(20200112L); //get data from cache - range.setCacheFlag(20200113L); //get data from cache - - range.buildDiskPartitionRange(newRangeList); - - cache.rewriteSelectStmt(newRangeList); - sql = ca.getRewriteStmt().toSql(); - LOG.warn("Sub rewrite={}", sql); - Assert.assertEquals(sql, - "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " - + "FROM (SELECT `eventdate`, count(`userid`) `pv` " - + "FROM `testDb`.`appevent` WHERE (`eventdate` > '2020-01-13') " - + "AND (`eventdate` < '2020-01-16') AND (`eventid` = 1) GROUP BY `eventdate`) tbl " - + "GROUP BY `eventdate`"); - } catch (Exception e) { - LOG.warn("sub ex={}", e); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testNotHitPartition() throws Exception { - StatementBase parseStmt = parseSql( - "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " - + "eventdate<=\"2020-01-14\" GROUP BY eventdate" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - cache.rewriteSelectStmt(null); - PartitionRange range = cache.getPartitionRange(); - range.analytics(); - hitRange = range.buildDiskPartitionRange(newRangeList); - Assert.assertEquals(hitRange, Cache.HitRange.None); - Assert.assertEquals(newRangeList.size(), 2); - Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200112); - Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200114); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } + // @Test + // public void testHitPartPartition() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + // + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // LOG.warn("Rewrite partition range size={}", size); + // Assert.assertEquals(size, 3); + // + // range.setCacheFlag(20200113); + // range.setCacheFlag(20200114); + // + // hitRange = range.buildDiskPartitionRange(newRangeList); + // Assert.assertEquals(hitRange, Cache.HitRange.Right); + // Assert.assertEquals(newRangeList.size(), 2); + // Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200112); + // Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200112); + // + // List updateRangeList = range.buildUpdatePartitionRange(); + // Assert.assertEquals(updateRangeList.size(), 1); + // Assert.assertEquals(updateRangeList.get(0).getCacheKey().realValue(), 20200112); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testNoUpdatePartition() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + // + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // LOG.warn("Rewrite partition range size={}", size); + // Assert.assertEquals(size, 3); + // + // range.setCacheFlag(20200112); //get data from cache + // range.setCacheFlag(20200113); + // range.setCacheFlag(20200114); + // + // hitRange = range.buildDiskPartitionRange(newRangeList); + // Assert.assertEquals(hitRange, Cache.HitRange.Full); + // Assert.assertEquals(newRangeList.size(), 0); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testUpdatePartition() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + // + "eventdate<=\"2020-01-15\" GROUP BY eventdate" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // LOG.warn("Rewrite partition range size={}", size); + // Assert.assertEquals(size, 4); + // + // String sql; + // range.setCacheFlag(20200112L); //get data from cache + // range.setTooNewByKey(20200115); + // + // range.buildDiskPartitionRange(newRangeList); + // Assert.assertEquals(newRangeList.size(), 2); + // cache.rewriteSelectStmt(newRangeList); + // + // sql = ca.getRewriteStmt().getWhereClause().toSql(); + // Assert.assertEquals(sql, "(`eventdate` >= '2020-01-13') AND (`eventdate` <= '2020-01-15')"); + // + // List updateRangeList = range.buildUpdatePartitionRange(); + // Assert.assertEquals(updateRangeList.size(), 2); + // Assert.assertEquals(updateRangeList.get(0).getCacheKey().realValue(), 20200113); + // Assert.assertEquals(updateRangeList.get(1).getCacheKey().realValue(), 20200114); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testRewriteMultiPredicate1() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>\"2020-01-11\" and " + // + "eventdate<\"2020-01-16\"" + // + " and eventid=1 GROUP BY eventdate" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // + // cache.rewriteSelectStmt(null); + // LOG.warn("Nokey multi={}", cache.getNokeyStmt().getWhereClause().toSql()); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause().toSql(), "(`eventid` = 1)"); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // Assert.assertEquals(size, 4); + // + // String sql; + // range.setCacheFlag(20200112L); //get data from cache + // range.setCacheFlag(20200113L); //get data from cache + // + // range.buildDiskPartitionRange(newRangeList); + // + // cache.rewriteSelectStmt(newRangeList); + // sql = ca.getRewriteStmt().getWhereClause().toSql(); + // LOG.warn("MultiPredicate={}", sql); + // Assert.assertEquals(sql, "(`eventdate` > '2020-01-13') AND (`eventdate` < '2020-01-16') AND (`eventid` = 1)"); + // } catch (Exception e) { + // LOG.warn("multi ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testRewriteJoin() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT appevent.eventdate, country, COUNT(appevent.userid) FROM appevent" + // + " INNER JOIN userprofile ON appevent.userid = userprofile.userid" + // + " WHERE appevent.eventdate>=\"2020-01-12\" and appevent.eventdate<=\"2020-01-15\"" + // + " and eventid=1 GROUP BY appevent.eventdate, country" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // cache.rewriteSelectStmt(null); + // LOG.warn("Join nokey={}", cache.getNokeyStmt().getWhereClause().toSql()); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause().toSql(), "(`eventid` = 1)"); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // Assert.assertEquals(size, 4); + // + // String sql; + // range.setCacheFlag(20200112L); //get data from cache + // range.setCacheFlag(20200113L); //get data from cache + // + // range.buildDiskPartitionRange(newRangeList); + // + // cache.rewriteSelectStmt(newRangeList); + // sql = ca.getRewriteStmt().getWhereClause().toSql(); + // LOG.warn("Join rewrite={}", sql); + // Assert.assertEquals(sql, "(`appevent`.`eventdate` >= '2020-01-14')" + // + " AND (`appevent`.`eventdate` <= '2020-01-15') AND (`eventid` = 1)"); + // } catch (Exception e) { + // LOG.warn("Join ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testSubSelect() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, sum(pv) FROM (SELECT eventdate, COUNT(userid) AS pv FROM appevent WHERE " + // + "eventdate>\"2020-01-11\" AND eventdate<\"2020-01-16\"" + // + " AND eventid=1 GROUP BY eventdate) tbl GROUP BY eventdate" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // + // cache.rewriteSelectStmt(null); + // LOG.warn("Sub nokey={}", cache.getNokeyStmt().toSql()); + // Assert.assertEquals(cache.getNokeyStmt().toSql(), + // "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " + // + "FROM (SELECT `eventdate`, count(`userid`) `pv` " + // + "FROM `testDb`.`appevent` WHERE (`eventid` = 1) GROUP BY `eventdate`) tbl " + // + "GROUP BY `eventdate`"); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // Assert.assertEquals(size, 4); + // + // String sql; + // range.setCacheFlag(20200112L); //get data from cache + // range.setCacheFlag(20200113L); //get data from cache + // + // range.buildDiskPartitionRange(newRangeList); + // + // cache.rewriteSelectStmt(newRangeList); + // sql = ca.getRewriteStmt().toSql(); + // LOG.warn("Sub rewrite={}", sql); + // Assert.assertEquals(sql, + // "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " + // + "FROM (SELECT `eventdate`, count(`userid`) `pv` " + // + "FROM `testDb`.`appevent` WHERE (`eventdate` > '2020-01-13') " + // + "AND (`eventdate` < '2020-01-16') AND (`eventid` = 1) GROUP BY `eventdate`) tbl " + // + "GROUP BY `eventdate`"); + // } catch (Exception e) { + // LOG.warn("sub ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testNotHitPartition() throws Exception { + // StatementBase parseStmt = parseSql( + // "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + // + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // cache.rewriteSelectStmt(null); + // PartitionRange range = cache.getPartitionRange(); + // range.analytics(); + // hitRange = range.buildDiskPartitionRange(newRangeList); + // Assert.assertEquals(hitRange, Cache.HitRange.None); + // Assert.assertEquals(newRangeList.size(), 2); + // Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200112); + // Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200114); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } @Test public void testSqlCacheKey() { @@ -1171,64 +1167,64 @@ public void testSqlCacheKeyWithSubSelectViewForNereids() { Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } - @Test - public void testPartitionCacheKeyWithView() { - StatementBase parseStmt = parseSql("SELECT * from testDb.view3"); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testDb`.`view3`.`eventdate` " - + "AS `eventdate`, `testDb`.`view3`.`__count_1` AS `__count_1` " - + "FROM `testDb`.`view3`|SELECT `eventdate` AS `eventdate`, count(`userid`) " - + "AS `__count_1` FROM `testDb`.`appevent` WHERE (`eventdate` >= '2020-01-12') " - + "AND (`eventdate` <= '2020-01-15') GROUP BY `eventdate`"); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testPartitionCacheKeyWithSubSelectView() { - StatementBase parseStmt = parseSql( - "select origin.eventdate as eventdate, origin.cnt as cnt\n" - + "from (\n" - + " SELECT eventdate, COUNT(userid) as cnt \n" - + " FROM view2 \n" - + " WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate\n" - + ") origin" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - Assert.assertEquals(cache.getSqlWithViewStmt(), - "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` " - + "FROM (SELECT `eventdate`, count(`userid`) `cnt` " - + "FROM `testDb`.`view2` GROUP BY `eventdate`) origin|SELECT `eventdate` " - + "AS `eventdate`, `userid` AS `userid` FROM `testDb`.`appevent`"); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } + // @Test + // public void testPartitionCacheKeyWithView() { + // StatementBase parseStmt = parseSql("SELECT * from testDb.view3"); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testDb`.`view3`.`eventdate` " + // + "AS `eventdate`, `testDb`.`view3`.`__count_1` AS `__count_1` " + // + "FROM `testDb`.`view3`|SELECT `eventdate` AS `eventdate`, count(`userid`) " + // + "AS `__count_1` FROM `testDb`.`appevent` WHERE (`eventdate` >= '2020-01-12') " + // + "AND (`eventdate` <= '2020-01-15') GROUP BY `eventdate`"); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } + + // @Test + // public void testPartitionCacheKeyWithSubSelectView() { + // StatementBase parseStmt = parseSql( + // "select origin.eventdate as eventdate, origin.cnt as cnt\n" + // + "from (\n" + // + " SELECT eventdate, COUNT(userid) as cnt \n" + // + " FROM view2 \n" + // + " WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate\n" + // + ") origin" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // Assert.assertEquals(cache.getSqlWithViewStmt(), + // "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` " + // + "FROM (SELECT `eventdate`, count(`userid`) `cnt` " + // + "FROM `testDb`.`view2` GROUP BY `eventdate`) origin|SELECT `eventdate` " + // + "AS `eventdate`, `userid` AS `userid` FROM `testDb`.`appevent`"); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } @Test public void testSqlCacheKeyWithNestedView() { @@ -1291,47 +1287,47 @@ public void testCacheLocalViewMultiOperand() { Assert.assertEquals(selectedPartitionIds.size() * 3, ((SqlCache) ca.getCache()).getSumOfPartitionNum()); } - @Test - // test that some partitions do not exist in the table - public void testNotExistPartitionSql() { - StatementBase parseStmt = parseSql( - "SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200110 and `date`<=20200115 GROUP BY date" - ); - ArrayList selectedPartitionIds - = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); - List scanNodes = Lists.newArrayList(createOrderScanNode(selectedPartitionIds)); - CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); - ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 - Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first - try { - PartitionCache cache = (PartitionCache) ca.getCache(); - cache.rewriteSelectStmt(null); - Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - - PartitionRange range = cache.getPartitionRange(); - boolean flag = range.analytics(); - Assert.assertEquals(flag, true); - - int size = range.getPartitionSingleList().size(); - LOG.warn("Rewrite partition range size={}", size); - Assert.assertEquals(size, 4); - - String sql; - range.setCacheFlag(20200112L); //get data from cache - range.setCacheFlag(20200113L); //get data from cache - - hitRange = range.buildDiskPartitionRange(newRangeList); - Assert.assertEquals(hitRange, Cache.HitRange.Left); - Assert.assertEquals(newRangeList.size(), 2); - Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200114); - Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200115); - - cache.rewriteSelectStmt(newRangeList); - sql = ca.getRewriteStmt().getWhereClause().toSql(); - Assert.assertEquals(sql, "(`date` >= 20200114) AND (`date` <= 20200115)"); - } catch (Exception e) { - LOG.warn("ex={}", e); - Assert.fail(e.getMessage()); - } - } + // @Test + // // test that some partitions do not exist in the table + // public void testNotExistPartitionSql() { + // StatementBase parseStmt = parseSql( + // "SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200110 and `date`<=20200115 GROUP BY date" + // ); + // ArrayList selectedPartitionIds + // = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + // List scanNodes = Lists.newArrayList(createOrderScanNode(selectedPartitionIds)); + // CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + // ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + // Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + // try { + // PartitionCache cache = (PartitionCache) ca.getCache(); + // cache.rewriteSelectStmt(null); + // Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + // + // PartitionRange range = cache.getPartitionRange(); + // boolean flag = range.analytics(); + // Assert.assertEquals(flag, true); + // + // int size = range.getPartitionSingleList().size(); + // LOG.warn("Rewrite partition range size={}", size); + // Assert.assertEquals(size, 4); + // + // String sql; + // range.setCacheFlag(20200112L); //get data from cache + // range.setCacheFlag(20200113L); //get data from cache + // + // hitRange = range.buildDiskPartitionRange(newRangeList); + // Assert.assertEquals(hitRange, Cache.HitRange.Left); + // Assert.assertEquals(newRangeList.size(), 2); + // Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200114); + // Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200115); + // + // cache.rewriteSelectStmt(newRangeList); + // sql = ca.getRewriteStmt().getWhereClause().toSql(); + // Assert.assertEquals(sql, "(`date` >= 20200114) AND (`date` <= 20200115)"); + // } catch (Exception e) { + // LOG.warn("ex={}", e); + // Assert.fail(e.getMessage()); + // } + // } }