From 8524f7bd0d432243871a5fdef6d7e868cb29f188 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 3 Sep 2024 20:28:04 +0800 Subject: [PATCH] fix digest --- .../org/apache/doris/nereids/NereidsPlanner.java | 6 +++++- .../doris/planner/QueryCacheNormalizerTest.java | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index ccf70456534396e..a304fe36062c80b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -79,6 +79,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -373,10 +374,13 @@ private void splitFragments(PhysicalPlan resultPlan) { normalizer.normalize(cascadesContext.getConnectContext()); if (queryCacheParam.isPresent()) { fragment.queryCacheParam = queryCacheParam.get(); + // after commons-codec 1.14 (include), Hex.encodeHexString will change ByteBuffer.pos, + // so we should copy a new byte buffer to print it + ByteBuffer digestCopy = fragment.queryCacheParam.digest.duplicate(); LOG.info("Use query cache for fragment {}, node id: {}, digest: {}, queryId: {}", seq, fragment.queryCacheParam.node_id, - Hex.encodeHexString(fragment.queryCacheParam.digest), queryId); + Hex.encodeHexString(digestCopy), queryId); } } catch (Throwable t) { // do nothing diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java index da2c6884cf0a317..829d1b0d1e507c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java @@ -122,6 +122,7 @@ protected void runBeforeAll() throws Exception { public void testNormalize() throws Exception { String digest1 = getDigest("select k1 as k, sum(v1) as v from db1.non_part group by 1"); String digest2 = getDigest("select sum(v1) as v1, k1 as k from db1.non_part group by 2"); + Assertions.assertEquals(64, digest1.length()); Assertions.assertEquals(digest1, digest2); String digest3 = getDigest("select k1 as k, sum(v1) as v from db1.non_part where v1 between 1 and 10 group by 1"); @@ -129,9 +130,11 @@ public void testNormalize() throws Exception { String digest4 = getDigest("select k1 as k, sum(v1) as v from db1.non_part where v1 >= 1 and v1 <= 10 group by 1"); Assertions.assertEquals(digest3, digest4); + Assertions.assertEquals(64, digest3.length()); String digest5 = getDigest("select k1 as k, sum(v1) as v from db1.non_part where v1 >= 1 and v1 < 11 group by 1"); Assertions.assertNotEquals(digest3, digest5); + Assertions.assertEquals(64, digest5.length()); } @Test @@ -139,9 +142,11 @@ public void testProjectOnOlapScan() throws Exception { String digest1 = getDigest("select k1 + 1, k2, sum(v1), sum(v2) as v from db1.non_part group by 1, 2"); String digest2 = getDigest("select sum(v2), k2, sum(v1), k1 + 1 from db1.non_part group by 2, 4"); Assertions.assertEquals(digest1, digest2); + Assertions.assertEquals(64, digest1.length()); String digest3 = getDigest("select k1 + 1, k2, sum(v1 + 1), sum(v2) as v from db1.non_part group by 1, 2"); Assertions.assertNotEquals(digest1, digest3); + Assertions.assertEquals(64, digest3.length()); } @Test @@ -156,6 +161,7 @@ public void testProjectOnAggregate() throws Exception { "select sum(v2) + 4, k2 + 2, sum(v1) + 3, k1 + 1 as v from db1.non_part group by k2, k1" ); Assertions.assertEquals(digest1, digest2); + Assertions.assertEquals(64, digest1.length()); } finally { connectContext.getSessionVariable() .setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); @@ -169,6 +175,7 @@ public void testPartitionTable() throws Throwable { TQueryCacheParam queryCacheParam2 = getQueryCacheParam( "select k1 as k, sum(v1) as v from db1.part1 where dt < '2025-01-01' group by 1"); Assertions.assertEquals(queryCacheParam1.digest, queryCacheParam2.digest); + Assertions.assertEquals(32, queryCacheParam1.digest.remaining()); Assertions.assertEquals(queryCacheParam1.tablet_to_range, queryCacheParam2.tablet_to_range); TQueryCacheParam queryCacheParam3 = getQueryCacheParam( @@ -178,6 +185,7 @@ public void testPartitionTable() throws Throwable { Lists.newArrayList(queryCacheParam1.tablet_to_range.values()), Lists.newArrayList(queryCacheParam3.tablet_to_range.values()) ); + Assertions.assertEquals(32, queryCacheParam3.digest.remaining()); TQueryCacheParam queryCacheParam4 = getQueryCacheParam( "select k1 as k, sum(v1) as v from db1.part1 where dt < '2024-04-20' group by 1"); @@ -186,6 +194,7 @@ public void testPartitionTable() throws Throwable { Lists.newArrayList(queryCacheParam1.tablet_to_range.values()), Lists.newArrayList(queryCacheParam4.tablet_to_range.values()) ); + Assertions.assertEquals(32, queryCacheParam4.digest.remaining()); } @Test @@ -238,6 +247,7 @@ public void testSelectFromWhereNoGroupBy() throws Throwable { TQueryCacheParam queryCacheParam1 = getQueryCacheParam("select sum(v1) from db1.part1"); TQueryCacheParam queryCacheParam2 = getQueryCacheParam("select sum(v1) from db1.part1 where dt <= '2024-04-20'"); Assertions.assertEquals(queryCacheParam1.digest, queryCacheParam2.digest); + Assertions.assertEquals(32, queryCacheParam1.digest.remaining()); Assertions.assertNotEquals( Lists.newArrayList(queryCacheParam1.tablet_to_range.values()), Lists.newArrayList(queryCacheParam2.tablet_to_range.values()) @@ -277,6 +287,7 @@ public void testSelectFromWhereNoGroupBy() throws Throwable { "select sum(v2 + 2) + 2, sum(v2), sum(v1 + 1), sum(v1) from db1.part1" ); Assertions.assertEquals(queryCacheParam3.digest, queryCacheParam4.digest); + Assertions.assertEquals(32, queryCacheParam3.digest.remaining()); Assertions.assertEquals(queryCacheParam3.tablet_to_range, queryCacheParam4.tablet_to_range); TQueryCacheParam queryCacheParam5 = getQueryCacheParam( @@ -286,6 +297,8 @@ public void testSelectFromWhereNoGroupBy() throws Throwable { "select sum(v1) from db1.part1 where dt between '2024-04-15' and '2024-04-20' and k1 = 2" ); Assertions.assertNotEquals(queryCacheParam5.digest, queryCacheParam6.digest); + Assertions.assertEquals(32, queryCacheParam5.digest.remaining()); + Assertions.assertEquals(32, queryCacheParam6.digest.remaining()); Assertions.assertEquals(queryCacheParam5.tablet_to_range, queryCacheParam6.tablet_to_range); } @@ -295,6 +308,8 @@ public void testSelectFromGroupBy() { TQueryCacheParam queryCacheParam1 = getQueryCacheParam("select k1, sum(v1) from db1.part1 group by k1"); TQueryCacheParam queryCacheParam2 = getQueryCacheParam("select k1, sum(v1) from db1.part1 group by k1 limit 10"); Assertions.assertNotEquals(queryCacheParam1.digest, queryCacheParam2.digest); + Assertions.assertEquals(32, queryCacheParam1.digest.remaining()); + Assertions.assertEquals(32, queryCacheParam2.digest.remaining()); TQueryCacheParam queryCacheParam3 = getQueryCacheParam("select sum(v1), k1 from db1.part1 group by k1"); Assertions.assertEquals(queryCacheParam1.digest, queryCacheParam3.digest);