Skip to content

Commit

Permalink
fix digest
Browse files Browse the repository at this point in the history
  • Loading branch information
@924060929
924060929 committed Sep 3, 2024
1 parent 73e2150 commit 8524f7b
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 1 deletion.
6 changes: 5 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
Expand Down Expand Up @@ -373,10 +374,13 @@ private void splitFragments(PhysicalPlan resultPlan) {
normalizer.normalize(cascadesContext.getConnectContext());
if (queryCacheParam.isPresent()) {
fragment.queryCacheParam = queryCacheParam.get();
// after commons-codec 1.14 (include), Hex.encodeHexString will change ByteBuffer.pos,
// so we should copy a new byte buffer to print it
ByteBuffer digestCopy = fragment.queryCacheParam.digest.duplicate();
LOG.info("Use query cache for fragment {}, node id: {}, digest: {}, queryId: {}",
seq,
fragment.queryCacheParam.node_id,
Hex.encodeHexString(fragment.queryCacheParam.digest), queryId);
Hex.encodeHexString(digestCopy), queryId);
}
} catch (Throwable t) {
// do nothing
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,26 +122,31 @@ protected void runBeforeAll() throws Exception {
public void testNormalize() throws Exception {
String digest1 = getDigest("select k1 as k, sum(v1) as v from db1.non_part group by 1");
String digest2 = getDigest("select sum(v1) as v1, k1 as k from db1.non_part group by 2");
Assertions.assertEquals(64, digest1.length());
Assertions.assertEquals(digest1, digest2);

String digest3 = getDigest("select k1 as k, sum(v1) as v from db1.non_part where v1 between 1 and 10 group by 1");
Assertions.assertNotEquals(digest1, digest3);

String digest4 = getDigest("select k1 as k, sum(v1) as v from db1.non_part where v1 >= 1 and v1 <= 10 group by 1");
Assertions.assertEquals(digest3, digest4);
Assertions.assertEquals(64, digest3.length());

String digest5 = getDigest("select k1 as k, sum(v1) as v from db1.non_part where v1 >= 1 and v1 < 11 group by 1");
Assertions.assertNotEquals(digest3, digest5);
Assertions.assertEquals(64, digest5.length());
}

@Test
public void testProjectOnOlapScan() throws Exception {
String digest1 = getDigest("select k1 + 1, k2, sum(v1), sum(v2) as v from db1.non_part group by 1, 2");
String digest2 = getDigest("select sum(v2), k2, sum(v1), k1 + 1 from db1.non_part group by 2, 4");
Assertions.assertEquals(digest1, digest2);
Assertions.assertEquals(64, digest1.length());

String digest3 = getDigest("select k1 + 1, k2, sum(v1 + 1), sum(v2) as v from db1.non_part group by 1, 2");
Assertions.assertNotEquals(digest1, digest3);
Assertions.assertEquals(64, digest3.length());
}

@Test
Expand All @@ -156,6 +161,7 @@ public void testProjectOnAggregate() throws Exception {
"select sum(v2) + 4, k2 + 2, sum(v1) + 3, k1 + 1 as v from db1.non_part group by k2, k1"
);
Assertions.assertEquals(digest1, digest2);
Assertions.assertEquals(64, digest1.length());
} finally {
connectContext.getSessionVariable()
.setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
Expand All @@ -169,6 +175,7 @@ public void testPartitionTable() throws Throwable {
TQueryCacheParam queryCacheParam2 = getQueryCacheParam(
"select k1 as k, sum(v1) as v from db1.part1 where dt < '2025-01-01' group by 1");
Assertions.assertEquals(queryCacheParam1.digest, queryCacheParam2.digest);
Assertions.assertEquals(32, queryCacheParam1.digest.remaining());
Assertions.assertEquals(queryCacheParam1.tablet_to_range, queryCacheParam2.tablet_to_range);

TQueryCacheParam queryCacheParam3 = getQueryCacheParam(
Expand All @@ -178,6 +185,7 @@ public void testPartitionTable() throws Throwable {
Lists.newArrayList(queryCacheParam1.tablet_to_range.values()),
Lists.newArrayList(queryCacheParam3.tablet_to_range.values())
);
Assertions.assertEquals(32, queryCacheParam3.digest.remaining());

TQueryCacheParam queryCacheParam4 = getQueryCacheParam(
"select k1 as k, sum(v1) as v from db1.part1 where dt < '2024-04-20' group by 1");
Expand All @@ -186,6 +194,7 @@ public void testPartitionTable() throws Throwable {
Lists.newArrayList(queryCacheParam1.tablet_to_range.values()),
Lists.newArrayList(queryCacheParam4.tablet_to_range.values())
);
Assertions.assertEquals(32, queryCacheParam4.digest.remaining());
}

@Test
Expand Down Expand Up @@ -238,6 +247,7 @@ public void testSelectFromWhereNoGroupBy() throws Throwable {
TQueryCacheParam queryCacheParam1 = getQueryCacheParam("select sum(v1) from db1.part1");
TQueryCacheParam queryCacheParam2 = getQueryCacheParam("select sum(v1) from db1.part1 where dt <= '2024-04-20'");
Assertions.assertEquals(queryCacheParam1.digest, queryCacheParam2.digest);
Assertions.assertEquals(32, queryCacheParam1.digest.remaining());
Assertions.assertNotEquals(
Lists.newArrayList(queryCacheParam1.tablet_to_range.values()),
Lists.newArrayList(queryCacheParam2.tablet_to_range.values())
Expand Down Expand Up @@ -277,6 +287,7 @@ public void testSelectFromWhereNoGroupBy() throws Throwable {
"select sum(v2 + 2) + 2, sum(v2), sum(v1 + 1), sum(v1) from db1.part1"
);
Assertions.assertEquals(queryCacheParam3.digest, queryCacheParam4.digest);
Assertions.assertEquals(32, queryCacheParam3.digest.remaining());
Assertions.assertEquals(queryCacheParam3.tablet_to_range, queryCacheParam4.tablet_to_range);

TQueryCacheParam queryCacheParam5 = getQueryCacheParam(
Expand All @@ -286,6 +297,8 @@ public void testSelectFromWhereNoGroupBy() throws Throwable {
"select sum(v1) from db1.part1 where dt between '2024-04-15' and '2024-04-20' and k1 = 2"
);
Assertions.assertNotEquals(queryCacheParam5.digest, queryCacheParam6.digest);
Assertions.assertEquals(32, queryCacheParam5.digest.remaining());
Assertions.assertEquals(32, queryCacheParam6.digest.remaining());
Assertions.assertEquals(queryCacheParam5.tablet_to_range, queryCacheParam6.tablet_to_range);
}

Expand All @@ -295,6 +308,8 @@ public void testSelectFromGroupBy() {
TQueryCacheParam queryCacheParam1 = getQueryCacheParam("select k1, sum(v1) from db1.part1 group by k1");
TQueryCacheParam queryCacheParam2 = getQueryCacheParam("select k1, sum(v1) from db1.part1 group by k1 limit 10");
Assertions.assertNotEquals(queryCacheParam1.digest, queryCacheParam2.digest);
Assertions.assertEquals(32, queryCacheParam1.digest.remaining());
Assertions.assertEquals(32, queryCacheParam2.digest.remaining());

TQueryCacheParam queryCacheParam3 = getQueryCacheParam("select sum(v1), k1 from db1.part1 group by k1");
Assertions.assertEquals(queryCacheParam1.digest, queryCacheParam3.digest);
Expand Down

0 comments on commit 8524f7b

Please sign in to comment.