From 5c03743895d2e6455478e8965274c59fda6e6592 Mon Sep 17 00:00:00 2001 From: srlch <111035020+srlch@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:16:42 +0800 Subject: [PATCH] [Enhancement] Delete garbage data throught recycle bin after truncate table in share data mode (#49975) Signed-off-by: srlch --- .../starrocks/catalog/CatalogRecycleBin.java | 5 + .../java/com/starrocks/catalog/OlapTable.java | 9 ++ .../catalog/ReplaceLakePartitionTest.java | 103 ++++++++++++++++-- 3 files changed, 108 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java index 7f7e254507e1d..1f0d53dd8c376 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java @@ -903,6 +903,11 @@ protected void runAfterCatalogReady() { } } + @VisibleForTesting + synchronized boolean isContainedInidToRecycleTime(long id) { + return idToRecycleTime.get(id) != null; + } + @Override public void write(DataOutput out) throws IOException { int count = idToDatabase.size(); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index cfa050c585455..2e9bfce16dd22 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -2105,6 +2105,15 @@ protected OlapTable selectiveCopyInternal(OlapTable copied, Collection r public Partition replacePartition(Partition newPartition) { Partition oldPartition = nameToPartition.remove(newPartition.getName()); + // For cloud native table, add partition into recycle Bin after truncate table. + // It is no necessary for share nothing mode because file will be deleted throught + // tablet report in this case. + if (this.isCloudNativeTableOrMaterializedView()) { + RecyclePartitionInfo recyclePartitionInfo = buildRecyclePartitionInfo(-1, oldPartition); + recyclePartitionInfo.setRecoverable(false); + GlobalStateMgr.getCurrentState().getRecycleBin().recyclePartition(recyclePartitionInfo); + } + oldPartition.getSubPartitions().forEach(physicalPartition -> { physicalPartitionIdToPartitionId.remove(physicalPartition.getId()); physicalPartitionNameToPartitionId.remove(physicalPartition.getName()); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/ReplaceLakePartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/ReplaceLakePartitionTest.java index e2d53779efe01..cf62ab256e2ef 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/ReplaceLakePartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/ReplaceLakePartitionTest.java @@ -14,6 +14,7 @@ package com.starrocks.catalog; import com.google.common.collect.Lists; +import com.google.common.collect.Range; import com.staros.client.StarClientException; import com.staros.proto.FilePathInfo; import com.staros.proto.ShardInfo; @@ -22,14 +23,17 @@ import com.starrocks.catalog.MaterializedIndex; import com.starrocks.catalog.Partition; import com.starrocks.catalog.PartitionInfo; +import com.starrocks.catalog.PartitionKey; import com.starrocks.catalog.PartitionType; import com.starrocks.catalog.TabletInvertedIndex; import com.starrocks.catalog.TabletMeta; import com.starrocks.catalog.Type; +import com.starrocks.common.ExceptionChecker; import com.starrocks.lake.DataCacheInfo; import com.starrocks.lake.LakeTable; import com.starrocks.lake.LakeTablet; import com.starrocks.lake.StarOSAgent; +import com.starrocks.persist.EditLog; import com.starrocks.server.GlobalStateMgr; import com.starrocks.server.WarehouseManager; import com.starrocks.thrift.TStorageMedium; @@ -39,8 +43,11 @@ import mockit.Mock; import mockit.MockUp; import mockit.Mocked; +import org.junit.Assert; import org.junit.Test; +import java.util.List; + public class ReplaceLakePartitionTest { long dbId = 9010; long tableId = 9011; @@ -51,6 +58,8 @@ public class ReplaceLakePartitionTest { long nextTxnId = 20000; String partitionName = "p0"; String tempPartitionName = "temp_" + partitionName; + long[] newTabletId = {9030, 90031}; + long newPartitionId = 9040; LakeTable tbl = null; private final ShardInfo shardInfo; @@ -61,6 +70,9 @@ public class ReplaceLakePartitionTest { @Mocked private WarehouseManager warehouseManager; + @Mocked + private EditLog editLog; + public ReplaceLakePartitionTest() { shardInfo = ShardInfo.newBuilder().setFilePath(FilePathInfo.newBuilder().setFullPath("oss://1/2")).build(); warehouseManager = new WarehouseManager(); @@ -78,7 +90,21 @@ LakeTable buildLakeTableWithTempPartition(PartitionType partitionType) { Partition partition = new Partition(partitionId, partitionName, index, null); Partition tempPartition = new Partition(tempPartitionId, tempPartitionName, index, null); - PartitionInfo partitionInfo = new PartitionInfo(partitionType); + PartitionInfo partitionInfo = null; + if (partitionType == PartitionType.UNPARTITIONED) { + partitionInfo = new PartitionInfo(partitionType); + } else if (partitionType == PartitionType.LIST) { + partitionInfo = new ListPartitionInfo(PartitionType.LIST, Lists.newArrayList(new Column("c0", Type.BIGINT))); + List values = Lists.newArrayList(); + values.add("123"); + ((ListPartitionInfo) partitionInfo).setValues(partitionId, values); + } else if (partitionType == PartitionType.RANGE) { + PartitionKey partitionKey = new PartitionKey(); + Range range = Range.closedOpen(partitionKey, partitionKey); + partitionInfo = new RangePartitionInfo(Lists.newArrayList(new Column("c0", Type.BIGINT))); + ((RangePartitionInfo) partitionInfo).setRange(partitionId, false, range); + } + partitionInfo.setReplicationNum(partitionId, (short) 1); partitionInfo.setIsInMemory(partitionId, false); partitionInfo.setDataCacheInfo(partitionId, new DataCacheInfo(true, false)); @@ -91,12 +117,20 @@ LakeTable buildLakeTableWithTempPartition(PartitionType partitionType) { table.addTempPartition(tempPartition); return table; } + + Partition buildPartitionForTruncateTable() { + MaterializedIndex index = new MaterializedIndex(indexId); + TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); + for (long id : newTabletId) { + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, newPartitionId, 0, 0, TStorageMedium.HDD, true); + invertedIndex.addTablet(id, tabletMeta); + index.addTablet(new LakeTablet(id), tabletMeta); + } - @Test - public void testUnPartitionedLakeTableReplacePartition() { - LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.UNPARTITIONED); - tbl.replacePartition(partitionName, tempPartitionName); + return new Partition(newPartitionId, partitionName, index, null); + } + private void erasePartition() { new Expectations() { { GlobalStateMgr.getCurrentState().getWarehouseMgr(); @@ -105,6 +139,13 @@ public void testUnPartitionedLakeTableReplacePartition() { } }; + new MockUp() { + @Mock + public EditLog getEditLog() { + return editLog; + } + }; + new MockUp() { @Mock public StarOSAgent getStarOSAgent() { @@ -119,6 +160,13 @@ public ShardInfo getShardInfo(long shardId, long workerGroupId) throws StarClien } }; + new MockUp() { + @Mock + public void logErasePartition(long partitionId) { + return; + } + }; + new MockUp() { @Mock public Warehouse getBackgroundWarehouse() { @@ -126,9 +174,46 @@ public Warehouse getBackgroundWarehouse() { } }; - try { - GlobalStateMgr.getCurrentState().getRecycleBin().erasePartition(Long.MAX_VALUE); - } catch (Exception ignore) { - } + ExceptionChecker.expectThrowsNoException(() + -> GlobalStateMgr.getCurrentState().getRecycleBin().erasePartition(Long.MAX_VALUE)); + } + + @Test + public void testUnPartitionedLakeTableReplacePartition() { + LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.UNPARTITIONED); + tbl.replacePartition(partitionName, tempPartitionName); + Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); + erasePartition(); + Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); + } + + @Test + public void testUnPartitionedLakeTableReplacePartitionForTruncateTable() { + LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.UNPARTITIONED); + Partition newPartition = buildPartitionForTruncateTable(); + tbl.replacePartition(newPartition); + Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); + erasePartition(); + Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); + } + + @Test + public void testListPartitionedLakeTableReplacePartitionForTruncateTable() { + LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.LIST); + Partition newPartition = buildPartitionForTruncateTable(); + tbl.replacePartition(newPartition); + Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); + erasePartition(); + Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); + } + + @Test + public void testRangePartitionedLakeTableReplacePartitionForTruncateTable() { + LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.RANGE); + Partition newPartition = buildPartitionForTruncateTable(); + tbl.replacePartition(newPartition); + Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); + erasePartition(); + Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId)); } }