Skip to content

Commit

Permalink
[Enhancement] Delete garbage data throught recycle bin after truncate…
Browse files Browse the repository at this point in the history 
… table in share data mode (#49975)

Signed-off-by: srlch <[email protected]>
  • Loading branch information
@srlch
srlch committed Aug 29, 2024
1 parent 4845e33 commit 5c03743
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 9 deletions.
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -903,6 +903,11 @@ protected void runAfterCatalogReady() {
}
}

@VisibleForTesting
synchronized boolean isContainedInidToRecycleTime(long id) {
return idToRecycleTime.get(id) != null;
}

@Override
public void write(DataOutput out) throws IOException {
int count = idToDatabase.size();
Expand Down
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2105,6 +2105,15 @@ protected OlapTable selectiveCopyInternal(OlapTable copied, Collection<String> r
public Partition replacePartition(Partition newPartition) {
Partition oldPartition = nameToPartition.remove(newPartition.getName());

// For cloud native table, add partition into recycle Bin after truncate table.
// It is no necessary for share nothing mode because file will be deleted throught
// tablet report in this case.
if (this.isCloudNativeTableOrMaterializedView()) {
RecyclePartitionInfo recyclePartitionInfo = buildRecyclePartitionInfo(-1, oldPartition);
recyclePartitionInfo.setRecoverable(false);
GlobalStateMgr.getCurrentState().getRecycleBin().recyclePartition(recyclePartitionInfo);
}

oldPartition.getSubPartitions().forEach(physicalPartition -> {
physicalPartitionIdToPartitionId.remove(physicalPartition.getId());
physicalPartitionNameToPartitionId.remove(physicalPartition.getName());
Expand Down
103 changes: 94 additions & 9 deletions fe/fe-core/src/test/java/com/starrocks/catalog/ReplaceLakePartitionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package com.starrocks.catalog;

import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import com.staros.client.StarClientException;
import com.staros.proto.FilePathInfo;
import com.staros.proto.ShardInfo;
Expand All @@ -22,14 +23,17 @@
import com.starrocks.catalog.MaterializedIndex;
import com.starrocks.catalog.Partition;
import com.starrocks.catalog.PartitionInfo;
import com.starrocks.catalog.PartitionKey;
import com.starrocks.catalog.PartitionType;
import com.starrocks.catalog.TabletInvertedIndex;
import com.starrocks.catalog.TabletMeta;
import com.starrocks.catalog.Type;
import com.starrocks.common.ExceptionChecker;
import com.starrocks.lake.DataCacheInfo;
import com.starrocks.lake.LakeTable;
import com.starrocks.lake.LakeTablet;
import com.starrocks.lake.StarOSAgent;
import com.starrocks.persist.EditLog;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.server.WarehouseManager;
import com.starrocks.thrift.TStorageMedium;
Expand All @@ -39,8 +43,11 @@
import mockit.Mock;
import mockit.MockUp;
import mockit.Mocked;
import org.junit.Assert;
import org.junit.Test;

import java.util.List;

public class ReplaceLakePartitionTest {
long dbId = 9010;
long tableId = 9011;
Expand All @@ -51,6 +58,8 @@ public class ReplaceLakePartitionTest {
long nextTxnId = 20000;
String partitionName = "p0";
String tempPartitionName = "temp_" + partitionName;
long[] newTabletId = {9030, 90031};
long newPartitionId = 9040;

LakeTable tbl = null;
private final ShardInfo shardInfo;
Expand All @@ -61,6 +70,9 @@ public class ReplaceLakePartitionTest {
@Mocked
private WarehouseManager warehouseManager;

@Mocked
private EditLog editLog;

public ReplaceLakePartitionTest() {
shardInfo = ShardInfo.newBuilder().setFilePath(FilePathInfo.newBuilder().setFullPath("oss://1/2")).build();
warehouseManager = new WarehouseManager();
Expand All @@ -78,7 +90,21 @@ LakeTable buildLakeTableWithTempPartition(PartitionType partitionType) {
Partition partition = new Partition(partitionId, partitionName, index, null);
Partition tempPartition = new Partition(tempPartitionId, tempPartitionName, index, null);

PartitionInfo partitionInfo = new PartitionInfo(partitionType);
PartitionInfo partitionInfo = null;
if (partitionType == PartitionType.UNPARTITIONED) {
partitionInfo = new PartitionInfo(partitionType);
} else if (partitionType == PartitionType.LIST) {
partitionInfo = new ListPartitionInfo(PartitionType.LIST, Lists.newArrayList(new Column("c0", Type.BIGINT)));
List<String> values = Lists.newArrayList();
values.add("123");
((ListPartitionInfo) partitionInfo).setValues(partitionId, values);
} else if (partitionType == PartitionType.RANGE) {
PartitionKey partitionKey = new PartitionKey();
Range<PartitionKey> range = Range.closedOpen(partitionKey, partitionKey);
partitionInfo = new RangePartitionInfo(Lists.newArrayList(new Column("c0", Type.BIGINT)));
((RangePartitionInfo) partitionInfo).setRange(partitionId, false, range);
}

partitionInfo.setReplicationNum(partitionId, (short) 1);
partitionInfo.setIsInMemory(partitionId, false);
partitionInfo.setDataCacheInfo(partitionId, new DataCacheInfo(true, false));
Expand All @@ -91,12 +117,20 @@ LakeTable buildLakeTableWithTempPartition(PartitionType partitionType) {
table.addTempPartition(tempPartition);
return table;
}

Partition buildPartitionForTruncateTable() {
MaterializedIndex index = new MaterializedIndex(indexId);
TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex();
for (long id : newTabletId) {
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, newPartitionId, 0, 0, TStorageMedium.HDD, true);
invertedIndex.addTablet(id, tabletMeta);
index.addTablet(new LakeTablet(id), tabletMeta);
}

@Test
public void testUnPartitionedLakeTableReplacePartition() {
LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.UNPARTITIONED);
tbl.replacePartition(partitionName, tempPartitionName);
return new Partition(newPartitionId, partitionName, index, null);
}

private void erasePartition() {
new Expectations() {
{
GlobalStateMgr.getCurrentState().getWarehouseMgr();
Expand All @@ -105,6 +139,13 @@ public void testUnPartitionedLakeTableReplacePartition() {
}
};

new MockUp<GlobalStateMgr>() {
@Mock
public EditLog getEditLog() {
return editLog;
}
};

new MockUp<GlobalStateMgr>() {
@Mock
public StarOSAgent getStarOSAgent() {
Expand All @@ -119,16 +160,60 @@ public ShardInfo getShardInfo(long shardId, long workerGroupId) throws StarClien
}
};

new MockUp<EditLog>() {
@Mock
public void logErasePartition(long partitionId) {
return;
}
};

new MockUp<WarehouseManager>() {
@Mock
public Warehouse getBackgroundWarehouse() {
return new DefaultWarehouse(WarehouseManager.DEFAULT_WAREHOUSE_ID, WarehouseManager.DEFAULT_WAREHOUSE_NAME);
}
};

try {
GlobalStateMgr.getCurrentState().getRecycleBin().erasePartition(Long.MAX_VALUE);
} catch (Exception ignore) {
}
ExceptionChecker.expectThrowsNoException(()
-> GlobalStateMgr.getCurrentState().getRecycleBin().erasePartition(Long.MAX_VALUE));
}

@Test
public void testUnPartitionedLakeTableReplacePartition() {
LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.UNPARTITIONED);
tbl.replacePartition(partitionName, tempPartitionName);
Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
erasePartition();
Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
}

@Test
public void testUnPartitionedLakeTableReplacePartitionForTruncateTable() {
LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.UNPARTITIONED);
Partition newPartition = buildPartitionForTruncateTable();
tbl.replacePartition(newPartition);
Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
erasePartition();
Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
}

@Test
public void testListPartitionedLakeTableReplacePartitionForTruncateTable() {
LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.LIST);
Partition newPartition = buildPartitionForTruncateTable();
tbl.replacePartition(newPartition);
Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
erasePartition();
Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
}

@Test
public void testRangePartitionedLakeTableReplacePartitionForTruncateTable() {
LakeTable tbl = buildLakeTableWithTempPartition(PartitionType.RANGE);
Partition newPartition = buildPartitionForTruncateTable();
tbl.replacePartition(newPartition);
Assert.assertTrue(GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
erasePartition();
Assert.assertTrue(!GlobalStateMgr.getCurrentState().getRecycleBin().isContainedInidToRecycleTime(partitionId));
}
}

0 comments on commit 5c03743

Please sign in to comment.