Skip to content

Commit

Permalink
Refactor PartitionImpl to Logical Partition and Physical Partition
Browse files Browse the repository at this point in the history 
Signed-off-by: HangyuanLiu <[email protected]>
  • Loading branch information
@HangyuanLiu
HangyuanLiu committed Sep 3, 2024
1 parent c610121 commit 1a5848e
Show file tree
Hide file tree
Showing 124 changed files with 1,504 additions and 1,779 deletions.
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

package com.starrocks.alter;


import com.google.gson.annotations.SerializedName;
import com.starrocks.catalog.Database;
import com.starrocks.catalog.MaterializedIndex;
Expand Down Expand Up @@ -52,7 +51,7 @@ public LakeTableAlterMetaJob(long jobId, long dbId, long tableId, String tableNa

@Override
protected TabletMetadataUpdateAgentTask createTask(PhysicalPartition partition,
MaterializedIndex index, long nodeId, Set<Long> tablets) {
MaterializedIndex index, long nodeId, Set<Long> tablets) {
return TabletMetadataUpdateAgentTaskFactory.createGenericBooleanPropertyUpdateTask(nodeId, tablets,
metaValue, metaType);
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ public void updatePhysicalPartitionTabletMeta(Database db, OlapTable table, Part
locker.unLockTablesWithIntensiveDbLock(db, Lists.newArrayList(table.getId()), LockType.READ);
}
for (MaterializedIndex index : indexList) {
updateIndexTabletMeta(db, table, partition, index);
updateIndexTabletMeta(db, table, partition.getDefaultPhysicalPartition(), index);
}
}

Expand Down
9 changes: 6 additions & 3 deletions fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,8 @@ private void onTaskFinished(Database db, OlapTable targetTable, OptimizeTask rew

Set<Tablet> sourceTablets = Sets.newHashSet();
Partition partition = targetTable.getPartition(sourcePartitionName);
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
for (MaterializedIndex index
: partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
sourceTablets.addAll(index.getTablets());
}

Expand Down Expand Up @@ -548,7 +549,8 @@ private void cancelInternal() {

Partition partition = targetTable.getPartition(pid);
if (partition != null) {
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
for (MaterializedIndex index : partition.getDefaultPhysicalPartition()
.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
// hash set is able to deduplicate the elements
tmpTablets.addAll(index.getTablets());
}
Expand Down Expand Up @@ -652,7 +654,8 @@ private void onReplayFinished(OnlineOptimizeJobV2 replayedJob, OlapTable targetT
for (long id : replayedJob.getTmpPartitionIds()) {
Partition partition = targetTable.getPartition(id);
if (partition != null) {
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
for (MaterializedIndex index : partition.getDefaultPhysicalPartition()
.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
sourceTablets.addAll(index.getTablets());
}
targetTable.dropTempPartition(partition.getName(), true);
Expand Down
9 changes: 6 additions & 3 deletions fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,8 @@ private void onFinished(Database db, OlapTable targetTable) throws AlterCancelEx
Set<Tablet> sourceTablets = Sets.newHashSet();
sourcePartitionNames.forEach(name -> {
Partition partition = targetTable.getPartition(name);
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
for (MaterializedIndex index
: partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
sourceTablets.addAll(index.getTablets());
}
});
Expand Down Expand Up @@ -573,7 +574,8 @@ private void cancelInternal() {

Partition partition = targetTable.getPartition(pid);
if (partition != null) {
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
for (MaterializedIndex index : partition.getDefaultPhysicalPartition()
.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
// hash set is able to deduplicate the elements
sourceTablets.addAll(index.getTablets());
}
Expand Down Expand Up @@ -666,7 +668,8 @@ private void onReplayFinished(OptimizeJobV2 replayedJob, OlapTable targetTable)
for (long id : replayedJob.getTmpPartitionIds()) {
Partition partition = targetTable.getPartition(id);
if (partition != null) {
for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
for (MaterializedIndex index
: partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
sourceTablets.addAll(index.getTablets());
}
targetTable.dropTempPartition(partition.getName(), true);
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -848,7 +848,8 @@ private void onFinished(OlapTable tbl) {
}
// replace the origin index with shadow index, set index state as NORMAL
for (Partition partition : tbl.getPartitions()) {
TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getParentId()).getStorageMedium();
TStorageMedium medium = tbl.getPartitionInfo()
.getDataProperty(partition.getDefaultPhysicalPartition().getParentId()).getStorageMedium();
// drop the origin index from partitions
for (Map.Entry<Long, Long> entry : indexIdMap.entrySet()) {
long shadowIdxId = entry.getKey();
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,9 +314,10 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db
BackupPartitionInfo partitionInfo = new BackupPartitionInfo();
partitionInfo.id = partition.getId();
partitionInfo.name = partition.getName();
partitionInfo.version = partition.getVisibleVersion();
partitionInfo.version = partition.getDefaultPhysicalPartition().getVisibleVersion();
if (partition.getSubPartitions().size() == 1) {
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
for (MaterializedIndex index : partition.getDefaultPhysicalPartition()
.getMaterializedIndices(IndexExtState.VISIBLE)) {
BackupIndexInfo idxInfo = new BackupIndexInfo();
idxInfo.id = index.getId();
idxInfo.name = olapTbl.getIndexNameById(index.getId());
Expand Down
17 changes: 10 additions & 7 deletions fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -988,20 +988,20 @@ public Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteTb

// generate new partition id
long newPartId = globalStateMgr.getNextId();
remotePart.setIdForRestore(newPartId);
remotePart.getDefaultPhysicalPartition().setIdForRestore(newPartId);

// indexes
Map<String, Long> localIdxNameToId = localTbl.getIndexNameToId();
for (String localIdxName : localIdxNameToId.keySet()) {
// set ids of indexes in remote partition to the local index ids
long remoteIdxId = remoteTbl.getIndexIdByName(localIdxName);
MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId);
MaterializedIndex remoteIdx = remotePart.getDefaultPhysicalPartition().getIndex(remoteIdxId);
long localIdxId = localIdxNameToId.get(localIdxName);
remoteIdx.setIdForRestore(localIdxId);
if (localIdxId != localTbl.getBaseIndexId()) {
// not base table, reset
remotePart.deleteRollupIndex(remoteIdxId);
remotePart.createRollupIndex(remoteIdx);
remotePart.getDefaultPhysicalPartition().deleteRollupIndex(remoteIdxId);
remotePart.getDefaultPhysicalPartition().createRollupIndex(remoteIdx);
}
}

Expand Down Expand Up @@ -1046,7 +1046,8 @@ protected void genFileMapping(OlapTable localTbl, Partition localPartition, Long

protected void genFileMappingWithPartition(OlapTable localTbl, Partition localPartition, Long remoteTblId,
BackupPartitionInfo backupPartInfo, boolean overwrite) {
for (MaterializedIndex localIdx : localPartition.getMaterializedIndices(IndexExtState.VISIBLE)) {
for (MaterializedIndex localIdx : localPartition.getDefaultPhysicalPartition()
.getMaterializedIndices(IndexExtState.VISIBLE)) {
BackupIndexInfo backupIdxInfo = backupPartInfo.getIdx(localTbl.getIndexNameById(localIdx.getId()));
Preconditions.checkState(backupIdxInfo.tablets.size() == localIdx.getTablets().size());
for (int i = 0; i < localIdx.getTablets().size(); i++) {
Expand Down Expand Up @@ -1150,7 +1151,8 @@ protected void addRestoredPartitions(Database db, boolean modify) {
}

protected void modifyInvertedIndex(OlapTable restoreTbl, Partition restorePart) {
for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
for (MaterializedIndex restoreIdx : restorePart.getDefaultPhysicalPartition()
.getMaterializedIndices(IndexExtState.VISIBLE)) {
int schemaHash = restoreTbl.getSchemaHashByIndexId(restoreIdx.getId());
TabletMeta tabletMeta = new TabletMeta(dbId, restoreTbl.getId(), restorePart.getId(),
restoreIdx.getId(), schemaHash, TStorageMedium.HDD);
Expand Down Expand Up @@ -1663,7 +1665,8 @@ public void cancelInternal(boolean isReplay) {
for (Table restoreTbl : restoredTbls) {
LOG.info("remove restored table when cancelled: {}", restoreTbl.getName());
for (Partition part : restoreTbl.getPartitions()) {
for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) {
for (MaterializedIndex idx : part.getDefaultPhysicalPartition()
.getMaterializedIndices(IndexExtState.VISIBLE)) {
for (Tablet tablet : idx.getTablets()) {
globalStateMgr.getTabletInvertedIndex().deleteTablet(tablet.getId());
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ public static int calAvgBucketNumOfRecentPartitions(OlapTable olapTable, int rec
List<Partition> partitions = (List<Partition>) olapTable.getRecentPartitions(recentPartitionNum);
boolean dataImported = true;
for (Partition partition : partitions) {
if (partition.getVisibleVersion() == 1) {
if (partition.getDefaultPhysicalPartition().getVisibleVersion() == 1) {
dataImported = false;
break;
}
Expand Down
19 changes: 11 additions & 8 deletions fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -471,12 +471,15 @@ private void updateMetaInternal(String dbName, TTableMeta meta, List<TBackendMet
}

for (TPartitionMeta partitionMeta : meta.getPartitions()) {
Partition partition = new Partition(partitionMeta.getPartition_id(),
Partition logicalPartition = new Partition(partitionMeta.getPartition_id(),
partitionMeta.getPartition_name(),
null, // TODO(wulei): fix it
defaultDistributionInfo);
partition.setNextVersion(partitionMeta.getNext_version());
partition.updateVisibleVersion(partitionMeta.getVisible_version(),

PhysicalPartition physicalPartition = logicalPartition.getDefaultPhysicalPartition();

physicalPartition.setNextVersion(partitionMeta.getNext_version());
physicalPartition.updateVisibleVersion(partitionMeta.getVisible_version(),
partitionMeta.getVisible_time());
for (TIndexMeta indexMeta : meta.getIndexes()) {
MaterializedIndex index = new MaterializedIndex(indexMeta.getIndex_id(),
Expand All @@ -503,18 +506,18 @@ private void updateMetaInternal(String dbName, TTableMeta meta, List<TBackendMet
tTabletMeta.getOld_schema_hash(), tTabletMeta.getStorage_medium());
index.addTablet(tablet, tabletMeta, false);
}
if (indexMeta.getPartition_id() == partition.getId()) {
if (indexMeta.getPartition_id() == physicalPartition.getId()) {
if (index.getId() != baseIndexId) {
partition.createRollupIndex(index);
physicalPartition.createRollupIndex(index);
} else {
partition.setBaseIndex(index);
physicalPartition.setBaseIndex(index);
}
}
}
if (partitionMeta.isSetIs_temp() && partitionMeta.isIs_temp()) {
addTempPartition(partition);
addTempPartition(logicalPartition);
} else {
addPartition(partition);
addPartition(logicalPartition);
}
}
long endOfTabletMetaBuild = System.currentTimeMillis();
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ public static BasePartitionInfo fromExternalTable(com.starrocks.connector.Partit
}

public static BasePartitionInfo fromOlapTable(Partition partition) {
return new BasePartitionInfo(partition.getId(), partition.getVisibleVersion(), -1);
return new BasePartitionInfo(partition.getId(), partition.getDefaultPhysicalPartition().getVisibleVersion(), -1);
}

public long getId() {
Expand Down
Loading

0 comments on commit 1a5848e

Please sign in to comment.