diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJob.java b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJob.java index 9262a9281a4c4..44262e4508a5c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJob.java @@ -14,7 +14,6 @@ package com.starrocks.alter; - import com.google.gson.annotations.SerializedName; import com.starrocks.catalog.Database; import com.starrocks.catalog.MaterializedIndex; @@ -52,7 +51,7 @@ public LakeTableAlterMetaJob(long jobId, long dbId, long tableId, String tableNa @Override protected TabletMetadataUpdateAgentTask createTask(PhysicalPartition partition, - MaterializedIndex index, long nodeId, Set tablets) { + MaterializedIndex index, long nodeId, Set tablets) { return TabletMetadataUpdateAgentTaskFactory.createGenericBooleanPropertyUpdateTask(nodeId, tablets, metaValue, metaType); } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java index 4c0effe5a1a5c..05e0f4381de7b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java @@ -305,7 +305,7 @@ public void updatePhysicalPartitionTabletMeta(Database db, OlapTable table, Part locker.unLockTablesWithIntensiveDbLock(db, Lists.newArrayList(table.getId()), LockType.READ); } for (MaterializedIndex index : indexList) { - updateIndexTabletMeta(db, table, partition, index); + updateIndexTabletMeta(db, table, partition.getDefaultPhysicalPartition(), index); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java index 696faf87385ee..1282bc7031146 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java @@ -446,7 +446,8 @@ private void onTaskFinished(Database db, OlapTable targetTable, OptimizeTask rew Set sourceTablets = Sets.newHashSet(); Partition partition = targetTable.getPartition(sourcePartitionName); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index + : partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { sourceTablets.addAll(index.getTablets()); } @@ -548,7 +549,8 @@ private void cancelInternal() { Partition partition = targetTable.getPartition(pid); if (partition != null) { - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { // hash set is able to deduplicate the elements tmpTablets.addAll(index.getTablets()); } @@ -652,7 +654,8 @@ private void onReplayFinished(OnlineOptimizeJobV2 replayedJob, OlapTable targetT for (long id : replayedJob.getTmpPartitionIds()) { Partition partition = targetTable.getPartition(id); if (partition != null) { - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { sourceTablets.addAll(index.getTablets()); } targetTable.dropTempPartition(partition.getName(), true); diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java index 229fa73f0c23c..5a1559decbf88 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java @@ -473,7 +473,8 @@ private void onFinished(Database db, OlapTable targetTable) throws AlterCancelEx Set sourceTablets = Sets.newHashSet(); sourcePartitionNames.forEach(name -> { Partition partition = targetTable.getPartition(name); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index + : partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { sourceTablets.addAll(index.getTablets()); } }); @@ -573,7 +574,8 @@ private void cancelInternal() { Partition partition = targetTable.getPartition(pid); if (partition != null) { - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { // hash set is able to deduplicate the elements sourceTablets.addAll(index.getTablets()); } @@ -666,7 +668,8 @@ private void onReplayFinished(OptimizeJobV2 replayedJob, OlapTable targetTable) for (long id : replayedJob.getTmpPartitionIds()) { Partition partition = targetTable.getPartition(id); if (partition != null) { - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index + : partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { sourceTablets.addAll(index.getTablets()); } targetTable.dropTempPartition(partition.getName(), true); diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java index cb14e7e8ae80d..33e34a377e53e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java @@ -848,7 +848,8 @@ private void onFinished(OlapTable tbl) { } // replace the origin index with shadow index, set index state as NORMAL for (Partition partition : tbl.getPartitions()) { - TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getParentId()).getStorageMedium(); + TStorageMedium medium = tbl.getPartitionInfo() + .getDataProperty(partition.getDefaultPhysicalPartition().getParentId()).getStorageMedium(); // drop the origin index from partitions for (Map.Entry entry : indexIdMap.entrySet()) { long shadowIdxId = entry.getKey(); diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java index 431b8603a63b1..3507d800f9aa8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java @@ -314,9 +314,10 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db BackupPartitionInfo partitionInfo = new BackupPartitionInfo(); partitionInfo.id = partition.getId(); partitionInfo.name = partition.getName(); - partitionInfo.version = partition.getVisibleVersion(); + partitionInfo.version = partition.getDefaultPhysicalPartition().getVisibleVersion(); if (partition.getSubPartitions().size() == 1) { - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { BackupIndexInfo idxInfo = new BackupIndexInfo(); idxInfo.id = index.getId(); idxInfo.name = olapTbl.getIndexNameById(index.getId()); diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java index fd092d2c12d13..f75954c0be962 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java @@ -988,20 +988,20 @@ public Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteTb // generate new partition id long newPartId = globalStateMgr.getNextId(); - remotePart.setIdForRestore(newPartId); + remotePart.getDefaultPhysicalPartition().setIdForRestore(newPartId); // indexes Map localIdxNameToId = localTbl.getIndexNameToId(); for (String localIdxName : localIdxNameToId.keySet()) { // set ids of indexes in remote partition to the local index ids long remoteIdxId = remoteTbl.getIndexIdByName(localIdxName); - MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId); + MaterializedIndex remoteIdx = remotePart.getDefaultPhysicalPartition().getIndex(remoteIdxId); long localIdxId = localIdxNameToId.get(localIdxName); remoteIdx.setIdForRestore(localIdxId); if (localIdxId != localTbl.getBaseIndexId()) { // not base table, reset - remotePart.deleteRollupIndex(remoteIdxId); - remotePart.createRollupIndex(remoteIdx); + remotePart.getDefaultPhysicalPartition().deleteRollupIndex(remoteIdxId); + remotePart.getDefaultPhysicalPartition().createRollupIndex(remoteIdx); } } @@ -1046,7 +1046,8 @@ protected void genFileMapping(OlapTable localTbl, Partition localPartition, Long protected void genFileMappingWithPartition(OlapTable localTbl, Partition localPartition, Long remoteTblId, BackupPartitionInfo backupPartInfo, boolean overwrite) { - for (MaterializedIndex localIdx : localPartition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex localIdx : localPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { BackupIndexInfo backupIdxInfo = backupPartInfo.getIdx(localTbl.getIndexNameById(localIdx.getId())); Preconditions.checkState(backupIdxInfo.tablets.size() == localIdx.getTablets().size()); for (int i = 0; i < localIdx.getTablets().size(); i++) { @@ -1150,7 +1151,8 @@ protected void addRestoredPartitions(Database db, boolean modify) { } protected void modifyInvertedIndex(OlapTable restoreTbl, Partition restorePart) { - for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex restoreIdx : restorePart.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { int schemaHash = restoreTbl.getSchemaHashByIndexId(restoreIdx.getId()); TabletMeta tabletMeta = new TabletMeta(dbId, restoreTbl.getId(), restorePart.getId(), restoreIdx.getId(), schemaHash, TStorageMedium.HDD); @@ -1663,7 +1665,8 @@ public void cancelInternal(boolean isReplay) { for (Table restoreTbl : restoredTbls) { LOG.info("remove restored table when cancelled: {}", restoreTbl.getName()); for (Partition part : restoreTbl.getPartitions()) { - for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex idx : part.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { globalStateMgr.getTabletInvertedIndex().deleteTablet(tablet.getId()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java index 14d5194d64e45..5119de1433d08 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java @@ -424,7 +424,7 @@ public static int calAvgBucketNumOfRecentPartitions(OlapTable olapTable, int rec List partitions = (List) olapTable.getRecentPartitions(recentPartitionNum); boolean dataImported = true; for (Partition partition : partitions) { - if (partition.getVisibleVersion() == 1) { + if (partition.getDefaultPhysicalPartition().getVisibleVersion() == 1) { dataImported = false; break; } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java index dbae6823389d4..e9fc4c07725fd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java @@ -471,12 +471,15 @@ private void updateMetaInternal(String dbName, TTableMeta meta, List schema, in public boolean hasMaterializedView() { Optional partition = idToPartition.values().stream().findFirst(); - if (!partition.isPresent()) { + if (partition.isEmpty()) { return false; } else { - return partition.get().hasMaterializedView(); + PhysicalPartition physicalPartition = partition.get().getDefaultPhysicalPartition(); + return physicalPartition.hasMaterializedView(); } } @@ -835,7 +836,7 @@ public Status resetIdsForRestore(GlobalStateMgr globalStateMgr, Database db, int }); idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); Partition partition = idToPartition.get(newPartId); - partition.setIdForRestore(newPartId); + partition.getDefaultPhysicalPartition().setIdForRestore(newPartId); partition.getSubPartitions().forEach(physicalPartition -> { if (physicalPartition.getId() != newPartId) { partition.removeSubPartition(physicalPartition.getId()); @@ -861,7 +862,7 @@ public Status resetIdsForRestore(GlobalStateMgr globalStateMgr, Database db, int }); idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); Partition partition = idToPartition.get(newPartId); - partition.setIdForRestore(newPartId); + partition.getDefaultPhysicalPartition().setIdForRestore(newPartId); partition.getSubPartitions().forEach(physicalPartition -> { if (physicalPartition.getId() != newPartId) { partition.removeSubPartition(physicalPartition.getId()); @@ -933,7 +934,7 @@ public Status createTabletsForRestore(int tabletNum, MaterializedIndex index, Gl ColocateTableIndex.GroupId groupId = colocateTableIndex.getGroup(this.id); List> backendsPerBucketSeq = colocateTableIndex.getBackendsPerBucketSeq(groupId); boolean chooseBackendsArbitrary = backendsPerBucketSeq == null || backendsPerBucketSeq.isEmpty(); - + if (chooseBackendsArbitrary) { backendsPerBucketSeq = Lists.newArrayList(); } @@ -942,7 +943,7 @@ public Status createTabletsForRestore(int tabletNum, MaterializedIndex index, Gl long newTabletId = globalStateMgr.getNextId(); LocalTablet newTablet = new LocalTablet(newTabletId); index.addTablet(newTablet, null /* tablet meta */, false/* update inverted index */); - + // replicas List beIds; if (chooseBackendsArbitrary) { @@ -980,7 +981,7 @@ public Status createTabletsForRestore(int tabletNum, MaterializedIndex index, Gl long newTabletId = globalStateMgr.getNextId(); LocalTablet newTablet = new LocalTablet(newTabletId); index.addTablet(newTablet, null /* tablet meta */, false/* update inverted index */); - + // replicas List beIds = GlobalStateMgr.getCurrentState().getNodeMgr() .getClusterInfo().getNodeSelector().seqChooseBackendIds(replicationNum, true, true, getLocation()); @@ -1599,7 +1600,8 @@ public Collection getRecentPartitions(int recentPartitionNum) { Collections.sort(partitions, new Comparator() { @Override public int compare(Partition h1, Partition h2) { - return (int) (h2.getVisibleVersion() - h1.getVisibleVersion()); + return (int) (h2.getDefaultPhysicalPartition().getVisibleVersion() + - h1.getDefaultPhysicalPartition().getVisibleVersion()); } }); return partitions.subList(0, recentPartitionNum); @@ -1811,7 +1813,9 @@ public TTableDescriptor toThrift(List partitions) { public long getRowCount() { long rowCount = 0; for (Map.Entry entry : idToPartition.entrySet()) { - rowCount += entry.getValue().getBaseIndex().getRowCount(); + for (PhysicalPartition partition : entry.getValue().getSubPartitions()) { + rowCount += partition.getBaseIndex().getRowCount(); + } } return rowCount; } @@ -2227,7 +2231,7 @@ public List> getArbitraryTabletBucketsSeq() throws DdlException { List> backendsPerBucketSeq = Lists.newArrayList(); Optional optionalPartition = idToPartition.values().stream().findFirst(); if (optionalPartition.isPresent()) { - Partition partition = optionalPartition.get(); + PhysicalPartition partition = optionalPartition.get().getDefaultPhysicalPartition(); short replicationNum = partitionInfo.getReplicationNum(partition.getId()); MaterializedIndex baseIdx = partition.getBaseIndex(); for (Long tabletId : baseIdx.getTabletIdsInOrder()) { @@ -2849,7 +2853,7 @@ public int getCompressionLevel() { if (tableProperty == null) { return -1; } - return tableProperty.getCompressionLevel(); + return tableProperty.getCompressionLevel(); } public void setPartitionLiveNumber(int number) { @@ -2865,7 +2869,7 @@ public Map buildBinlogAvailableVersion() { Collection partitions = getPartitions(); for (Partition partition : partitions) { result.put(TableProperty.BINLOG_PARTITION + partition.getId(), - String.valueOf(partition.getVisibleVersion())); + String.valueOf(partition.getDefaultPhysicalPartition().getVisibleVersion())); } return result; } @@ -3107,7 +3111,8 @@ public void removeTabletsFromInvertedIndex() { TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); Collection allPartitions = getAllPartitions(); for (Partition partition : allPartitions) { - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java index b868ccdb3caeb..7be99b8a6c80b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java @@ -35,14 +35,10 @@ package com.starrocks.catalog; import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; -import com.starrocks.catalog.MaterializedIndex.IndexExtState; -import com.starrocks.catalog.MaterializedIndex.IndexState; -import com.starrocks.common.FeConstants; import com.starrocks.persist.gson.GsonPostProcessable; +import com.starrocks.persist.gson.GsonUtils; import com.starrocks.server.GlobalStateMgr; import com.starrocks.transaction.TransactionType; import org.apache.logging.log4j.LogManager; @@ -50,15 +46,13 @@ import java.io.IOException; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; /** * Internal representation of partition-related metadata. */ -public class Partition extends MetaObject implements PhysicalPartition, GsonPostProcessable { +public class Partition extends MetaObject implements GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(Partition.class); public static final long PARTITION_INIT_VERSION = 1L; @@ -74,82 +68,28 @@ public enum PartitionState { @SerializedName(value = "id") private long id; - private long beforeRestoreId; - @SerializedName(value = "name") private String name; + @SerializedName(value = "state") private PartitionState state; + + @SerializedName(value = "dpid") + private long defaultPhysicalPartitionId; + @SerializedName(value = "idToSubPartition") - private Map idToSubPartition = Maps.newHashMap(); - private Map nameToSubPartition = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + private Map idToSubPartition = Maps.newHashMap(); + private Map nameToSubPartition = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); @SerializedName(value = "distributionInfo") private DistributionInfo distributionInfo; - @SerializedName(value = "shardGroupId") - private long shardGroupId; - - /* Physical Partition Member */ - @SerializedName(value = "isImmutable") - private AtomicBoolean isImmutable = new AtomicBoolean(false); - - @SerializedName(value = "baseIndex") - private MaterializedIndex baseIndex; - /** - * Visible rollup indexes are indexes which are visible to user. - * User can do query on them, show them in related 'show' stmt. - */ - @SerializedName(value = "idToVisibleRollupIndex") - private Map idToVisibleRollupIndex = Maps.newHashMap(); - /** - * Shadow indexes are indexes which are not visible to user. - * Query will not run on these shadow indexes, and user can not see them neither. - * But load process will load data into these shadow indexes. - */ - @SerializedName(value = "idToShadowIndex") - private Map idToShadowIndex = Maps.newHashMap(); - - /** - * committed version(hash): after txn is committed, set committed version(hash) - * visible version(hash): after txn is published, set visible version - * next version(hash): next version is set after finished committing, it should equal to committed version + 1 - */ - - // not have committedVersion because committedVersion = nextVersion - 1 - @SerializedName(value = "visibleVersion") - private volatile long visibleVersion; - @SerializedName(value = "visibleVersionTime") - private volatile long visibleVersionTime; - @SerializedName(value = "nextVersion") - private volatile long nextVersion; - - /* - * in shared-nothing mode, data version is always equals to visible version - * in shared-data mode, compactions increase visible version but not data version - */ - @SerializedName(value = "dataVersion") - private volatile long dataVersion; - @SerializedName(value = "nextDataVersion") - private volatile long nextDataVersion; - - /* - * if the visible version and version epoch are unchanged, the data is unchanged - */ - @SerializedName(value = "versionEpoch") - private volatile long versionEpoch; - @SerializedName(value = "versionTxnType") - private volatile TransactionType versionTxnType; - - /** - * ID of the transaction that has committed current visible version. - * Just for tracing the txn log, no need to persist. - */ - private volatile long visibleTxnId = -1; - - private volatile long lastVacuumTime = 0; - - private volatile long minRetainVersion = 0; + public Partition(long id, String name, DistributionInfo distributionInfo) { + this.id = id; + this.name = name; + this.state = PartitionState.NORMAL; + this.distributionInfo = distributionInfo; + } private Partition() { } @@ -157,6 +97,12 @@ private Partition() { public Partition(long id, String name, MaterializedIndex baseIndex, DistributionInfo distributionInfo) { + this(id, name, baseIndex, distributionInfo, 0); + } + + public Partition(long id, String name, + MaterializedIndex baseIndex, + DistributionInfo distributionInfo, long shardGroupId) { this.id = id; this.name = name; this.state = PartitionState.NORMAL; @@ -171,15 +117,12 @@ public Partition(long id, String name, this.nextDataVersion = this.nextVersion; this.versionEpoch = this.nextVersionEpoch(); this.versionTxnType = TransactionType.TXN_NORMAL; - this.distributionInfo = distributionInfo; - } - - public Partition(long id, String name, - MaterializedIndex baseIndex, - DistributionInfo distributionInfo, long shardGroupId) { - this(id, name, baseIndex, distributionInfo); this.shardGroupId = shardGroupId; + + this.defaultPhysicalPartitionId = id; + PhysicalPartition physicalPartition = new PhysicalPartition(id, name, id, this.shardGroupId, baseIndex); + this.idToSubPartition.put(id, physicalPartition); } public Partition shallowCopy() { @@ -204,74 +147,10 @@ public Partition shallowCopy() { return partition; } - @Override - public void setIdForRestore(long id) { - this.beforeRestoreId = this.id; - this.id = id; - } - public long getId() { return this.id; } - @Override - public long getBeforeRestoreId() { - return beforeRestoreId; - } - - @Override - public void setImmutable(boolean isImmutable) { - this.isImmutable.set(isImmutable); - } - - @Override - public boolean isImmutable() { - return this.isImmutable.get(); - } - - public void addSubPartition(PhysicalPartition subPartition) { - if (subPartition instanceof PhysicalPartitionImpl) { - if (subPartition.getName() == null) { - subPartition.setName(generatePhysicalPartitionName(subPartition.getId())); - } - idToSubPartition.put(subPartition.getId(), (PhysicalPartitionImpl) subPartition); - nameToSubPartition.put(subPartition.getName(), (PhysicalPartitionImpl) subPartition); - } - } - - public void removeSubPartition(long id) { - PhysicalPartitionImpl subPartition = idToSubPartition.remove(id); - if (subPartition != null) { - nameToSubPartition.remove(subPartition.getName()); - } - } - - public Collection getSubPartitions() { - List subPartitions = idToSubPartition.values().stream().collect(Collectors.toList()); - subPartitions.add(this); - return subPartitions; - } - - public PhysicalPartition getSubPartition(long id) { - return this.id == id ? this : idToSubPartition.get(id); - } - - public PhysicalPartition getSubPartition(String name) { - return this.name.equals(name) ? this : nameToSubPartition.get(name); - } - - public long getParentId() { - return this.id; - } - - public void setParentId(long parentId) { - return; - } - - public long getShardGroupId() { - return this.shardGroupId; - } - public void setName(String newName) { this.name = newName; } @@ -284,58 +163,6 @@ public void setState(PartitionState state) { this.state = state; } - /* - * If a partition is overwritten by a restore job, we need to reset all version info to - * the restored partition version info) - */ - public void updateVersionForRestore(long visibleVersion) { - this.setVisibleVersion(visibleVersion); - this.nextVersion = this.visibleVersion + 1; - LOG.info("update partition {} version for restore: visible: {}, next: {}", - name, visibleVersion, nextVersion); - } - - public void updateVisibleVersion(long visibleVersion) { - updateVisibleVersion(visibleVersion, System.currentTimeMillis()); - } - - public void updateVisibleVersion(long visibleVersion, long visibleVersionTime) { - this.setVisibleVersion(visibleVersion, visibleVersionTime); - } - - public void updateVisibleVersion(long visibleVersion, long visibleVersionTime, long visibleTxnId) { - setVisibleVersion(visibleVersion, visibleVersionTime, visibleTxnId); - } - - public long getVisibleVersion() { - return visibleVersion; - } - - public long getVisibleVersionTime() { - return visibleVersionTime; - } - - // The method updateVisibleVersion is called when fe restart, the visibleVersionTime is updated - private void setVisibleVersion(long visibleVersion) { - this.visibleVersion = visibleVersion; - this.visibleVersionTime = System.currentTimeMillis(); - } - - public void setVisibleVersion(long visibleVersion, long visibleVersionTime) { - this.visibleVersion = visibleVersion; - this.visibleVersionTime = visibleVersionTime; - } - - public void setVisibleVersion(long visibleVersion, long visibleVersionTime, long visibleTxnId) { - this.visibleVersion = visibleVersion; - this.visibleVersionTime = visibleVersionTime; - this.visibleTxnId = visibleTxnId; - } - - public long getVisibleTxnId() { - return visibleTxnId; - } - public PartitionState getState() { return this.state; } @@ -348,129 +175,50 @@ public void setDistributionInfo(DistributionInfo distributionInfo) { this.distributionInfo = distributionInfo; } - public void createRollupIndex(MaterializedIndex mIndex) { - if (mIndex.getState().isVisible()) { - this.idToVisibleRollupIndex.put(mIndex.getId(), mIndex); - } else { - this.idToShadowIndex.put(mIndex.getId(), mIndex); - } + public String generatePhysicalPartitionName(long physicalParitionId) { + return this.name + '_' + physicalParitionId; } - public MaterializedIndex deleteRollupIndex(long indexId) { - if (this.idToVisibleRollupIndex.containsKey(indexId)) { - return idToVisibleRollupIndex.remove(indexId); - } else { - return idToShadowIndex.remove(indexId); + public void addSubPartition(PhysicalPartition subPartition) { + if (defaultPhysicalPartitionId == 0) { + defaultPhysicalPartitionId = subPartition.getId(); } + if (subPartition.getName() == null) { + subPartition.setName(generatePhysicalPartitionName(subPartition.getId())); + } + idToSubPartition.put(subPartition.getId(), subPartition); + nameToSubPartition.put(subPartition.getName(), subPartition); } - public void setBaseIndex(MaterializedIndex baseIndex) { - this.baseIndex = baseIndex; - } - - public MaterializedIndex getBaseIndex() { - return baseIndex; - } - - public long getNextVersion() { - return nextVersion; - } - - public void setNextVersion(long nextVersion) { - this.nextVersion = nextVersion; - } - - public long getCommittedVersion() { - return this.nextVersion - 1; - } - - public long getDataVersion() { - return dataVersion; - } - - public void setDataVersion(long dataVersion) { - this.dataVersion = dataVersion; - } - - public long getNextDataVersion() { - return nextDataVersion; - } - - public void setNextDataVersion(long nextDataVersion) { - this.nextDataVersion = nextDataVersion; - } - - public long getCommittedDataVersion() { - return this.nextDataVersion - 1; - } - - public long getVersionEpoch() { - return versionEpoch; - } - - public void setVersionEpoch(long versionEpoch) { - this.versionEpoch = versionEpoch; - } - - public long nextVersionEpoch() { - return GlobalStateMgr.getCurrentState().getGtidGenerator().nextGtid(); - } - - public TransactionType getVersionTxnType() { - return versionTxnType; + public void removeSubPartition(long id) { + PhysicalPartition subPartition = idToSubPartition.remove(id); + if (subPartition != null) { + nameToSubPartition.remove(subPartition.getName()); + } } - public void setVersionTxnType(TransactionType versionTxnType) { - this.versionTxnType = versionTxnType; + public Collection getSubPartitions() { + return idToSubPartition.values(); } - public MaterializedIndex getIndex(long indexId) { - if (baseIndex.getId() == indexId) { - return baseIndex; - } - if (idToVisibleRollupIndex.containsKey(indexId)) { - return idToVisibleRollupIndex.get(indexId); - } else { - return idToShadowIndex.get(indexId); - } + public PhysicalPartition getSubPartition(long id) { + return idToSubPartition.get(id); } - public List getMaterializedIndices(IndexExtState extState) { - int expectedSize = 1 + idToVisibleRollupIndex.size() + idToShadowIndex.size(); - List indices = Lists.newArrayListWithExpectedSize(expectedSize); - switch (extState) { - case ALL: - indices.add(baseIndex); - indices.addAll(idToVisibleRollupIndex.values()); - indices.addAll(idToShadowIndex.values()); - break; - case VISIBLE: - indices.add(baseIndex); - indices.addAll(idToVisibleRollupIndex.values()); - break; - case SHADOW: - indices.addAll(idToShadowIndex.values()); - default: - break; - } - return indices; + public PhysicalPartition getSubPartition(String name) { + return nameToSubPartition.get(name); } - @Override - public long getTabletMaxDataSize() { - long maxDataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - maxDataSize = Math.max(maxDataSize, mIndex.getTabletMaxDataSize()); - } - return maxDataSize; + public PhysicalPartition getDefaultPhysicalPartition() { + return idToSubPartition.get(defaultPhysicalPartitionId); } - public long storageDataSize() { - long dataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - dataSize += mIndex.getDataSize(); + public boolean hasData() { + boolean hasData = false; + for (PhysicalPartition subPartition : getSubPartitions()) { + hasData |= subPartition.hasStorageData(); } - return dataSize; + return hasData; } public long getDataSize() { @@ -481,29 +229,13 @@ public long getDataSize() { return dataSize; } - public long storageRowCount() { - long rowCount = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - rowCount += mIndex.getRowCount(); - } - return rowCount; - } - public long getRowCount() { long rowCount = 0; - for (PhysicalPartition subPartition : idToSubPartition.values()) { + for (PhysicalPartition subPartition : getSubPartitions()) { rowCount += subPartition.storageRowCount(); } - rowCount += this.storageRowCount(); - return rowCount; - } - public long storageReplicaCount() { - long replicaCount = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - replicaCount += mIndex.getReplicaCount(); - } - return replicaCount; + return rowCount; } public long getReplicaCount() { @@ -514,50 +246,6 @@ public long getReplicaCount() { return replicaCount; } - public boolean hasMaterializedView() { - return !idToVisibleRollupIndex.isEmpty(); - } - - public boolean hasStorageData() { - // The fe unit test need to check the selected index id without any data. - // So if set FeConstants.runningUnitTest, we can ensure that the number of partitions is not empty, - // And the test case can continue to execute the logic of 'select best roll up' - return ((visibleVersion != PARTITION_INIT_VERSION) - || FeConstants.runningUnitTest); - } - - public boolean hasData() { - boolean hasData = false; - for (PhysicalPartition subPartition : getSubPartitions()) { - hasData |= subPartition.hasStorageData(); - } - return hasData; - } - - public boolean isFirstLoad() { - return visibleVersion == PARTITION_INIT_VERSION + 1; - } - - /* - * Change the index' state from SHADOW to NORMAL - * Also move it to idToVisibleRollupIndex if it is not the base index. - */ - public boolean visualiseShadowIndex(long shadowIndexId, boolean isBaseIndex) { - MaterializedIndex shadowIdx = idToShadowIndex.remove(shadowIndexId); - if (shadowIdx == null) { - return false; - } - Preconditions.checkState(!idToVisibleRollupIndex.containsKey(shadowIndexId), shadowIndexId); - shadowIdx.setState(IndexState.NORMAL); - if (isBaseIndex) { - baseIndex = shadowIdx; - } else { - idToVisibleRollupIndex.put(shadowIndexId, shadowIdx); - } - LOG.info("visualise the shadow index: {}", shadowIndexId); - return true; - } - @Override public int hashCode() { return Objects.hashCode(id, visibleVersion, baseIndex, distributionInfo); @@ -599,11 +287,11 @@ public String toString() { } buffer.append("visibleVersion: ").append(visibleVersion).append("; "); - buffer.append("committedVersion: ").append(getCommittedVersion()).append("; "); + buffer.append("committedVersion: ").append(this.nextVersion - 1).append("; "); buffer.append("nextVersion: ").append(nextVersion).append("; "); buffer.append("dataVersion: ").append(dataVersion).append("; "); - buffer.append("committedDataVersion: ").append(getCommittedDataVersion()).append("; "); + buffer.append("committedDataVersion: ").append(this.nextDataVersion - 1).append("; "); buffer.append("versionEpoch: ").append(versionEpoch).append("; "); buffer.append("versionTxnType: ").append(versionTxnType).append("; "); @@ -614,26 +302,6 @@ public String toString() { return buffer.toString(); } - public long getLastVacuumTime() { - return lastVacuumTime; - } - - public void setLastVacuumTime(long lastVacuumTime) { - this.lastVacuumTime = lastVacuumTime; - } - - public long getMinRetainVersion() { - return minRetainVersion; - } - - public void setMinRetainVersion(long minRetainVersion) { - this.minRetainVersion = minRetainVersion; - } - - public String generatePhysicalPartitionName(long physicalParitionId) { - return this.name + '_' + physicalParitionId; - } - @Override public void gsonPostProcess() throws IOException { if (dataVersion == 0) { @@ -649,11 +317,80 @@ public void gsonPostProcess() throws IOException { versionTxnType = TransactionType.TXN_NORMAL; } - for (PhysicalPartitionImpl subPartition : idToSubPartition.values()) { + for (PhysicalPartition subPartition : idToSubPartition.values()) { if (subPartition.getName() == null) { subPartition.setName(generatePhysicalPartitionName(subPartition.getId())); } nameToSubPartition.put(subPartition.getName(), subPartition); } + + if (defaultPhysicalPartitionId == 0) { + defaultPhysicalPartitionId = id; + String partitionJson = GsonUtils.GSON.toJson(this); + PhysicalPartition physicalPartition = GsonUtils.GSON.fromJson(partitionJson, PhysicalPartition.class); + physicalPartition.setParentId(id); + idToSubPartition.put(id, physicalPartition); + nameToSubPartition.put(name, physicalPartition); + } + } + + /**************************************PhysicalPartition **********************************************/ + + @SerializedName(value = "shardGroupId") + private long shardGroupId; + + /* Physical Partition Member */ + @SerializedName(value = "isImmutable") + private AtomicBoolean isImmutable = new AtomicBoolean(false); + + @SerializedName(value = "baseIndex") + private MaterializedIndex baseIndex; + /** + * Visible rollup indexes are indexes which are visible to user. + * User can do query on them, show them in related 'show' stmt. + */ + @SerializedName(value = "idToVisibleRollupIndex") + private Map idToVisibleRollupIndex = Maps.newHashMap(); + /** + * Shadow indexes are indexes which are not visible to user. + * Query will not run on these shadow indexes, and user can not see them neither. + * But load process will load data into these shadow indexes. + */ + @SerializedName(value = "idToShadowIndex") + private Map idToShadowIndex = Maps.newHashMap(); + + /** + * committed version(hash): after txn is committed, set committed version(hash) + * visible version(hash): after txn is published, set visible version + * next version(hash): next version is set after finished committing, it should equal to committed version + 1 + */ + + // not have committedVersion because committedVersion = nextVersion - 1 + @SerializedName(value = "visibleVersion") + private volatile long visibleVersion; + @SerializedName(value = "visibleVersionTime") + private volatile long visibleVersionTime; + @SerializedName(value = "nextVersion") + private volatile long nextVersion; + + /* + * in shared-nothing mode, data version is always equals to visible version + * in shared-data mode, compactions increase visible version but not data version + */ + @SerializedName(value = "dataVersion") + private volatile long dataVersion; + @SerializedName(value = "nextDataVersion") + private volatile long nextDataVersion; + + /* + * if the visible version and version epoch are unchanged, the data is unchanged + */ + @SerializedName(value = "versionEpoch") + private volatile long versionEpoch; + @SerializedName(value = "versionTxnType") + private volatile TransactionType versionTxnType; + + public long nextVersionEpoch() { + return GlobalStateMgr.getCurrentState().getGtidGenerator().nextGtid(); } -} +} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartition.java b/fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartition.java index 8e524d9758c62..abc56d11ba350 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartition.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartition.java @@ -14,92 +14,472 @@ package com.starrocks.catalog; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; import com.starrocks.catalog.MaterializedIndex.IndexExtState; +import com.starrocks.catalog.MaterializedIndex.IndexState; +import com.starrocks.common.FeConstants; +import com.starrocks.persist.gson.GsonPostProcessable; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.transaction.TransactionType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicBoolean; -/* - * PhysicalPartition is the interface that describes the physical storage of a partition. - * It includes version information and one or more MaterializedIndexes. - * Each MaterializedIndex contains multiple tablets. +/** + * Physical Partition implementation */ -public interface PhysicalPartition { - - // partition id which contains this physical partition - public long getParentId(); - public void setParentId(long parentId); - - // physical partition id - public long getId(); - public String getName(); - public void setName(String name); - public void setIdForRestore(long id); - public long getBeforeRestoreId(); - - public long getShardGroupId(); - - public void setImmutable(boolean isImmutable); - public boolean isImmutable(); - - // version interface - - public void updateVersionForRestore(long visibleVersion); - - public void updateVisibleVersion(long visibleVersion); - - public void updateVisibleVersion(long visibleVersion, long visibleVersionTime); - public void updateVisibleVersion(long visibleVersion, long visibleVersionTime, long visibleTxnId); - public long getVisibleVersion(); - public long getVisibleVersionTime(); - public void setVisibleVersion(long visibleVersion, long visibleVersionTime); - public long getNextVersion(); - public void setNextVersion(long nextVersion); - public long getCommittedVersion(); - public long getDataVersion(); - public void setDataVersion(long dataVersion); - public long getNextDataVersion(); - public void setNextDataVersion(long nextDataVersion); - public long getCommittedDataVersion(); - public long getVersionEpoch(); - public void setVersionEpoch(long versionEpoch); - public long nextVersionEpoch(); - public TransactionType getVersionTxnType(); - public void setVersionTxnType(TransactionType versionTxnType); - public long getVisibleTxnId(); - - // materialized index interface - - public void createRollupIndex(MaterializedIndex mIndex); - public MaterializedIndex deleteRollupIndex(long indexId); - public void setBaseIndex(MaterializedIndex baseIndex); - public MaterializedIndex getBaseIndex(); - public MaterializedIndex getIndex(long indexId); - public List getMaterializedIndices(IndexExtState extState); +public class PhysicalPartition extends MetaObject implements GsonPostProcessable { + private static final Logger LOG = LogManager.getLogger(PhysicalPartition.class); + + public static final long PARTITION_INIT_VERSION = 1L; + + @SerializedName(value = "id") + private long id; + + @SerializedName(value = "name") + private String name; + + private long beforeRestoreId; + + @SerializedName(value = "parentId") + private long parentId; + + @SerializedName(value = "shardGroupId") + private long shardGroupId; + + /* Physical Partition Member */ + @SerializedName(value = "isImmutable") + private AtomicBoolean isImmutable = new AtomicBoolean(false); + + @SerializedName(value = "baseIndex") + private MaterializedIndex baseIndex; + /** + * Visible rollup indexes are indexes which are visible to user. + * User can do query on them, show them in related 'show' stmt. + */ + @SerializedName(value = "idToVisibleRollupIndex") + private Map idToVisibleRollupIndex = Maps.newHashMap(); + /** + * Shadow indexes are indexes which are not visible to user. + * Query will not run on these shadow indexes, and user can not see them neither. + * But load process will load data into these shadow indexes. + */ + @SerializedName(value = "idToShadowIndex") + private Map idToShadowIndex = Maps.newHashMap(); + + /** + * committed version(hash): after txn is committed, set committed version(hash) + * visible version(hash): after txn is published, set visible version + * next version(hash): next version is set after finished committing, it should equal to committed version + 1 + */ + + // not have committedVersion because committedVersion = nextVersion - 1 + @SerializedName(value = "visibleVersion") + private long visibleVersion; + @SerializedName(value = "visibleVersionTime") + private long visibleVersionTime; + @SerializedName(value = "nextVersion") + private long nextVersion; + + @SerializedName(value = "dataVersion") + private long dataVersion; + @SerializedName(value = "nextDataVersion") + private long nextDataVersion; + + @SerializedName(value = "versionEpoch") + private long versionEpoch; + @SerializedName(value = "versionTxnType") + private TransactionType versionTxnType; + /** + * ID of the transaction that has committed current visible version. + * Just for tracing the txn log, no need to persist. + */ + private long visibleTxnId = -1; + + private volatile long lastVacuumTime = 0; + + private volatile long minRetainVersion = 0; + + public PhysicalPartition(long id, String name, long parentId, long sharedGroupId, MaterializedIndex baseIndex) { + this.id = id; + this.name = name; + this.parentId = parentId; + this.baseIndex = baseIndex; + this.visibleVersion = PARTITION_INIT_VERSION; + this.visibleVersionTime = System.currentTimeMillis(); + this.nextVersion = this.visibleVersion + 1; + this.dataVersion = this.visibleVersion; + this.nextDataVersion = this.nextVersion; + this.versionEpoch = this.nextVersionEpoch(); + this.versionTxnType = TransactionType.TXN_NORMAL; + this.shardGroupId = sharedGroupId; + } + + public long getId() { + return this.id; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public void setIdForRestore(long id) { + this.beforeRestoreId = this.id; + this.id = id; + } + + public long getBeforeRestoreId() { + return this.beforeRestoreId; + } + + public long getParentId() { + return this.parentId; + } + + public void setParentId(long parentId) { + this.parentId = parentId; + } + + public long getShardGroupId() { + return this.shardGroupId; + } + + public void setImmutable(boolean isImmutable) { + this.isImmutable.set(isImmutable); + } + + public boolean isImmutable() { + return this.isImmutable.get(); + } + + public long getLastVacuumTime() { + return lastVacuumTime; + } + + public void setLastVacuumTime(long lastVacuumTime) { + this.lastVacuumTime = lastVacuumTime; + } + + public long getMinRetainVersion() { + return minRetainVersion; + } + + public void setMinRetainVersion(long minRetainVersion) { + this.minRetainVersion = minRetainVersion; + } + + /* + * If a partition is overwritten by a restore job, we need to reset all version info to + * the restored partition version info) + */ + + public void updateVersionForRestore(long visibleVersion) { + this.setVisibleVersion(visibleVersion, System.currentTimeMillis()); + this.nextVersion = this.visibleVersion + 1; + LOG.info("update partition {} version for restore: visible: {}, next: {}", + id, visibleVersion, nextVersion); + } + + public void updateVisibleVersion(long visibleVersion) { + updateVisibleVersion(visibleVersion, System.currentTimeMillis()); + } + + public void updateVisibleVersion(long visibleVersion, long visibleVersionTime) { + this.setVisibleVersion(visibleVersion, visibleVersionTime); + } + + public void updateVisibleVersion(long visibleVersion, long visibleVersionTime, long visibleTxnId) { + setVisibleVersion(visibleVersion, visibleVersionTime); + this.visibleTxnId = visibleTxnId; + } + + public long getVisibleTxnId() { + return visibleTxnId; + } + + public long getVisibleVersion() { + return visibleVersion; + } + + public long getVisibleVersionTime() { + return visibleVersionTime; + } + + public void setVisibleVersion(long visibleVersion, long visibleVersionTime) { + this.visibleVersion = visibleVersion; + this.visibleVersionTime = visibleVersionTime; + } + + public void createRollupIndex(MaterializedIndex mIndex) { + if (mIndex.getState().isVisible()) { + this.idToVisibleRollupIndex.put(mIndex.getId(), mIndex); + } else { + this.idToShadowIndex.put(mIndex.getId(), mIndex); + } + } + + public MaterializedIndex deleteRollupIndex(long indexId) { + if (this.idToVisibleRollupIndex.containsKey(indexId)) { + return idToVisibleRollupIndex.remove(indexId); + } else { + return idToShadowIndex.remove(indexId); + } + } + + public void setBaseIndex(MaterializedIndex baseIndex) { + this.baseIndex = baseIndex; + } + + public MaterializedIndex getBaseIndex() { + return baseIndex; + } + + public long getNextVersion() { + return nextVersion; + } + + public void setNextVersion(long nextVersion) { + this.nextVersion = nextVersion; + } + + public long getCommittedVersion() { + return this.nextVersion - 1; + } + + public long getDataVersion() { + return dataVersion; + } + + public void setDataVersion(long dataVersion) { + this.dataVersion = dataVersion; + } + + public long getNextDataVersion() { + return nextDataVersion; + } + + public void setNextDataVersion(long nextDataVersion) { + this.nextDataVersion = nextDataVersion; + } + + public long getCommittedDataVersion() { + return this.nextDataVersion - 1; + } + + public long getVersionEpoch() { + return versionEpoch; + } + + public void setVersionEpoch(long versionEpoch) { + this.versionEpoch = versionEpoch; + } + + public long nextVersionEpoch() { + return GlobalStateMgr.getCurrentState().getGtidGenerator().nextGtid(); + } + + public TransactionType getVersionTxnType() { + return versionTxnType; + } + + public void setVersionTxnType(TransactionType versionTxnType) { + this.versionTxnType = versionTxnType; + } + + public MaterializedIndex getIndex(long indexId) { + if (baseIndex.getId() == indexId) { + return baseIndex; + } + if (idToVisibleRollupIndex.containsKey(indexId)) { + return idToVisibleRollupIndex.get(indexId); + } else { + return idToShadowIndex.get(indexId); + } + } + + public List getMaterializedIndices(IndexExtState extState) { + int expectedSize = 1 + idToVisibleRollupIndex.size() + idToShadowIndex.size(); + List indices = Lists.newArrayListWithExpectedSize(expectedSize); + switch (extState) { + case ALL: + indices.add(baseIndex); + indices.addAll(idToVisibleRollupIndex.values()); + indices.addAll(idToShadowIndex.values()); + break; + case VISIBLE: + indices.add(baseIndex); + indices.addAll(idToVisibleRollupIndex.values()); + break; + case SHADOW: + indices.addAll(idToShadowIndex.values()); + default: + break; + } + return indices; + } + + public long getTabletMaxDataSize() { + long maxDataSize = 0; + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + maxDataSize = Math.max(maxDataSize, mIndex.getTabletMaxDataSize()); + } + return maxDataSize; + } + + public long storageDataSize() { + long dataSize = 0; + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + dataSize += mIndex.getDataSize(); + } + return dataSize; + } + + public long storageRowCount() { + long rowCount = 0; + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + rowCount += mIndex.getRowCount(); + } + return rowCount; + } + + public long storageReplicaCount() { + long replicaCount = 0; + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + replicaCount += mIndex.getReplicaCount(); + } + return replicaCount; + } + + public boolean hasMaterializedView() { + return !idToVisibleRollupIndex.isEmpty(); + } + + public boolean hasStorageData() { + // The fe unit test need to check the selected index id without any data. + // So if set FeConstants.runningUnitTest, we can ensure that the number of partitions is not empty, + // And the test case can continue to execute the logic of 'select best roll up' + return ((visibleVersion != PARTITION_INIT_VERSION) + || FeConstants.runningUnitTest); + } + + public boolean isFirstLoad() { + return visibleVersion == PARTITION_INIT_VERSION + 1; + } + /* * Change the index' state from SHADOW to NORMAL + * Also move it to idToVisibleRollupIndex if it is not the base index. */ - public boolean visualiseShadowIndex(long shadowIndexId, boolean isBaseIndex); - - // statistic interface - - // max data size of one tablet in this physical partition - public long getTabletMaxDataSize(); - // partition data size reported by be, but may be not accurate - public long storageDataSize(); - // partition row count reported by be, but may be not accurate - public long storageRowCount(); - // partition replica count, it's accurate - public long storageReplicaCount(); - // has data judge by fe version, it's accurate - public boolean hasStorageData(); - public boolean hasMaterializedView(); - public boolean isFirstLoad(); - - // for lake partition - public long getMinRetainVersion(); - public void setMinRetainVersion(long minRetainVersion); - public long getLastVacuumTime(); - public void setLastVacuumTime(long lastVacuumTime); + public boolean visualiseShadowIndex(long shadowIndexId, boolean isBaseIndex) { + MaterializedIndex shadowIdx = idToShadowIndex.remove(shadowIndexId); + if (shadowIdx == null) { + return false; + } + Preconditions.checkState(!idToVisibleRollupIndex.containsKey(shadowIndexId), shadowIndexId); + shadowIdx.setState(IndexState.NORMAL); + if (isBaseIndex) { + baseIndex = shadowIdx; + } else { + idToVisibleRollupIndex.put(shadowIndexId, shadowIdx); + } + LOG.info("visualise the shadow index: {}", shadowIndexId); + return true; + } + + public int hashCode() { + return Objects.hashCode(visibleVersion, baseIndex); + } + + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof PhysicalPartition)) { + return false; + } + + PhysicalPartition partition = (PhysicalPartition) obj; + if (idToVisibleRollupIndex != partition.idToVisibleRollupIndex) { + if (idToVisibleRollupIndex.size() != partition.idToVisibleRollupIndex.size()) { + return false; + } + for (Entry entry : idToVisibleRollupIndex.entrySet()) { + long key = entry.getKey(); + if (!partition.idToVisibleRollupIndex.containsKey(key)) { + return false; + } + if (!entry.getValue().equals(partition.idToVisibleRollupIndex.get(key))) { + return false; + } + } + } + + return (visibleVersion == partition.visibleVersion) + && (baseIndex.equals(partition.baseIndex)); + } + + public String toString() { + StringBuilder buffer = new StringBuilder(); + buffer.append("partitionId: ").append(id).append("; "); + buffer.append("partitionName: ").append(name).append("; "); + buffer.append("parentPartitionId: ").append(parentId).append("; "); + buffer.append("shardGroupId: ").append(shardGroupId).append("; "); + buffer.append("isImmutable: ").append(isImmutable()).append("; "); + + buffer.append("baseIndex: ").append(baseIndex.toString()).append("; "); + + int rollupCount = (idToVisibleRollupIndex != null) ? idToVisibleRollupIndex.size() : 0; + buffer.append("rollupCount: ").append(rollupCount).append("; "); + + if (idToVisibleRollupIndex != null) { + for (Map.Entry entry : idToVisibleRollupIndex.entrySet()) { + buffer.append("rollupIndex: ").append(entry.getValue().toString()).append("; "); + } + } + + buffer.append("visibleVersion: ").append(visibleVersion).append("; "); + buffer.append("visibleVersionTime: ").append(visibleVersionTime).append("; "); + buffer.append("committedVersion: ").append(getCommittedVersion()).append("; "); + + buffer.append("dataVersion: ").append(dataVersion).append("; "); + buffer.append("committedDataVersion: ").append(getCommittedDataVersion()).append("; "); + + buffer.append("versionEpoch: ").append(versionEpoch).append("; "); + buffer.append("versionTxnType: ").append(versionTxnType).append("; "); + + buffer.append("storageDataSize: ").append(storageDataSize()).append("; "); + buffer.append("storageRowCount: ").append(storageRowCount()).append("; "); + buffer.append("storageReplicaCount: ").append(storageReplicaCount()).append("; "); + + return buffer.toString(); + } + + public void gsonPostProcess() throws IOException { + if (dataVersion == 0) { + dataVersion = visibleVersion; + } + if (nextDataVersion == 0) { + nextDataVersion = nextVersion; + } + if (versionEpoch == 0) { + versionEpoch = nextVersionEpoch(); + } + if (versionTxnType == null) { + versionTxnType = TransactionType.TXN_NORMAL; + } + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartitionImpl.java b/fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartitionImpl.java deleted file mode 100644 index 0d69444053c9f..0000000000000 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartitionImpl.java +++ /dev/null @@ -1,533 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.starrocks.catalog; - -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.gson.annotations.SerializedName; -import com.starrocks.catalog.MaterializedIndex.IndexExtState; -import com.starrocks.catalog.MaterializedIndex.IndexState; -import com.starrocks.common.FeConstants; -import com.starrocks.persist.gson.GsonPostProcessable; -import com.starrocks.server.GlobalStateMgr; -import com.starrocks.transaction.TransactionType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Physical Partition implementation - */ -public class PhysicalPartitionImpl extends MetaObject implements PhysicalPartition, GsonPostProcessable { - private static final Logger LOG = LogManager.getLogger(PhysicalPartitionImpl.class); - - public static final long PARTITION_INIT_VERSION = 1L; - - @SerializedName(value = "id") - private long id; - - @SerializedName(value = "name") - private String name; - - private long beforeRestoreId; - - @SerializedName(value = "parentId") - private long parentId; - - @SerializedName(value = "shardGroupId") - private long shardGroupId; - - /* Physical Partition Member */ - @SerializedName(value = "isImmutable") - private AtomicBoolean isImmutable = new AtomicBoolean(false); - - @SerializedName(value = "baseIndex") - private MaterializedIndex baseIndex; - /** - * Visible rollup indexes are indexes which are visible to user. - * User can do query on them, show them in related 'show' stmt. - */ - @SerializedName(value = "idToVisibleRollupIndex") - private Map idToVisibleRollupIndex = Maps.newHashMap(); - /** - * Shadow indexes are indexes which are not visible to user. - * Query will not run on these shadow indexes, and user can not see them neither. - * But load process will load data into these shadow indexes. - */ - @SerializedName(value = "idToShadowIndex") - private Map idToShadowIndex = Maps.newHashMap(); - - /** - * committed version(hash): after txn is committed, set committed version(hash) - * visible version(hash): after txn is published, set visible version - * next version(hash): next version is set after finished committing, it should equal to committed version + 1 - */ - - // not have committedVersion because committedVersion = nextVersion - 1 - @SerializedName(value = "visibleVersion") - private long visibleVersion; - @SerializedName(value = "visibleVersionTime") - private long visibleVersionTime; - @SerializedName(value = "nextVersion") - private long nextVersion; - - @SerializedName(value = "dataVersion") - private long dataVersion; - @SerializedName(value = "nextDataVersion") - private long nextDataVersion; - - @SerializedName(value = "versionEpoch") - private long versionEpoch; - @SerializedName(value = "versionTxnType") - private TransactionType versionTxnType; - /** - * ID of the transaction that has committed current visible version. - * Just for tracing the txn log, no need to persist. - */ - private long visibleTxnId = -1; - - private volatile long lastVacuumTime = 0; - - private volatile long minRetainVersion = 0; - - public PhysicalPartitionImpl(long id, String name, long parentId, long sharedGroupId, MaterializedIndex baseIndex) { - this.id = id; - this.name = name; - this.parentId = parentId; - this.baseIndex = baseIndex; - this.visibleVersion = PARTITION_INIT_VERSION; - this.visibleVersionTime = System.currentTimeMillis(); - this.nextVersion = this.visibleVersion + 1; - this.dataVersion = this.visibleVersion; - this.nextDataVersion = this.nextVersion; - this.versionEpoch = this.nextVersionEpoch(); - this.versionTxnType = TransactionType.TXN_NORMAL; - this.shardGroupId = sharedGroupId; - } - - @Override - public long getId() { - return this.id; - } - - @Override - public String getName() { - return this.name; - } - - @Override - public void setName(String name) { - this.name = name; - } - - @Override - public void setIdForRestore(long id) { - this.beforeRestoreId = this.id; - this.id = id; - } - - @Override - public long getBeforeRestoreId() { - return this.beforeRestoreId; - } - - @Override - public long getParentId() { - return this.parentId; - } - - @Override - public void setParentId(long parentId) { - this.parentId = parentId; - } - - @Override - public long getShardGroupId() { - return this.shardGroupId; - } - - @Override - public void setImmutable(boolean isImmutable) { - this.isImmutable.set(isImmutable); - } - - @Override - public boolean isImmutable() { - return this.isImmutable.get(); - } - - @Override - public long getLastVacuumTime() { - return lastVacuumTime; - } - - @Override - public void setLastVacuumTime(long lastVacuumTime) { - this.lastVacuumTime = lastVacuumTime; - } - - @Override - public long getMinRetainVersion() { - return minRetainVersion; - } - - @Override - public void setMinRetainVersion(long minRetainVersion) { - this.minRetainVersion = minRetainVersion; - } - - /* - * If a partition is overwritten by a restore job, we need to reset all version info to - * the restored partition version info) - */ - @Override - public void updateVersionForRestore(long visibleVersion) { - this.setVisibleVersion(visibleVersion, System.currentTimeMillis()); - this.nextVersion = this.visibleVersion + 1; - LOG.info("update partition {} version for restore: visible: {}, next: {}", - id, visibleVersion, nextVersion); - } - - @Override - public void updateVisibleVersion(long visibleVersion) { - updateVisibleVersion(visibleVersion, System.currentTimeMillis()); - } - - @Override - public void updateVisibleVersion(long visibleVersion, long visibleVersionTime) { - this.setVisibleVersion(visibleVersion, visibleVersionTime); - } - - @Override - public void updateVisibleVersion(long visibleVersion, long visibleVersionTime, long visibleTxnId) { - setVisibleVersion(visibleVersion, visibleVersionTime); - this.visibleTxnId = visibleTxnId; - } - - @Override - public long getVisibleTxnId() { - return visibleTxnId; - } - - @Override - public long getVisibleVersion() { - return visibleVersion; - } - - @Override - public long getVisibleVersionTime() { - return visibleVersionTime; - } - - @Override - public void setVisibleVersion(long visibleVersion, long visibleVersionTime) { - this.visibleVersion = visibleVersion; - this.visibleVersionTime = visibleVersionTime; - } - - @Override - public void createRollupIndex(MaterializedIndex mIndex) { - if (mIndex.getState().isVisible()) { - this.idToVisibleRollupIndex.put(mIndex.getId(), mIndex); - } else { - this.idToShadowIndex.put(mIndex.getId(), mIndex); - } - } - - @Override - public MaterializedIndex deleteRollupIndex(long indexId) { - if (this.idToVisibleRollupIndex.containsKey(indexId)) { - return idToVisibleRollupIndex.remove(indexId); - } else { - return idToShadowIndex.remove(indexId); - } - } - - @Override - public void setBaseIndex(MaterializedIndex baseIndex) { - this.baseIndex = baseIndex; - } - - @Override - public MaterializedIndex getBaseIndex() { - return baseIndex; - } - - @Override - public long getNextVersion() { - return nextVersion; - } - - @Override - public void setNextVersion(long nextVersion) { - this.nextVersion = nextVersion; - } - - @Override - public long getCommittedVersion() { - return this.nextVersion - 1; - } - - @Override - public long getDataVersion() { - return dataVersion; - } - - @Override - public void setDataVersion(long dataVersion) { - this.dataVersion = dataVersion; - } - - @Override - public long getNextDataVersion() { - return nextDataVersion; - } - - @Override - public void setNextDataVersion(long nextDataVersion) { - this.nextDataVersion = nextDataVersion; - } - - @Override - public long getCommittedDataVersion() { - return this.nextDataVersion - 1; - } - - @Override - public long getVersionEpoch() { - return versionEpoch; - } - - @Override - public void setVersionEpoch(long versionEpoch) { - this.versionEpoch = versionEpoch; - } - - @Override - public long nextVersionEpoch() { - return GlobalStateMgr.getCurrentState().getGtidGenerator().nextGtid(); - } - - public TransactionType getVersionTxnType() { - return versionTxnType; - } - - public void setVersionTxnType(TransactionType versionTxnType) { - this.versionTxnType = versionTxnType; - } - - @Override - public MaterializedIndex getIndex(long indexId) { - if (baseIndex.getId() == indexId) { - return baseIndex; - } - if (idToVisibleRollupIndex.containsKey(indexId)) { - return idToVisibleRollupIndex.get(indexId); - } else { - return idToShadowIndex.get(indexId); - } - } - - @Override - public List getMaterializedIndices(IndexExtState extState) { - List indices = Lists.newArrayList(); - switch (extState) { - case ALL: - indices.add(baseIndex); - indices.addAll(idToVisibleRollupIndex.values()); - indices.addAll(idToShadowIndex.values()); - break; - case VISIBLE: - indices.add(baseIndex); - indices.addAll(idToVisibleRollupIndex.values()); - break; - case SHADOW: - indices.addAll(idToShadowIndex.values()); - default: - break; - } - return indices; - } - - @Override - public long getTabletMaxDataSize() { - long maxDataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - maxDataSize = Math.max(maxDataSize, mIndex.getTabletMaxDataSize()); - } - return maxDataSize; - } - - @Override - public long storageDataSize() { - long dataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - dataSize += mIndex.getDataSize(); - } - return dataSize; - } - - @Override - public long storageRowCount() { - long rowCount = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - rowCount += mIndex.getRowCount(); - } - return rowCount; - } - - @Override - public long storageReplicaCount() { - long replicaCount = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - replicaCount += mIndex.getReplicaCount(); - } - return replicaCount; - } - - @Override - public boolean hasMaterializedView() { - return !idToVisibleRollupIndex.isEmpty(); - } - - @Override - public boolean hasStorageData() { - // The fe unit test need to check the selected index id without any data. - // So if set FeConstants.runningUnitTest, we can ensure that the number of partitions is not empty, - // And the test case can continue to execute the logic of 'select best roll up' - return ((visibleVersion != PARTITION_INIT_VERSION) - || FeConstants.runningUnitTest); - } - - @Override - public boolean isFirstLoad() { - return visibleVersion == PARTITION_INIT_VERSION + 1; - } - - /* - * Change the index' state from SHADOW to NORMAL - * Also move it to idToVisibleRollupIndex if it is not the base index. - */ - @Override - public boolean visualiseShadowIndex(long shadowIndexId, boolean isBaseIndex) { - MaterializedIndex shadowIdx = idToShadowIndex.remove(shadowIndexId); - if (shadowIdx == null) { - return false; - } - Preconditions.checkState(!idToVisibleRollupIndex.containsKey(shadowIndexId), shadowIndexId); - shadowIdx.setState(IndexState.NORMAL); - if (isBaseIndex) { - baseIndex = shadowIdx; - } else { - idToVisibleRollupIndex.put(shadowIndexId, shadowIdx); - } - LOG.info("visualise the shadow index: {}", shadowIndexId); - return true; - } - - @Override - public int hashCode() { - return Objects.hashCode(visibleVersion, baseIndex); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof PhysicalPartitionImpl)) { - return false; - } - - PhysicalPartitionImpl partition = (PhysicalPartitionImpl) obj; - if (idToVisibleRollupIndex != partition.idToVisibleRollupIndex) { - if (idToVisibleRollupIndex.size() != partition.idToVisibleRollupIndex.size()) { - return false; - } - for (Entry entry : idToVisibleRollupIndex.entrySet()) { - long key = entry.getKey(); - if (!partition.idToVisibleRollupIndex.containsKey(key)) { - return false; - } - if (!entry.getValue().equals(partition.idToVisibleRollupIndex.get(key))) { - return false; - } - } - } - - return (visibleVersion == partition.visibleVersion) - && (baseIndex.equals(partition.baseIndex)); - } - - @Override - public String toString() { - StringBuilder buffer = new StringBuilder(); - buffer.append("partitionId: ").append(id).append("; "); - buffer.append("partitionName: ").append(name).append("; "); - buffer.append("parentPartitionId: ").append(parentId).append("; "); - buffer.append("shardGroupId: ").append(shardGroupId).append("; "); - buffer.append("isImmutable: ").append(isImmutable()).append("; "); - - buffer.append("baseIndex: ").append(baseIndex.toString()).append("; "); - - int rollupCount = (idToVisibleRollupIndex != null) ? idToVisibleRollupIndex.size() : 0; - buffer.append("rollupCount: ").append(rollupCount).append("; "); - - if (idToVisibleRollupIndex != null) { - for (Map.Entry entry : idToVisibleRollupIndex.entrySet()) { - buffer.append("rollupIndex: ").append(entry.getValue().toString()).append("; "); - } - } - - buffer.append("visibleVersion: ").append(visibleVersion).append("; "); - buffer.append("visibleVersionTime: ").append(visibleVersionTime).append("; "); - buffer.append("committedVersion: ").append(getCommittedVersion()).append("; "); - - buffer.append("dataVersion: ").append(dataVersion).append("; "); - buffer.append("committedDataVersion: ").append(getCommittedDataVersion()).append("; "); - - buffer.append("versionEpoch: ").append(versionEpoch).append("; "); - buffer.append("versionTxnType: ").append(versionTxnType).append("; "); - - buffer.append("storageDataSize: ").append(storageDataSize()).append("; "); - buffer.append("storageRowCount: ").append(storageRowCount()).append("; "); - buffer.append("storageReplicaCount: ").append(storageReplicaCount()).append("; "); - - return buffer.toString(); - } - - @Override - public void gsonPostProcess() throws IOException { - if (dataVersion == 0) { - dataVersion = visibleVersion; - } - if (nextDataVersion == 0) { - nextDataVersion = nextVersion; - } - if (versionEpoch == 0) { - versionEpoch = nextVersionEpoch(); - } - if (versionTxnType == null) { - versionTxnType = TransactionType.TXN_NORMAL; - } - } -} diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TempPartitions.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TempPartitions.java index 8a3e82335eba5..ddb8f0095ff5e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TempPartitions.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TempPartitions.java @@ -85,7 +85,8 @@ public void dropPartition(String partitionName, boolean needDropTablet) { nameToPartition.remove(partitionName); if (!GlobalStateMgr.isCheckpointThread() && needDropTablet) { TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { + for (MaterializedIndex index : + partition.getDefaultPhysicalPartition().getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessArbiter.java b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessArbiter.java index 8a784674e500f..d796ad17ff323 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessArbiter.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessArbiter.java @@ -199,7 +199,7 @@ protected void addEmptyPartitionsToRefresh(MvUpdateInfo mvUpdateInfo) { return; } mv.getRangePartitionMap().keySet().forEach(mvPartitionName -> { - if (!mv.getPartition(mvPartitionName).hasStorageData()) { + if (!mv.getPartition(mvPartitionName).getDefaultPhysicalPartition().hasStorageData()) { // add empty partitions mvUpdateInfo.addMvToRefreshPartitionNames(mvPartitionName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessNonPartitionArbiter.java b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessNonPartitionArbiter.java index 3b5375b58f5e0..ce7c5e75b1fe0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessNonPartitionArbiter.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessNonPartitionArbiter.java @@ -83,7 +83,7 @@ protected MvUpdateInfo getMVTimelinessUpdateInfoInChecked() { @Override protected MvUpdateInfo getMVTimelinessUpdateInfoInLoose() { List partitions = Lists.newArrayList(mv.getPartitions()); - if (partitions.size() > 0 && partitions.get(0).getVisibleVersion() <= 1) { + if (partitions.size() > 0 && partitions.get(0).getDefaultPhysicalPartition().getVisibleVersion() <= 1) { // the mv is newly created, can not use it to rewrite query. return new MvUpdateInfo(MvUpdateInfo.MvToRefreshType.FULL); } diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java index 17405f9f91742..ade69d6c0a3d4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java @@ -789,10 +789,11 @@ private ColocateMatchResult doMatchOneGroup(GroupId groupId, continue; } - long visibleVersion = partition.getVisibleVersion(); + long visibleVersion = partition.getDefaultPhysicalPartition().getVisibleVersion(); // Here we only get VISIBLE indexes. All other indexes are not queryable. // So it does not matter if tablets of other indexes are not matched. - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : + partition.getDefaultPhysicalPartition().getMaterializedIndices(IndexExtState.VISIBLE)) { Preconditions.checkState(backendBucketsSeq.size() == index.getTablets().size(), backendBucketsSeq.size() + " v.s. " + index.getTablets().size()); int idx = 0; diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java index d895abf3d161f..972c4d14b7114 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java @@ -1552,7 +1552,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, return true; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getIndex(tabletMeta.getIndexId()); if (index == null) { return true; } @@ -1572,7 +1572,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, TabletChecker.getTabletHealthStatusWithPriority( tablet, globalStateMgr.getNodeMgr().getClusterInfo(), - partition.getVisibleVersion(), + partition.getDefaultPhysicalPartition().getVisibleVersion(), replicaNum, aliveBeIds, olapTable.getLocation()); @@ -1691,7 +1691,7 @@ private Map, PartitionStat> getPartitionStats(TStorageMedium me /* * Tablet in SHADOW index can not be repaired of balanced */ - for (MaterializedIndex idx : partition + for (MaterializedIndex idx : partition.getDefaultPhysicalPartition() .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { PartitionStat pStat = new PartitionStat(dbId, table.getId(), 0, replicaNum, replicationFactor); diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/PartitionsProcDir.java index 3004d19c82bec..abcbe12c1fc0b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/PartitionsProcDir.java @@ -439,9 +439,9 @@ public ProcNodeInterface lookup(String partitionIdOrName) throws AnalysisExcepti try { partition = table.getPhysicalPartition(Long.parseLong(partitionIdOrName)); } catch (NumberFormatException e) { - partition = table.getPartition(partitionIdOrName, false); + partition = table.getPartition(partitionIdOrName, false).getDefaultPhysicalPartition(); if (partition == null) { - partition = table.getPartition(partitionIdOrName, true); + partition = table.getPartition(partitionIdOrName, true).getDefaultPhysicalPartition(); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java b/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java index 8810246b7daf8..e51ecb07bcff2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/OlapPartitionTraits.java @@ -84,8 +84,9 @@ public Set getUpdatedPartitionNames(List baseTables, List baseTablePartitionInfos = Lists.newArrayList(); for (String p : baseTable.getVisiblePartitionNames()) { Partition partition = baseTable.getPartition(p); - baseTablePartitionInfos.add(String.format("%s:%s:%s:%s", p, partition.getId(), partition.getVisibleVersion(), - partition.getVisibleVersionTime())); + baseTablePartitionInfos.add(String.format("%s:%s:%s:%s", p, partition.getId(), + partition.getDefaultPhysicalPartition().getVisibleVersion(), + partition.getDefaultPhysicalPartition().getVisibleVersionTime())); } LOG.debug("baseTable: {}, baseTablePartitions:{}, mvBaseTableVisibleVersionMap: {}", baseTable.getName(), baseTablePartitionInfos, mvBaseTableVisibleVersionMap); @@ -96,7 +97,7 @@ public Set getUpdatedPartitionNames(List baseTables, for (String partitionName : baseTable.getVisiblePartitionNames()) { if (!mvBaseTableVisibleVersionMap.containsKey(partitionName)) { Partition partition = baseTable.getPartition(partitionName); - if (partition.getVisibleVersion() != 1) { + if (partition.getDefaultPhysicalPartition().getVisibleVersion() != 1) { result.add(partitionName); } } @@ -134,8 +135,9 @@ public Set getUpdatedPartitionNames(List baseTables, public static boolean isBaseTableChanged(Partition partition, MaterializedView.BasePartitionInfo mvRefreshedPartitionInfo) { return mvRefreshedPartitionInfo.getId() != partition.getId() - || partition.getVisibleVersion() != mvRefreshedPartitionInfo.getVersion() - || partition.getVisibleVersionTime() > mvRefreshedPartitionInfo.getLastRefreshTime(); + || partition.getDefaultPhysicalPartition().getVisibleVersion() != mvRefreshedPartitionInfo.getVersion() + || partition.getDefaultPhysicalPartition().getVisibleVersionTime() + > mvRefreshedPartitionInfo.getLastRefreshTime(); } public List getPartitionColumns() { diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java index 21bc56346f029..04cbf4e341ec1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java @@ -160,7 +160,7 @@ public boolean sendTasks() { return false; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getIndex(tabletMeta.getIndexId()); if (index == null) { LOG.debug("index[{}] does not exist", tabletMeta.getIndexId()); return false; @@ -172,7 +172,7 @@ public boolean sendTasks() { return false; } - checkedVersion = partition.getVisibleVersion(); + checkedVersion = partition.getDefaultPhysicalPartition().getVisibleVersion(); checkedSchemaHash = olapTable.getSchemaHashByIndexId(tabletMeta.getIndexId()); int sentTaskReplicaNum = 0; @@ -278,7 +278,7 @@ public synchronized int tryFinishJob() { return -1; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getIndex(tabletMeta.getIndexId()); if (index == null) { LOG.warn("index[{}] does not exist", tabletMeta.getIndexId()); return -1; diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java index 7f4f0c8067576..88b148f491990 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java @@ -45,7 +45,6 @@ import com.starrocks.catalog.OlapTable.OlapTableState; import com.starrocks.catalog.Partition; import com.starrocks.catalog.PhysicalPartition; -import com.starrocks.catalog.PhysicalPartitionImpl; import com.starrocks.catalog.Table; import com.starrocks.common.Config; import com.starrocks.common.util.FrontendDaemon; @@ -77,7 +76,7 @@ public class ConsistencyChecker extends FrontendDaemon { private static final int MAX_JOB_NUM = 100; private static final Comparator COMPARATOR = - (first, second) -> Long.signum(first.getLastCheckTime() - second.getLastCheckTime()); + (first, second) -> Long.signum(first.getLastCheckTime() - second.getLastCheckTime()); // tabletId -> job private final Map jobs; @@ -124,7 +123,7 @@ private boolean initWorkTime() { LOG.info("parsed startDate: {}, endDate: {}", startDate, endDate); } catch (ParseException e) { LOG.error("failed to parse start/end time: {}, {}", Config.consistency_check_start_time, - Config.consistency_check_end_time, e); + Config.consistency_check_end_time, e); return false; } @@ -218,7 +217,7 @@ private boolean itsTime() { if (!isTime) { LOG.debug("current time is {}:00, waiting to {}:00 to {}:00", - currentTime, startTime, endTime); + currentTime, startTime, endTime); } return isTime; @@ -319,7 +318,7 @@ protected List chooseTablets() { // sort partitions Queue partitionQueue = - new PriorityQueue<>(Math.max(table.getAllPhysicalPartitions().size(), 1), COMPARATOR); + new PriorityQueue<>(Math.max(table.getAllPhysicalPartitions().size(), 1), COMPARATOR); for (PhysicalPartition partition : table.getPhysicalPartitions()) { // check partition's replication num. if 1 replication. skip if (table.getPartitionInfo().getReplicationNum(partition.getParentId()) == (short) 1) { @@ -330,14 +329,10 @@ protected List chooseTablets() { // check if this partition has no data if (partition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION) { LOG.debug("partition[{}]'s version is {}. ignore", partition.getId(), - Partition.PARTITION_INIT_VERSION); + Partition.PARTITION_INIT_VERSION); continue; } - if (partition instanceof Partition) { - partitionQueue.add((Partition) partition); - } else if (partition instanceof PhysicalPartitionImpl) { - partitionQueue.add((PhysicalPartitionImpl) partition); - } + partitionQueue.add(partition); } while ((chosenOne = partitionQueue.poll()) != null) { @@ -345,9 +340,9 @@ protected List chooseTablets() { // sort materializedIndices List visibleIndexes = - partition.getMaterializedIndices(IndexExtState.VISIBLE); + partition.getMaterializedIndices(IndexExtState.VISIBLE); Queue indexQueue = - new PriorityQueue<>(Math.max(visibleIndexes.size(), 1), COMPARATOR); + new PriorityQueue<>(Math.max(visibleIndexes.size(), 1), COMPARATOR); indexQueue.addAll(visibleIndexes); while ((chosenOne = indexQueue.poll()) != null) { @@ -355,7 +350,7 @@ protected List chooseTablets() { // sort tablets Queue tabletQueue = - new PriorityQueue<>(Math.max(index.getTablets().size(), 1), COMPARATOR); + new PriorityQueue<>(Math.max(index.getTablets().size(), 1), COMPARATOR); tabletQueue.addAll(index.getTablets()); while ((chosenOne = tabletQueue.poll()) != null) { @@ -370,12 +365,12 @@ protected List chooseTablets() { if (partition.getVisibleVersion() == tablet.getCheckedVersion()) { if (tablet.isConsistent()) { LOG.debug("tablet[{}]'s version[{}-{}] has been checked. ignore", - chosenTabletId, tablet.getCheckedVersion(), - partition.getVisibleVersion()); + chosenTabletId, tablet.getCheckedVersion(), + partition.getVisibleVersion()); } } else { LOG.info("chose tablet[{}-{}-{}-{}-{}] to check consistency", db.getId(), - table.getId(), partition.getId(), index.getId(), chosenTabletId); + table.getId(), partition.getId(), index.getId(), chosenTabletId); chosenTablets.add(chosenTabletId); } @@ -391,7 +386,7 @@ protected List chooseTablets() { // Since only at most `MAX_JOB_NUM` tablet are chosen, we don't need to release the db read lock // from time to time, just log the time cost here. LOG.info("choose tablets from db[{}-{}](with read lock held) took {}ms", - db.getFullName(), db.getId(), System.currentTimeMillis() - startTime); + db.getFullName(), db.getId(), System.currentTimeMillis() - startTime); locker.unLockDatabase(db, LockType.READ); } } // end while dbQueue @@ -422,7 +417,7 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM return; } OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() - .getTable(db.getId(), info.getTableId()); + .getTable(db.getId(), info.getTableId()); if (table == null) { LOG.warn("replay finish consistency check failed, table is null, info: {}", info); return; @@ -435,7 +430,7 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM LOG.warn("replay finish consistency check failed, partition is null, info: {}", info); return; } - MaterializedIndex index = partition.getIndex(info.getIndexId()); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getIndex(info.getIndexId()); if (index == null) { LOG.warn("replay finish consistency check failed, index is null, info: {}", info); return; diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java index 0b822f6182460..610171699da82 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java @@ -119,7 +119,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response for (Partition partition : olapTable.getPartitions()) { String partitionName = partition.getName(); - MaterializedIndex baseIndex = partition.getBaseIndex(); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Tablet tablet : baseIndex.getTablets()) { List row = Lists.newArrayList(); row.add(tableName); @@ -145,7 +145,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response for (Partition partition : olapTable.getPartitions()) { String partitionName = partition.getName(); - MaterializedIndex baseIndex = partition.getBaseIndex(); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Tablet tablet : baseIndex.getTablets()) { List row = Lists.newArrayList(); row.add(tableName); diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/vo/PartitionInfoView.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/vo/PartitionInfoView.java index 316695f3289cf..01717ec1a5fb7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/vo/PartitionInfoView.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/vo/PartitionInfoView.java @@ -21,6 +21,7 @@ import com.starrocks.catalog.Partition; import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.PartitionType; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.RangePartitionInfo; import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; @@ -140,9 +141,11 @@ public static PartitionView createFrom(PartitionInfo partitionInfo, Partition pa pvo.setDistributionType(distributionInfo.getTypeStr()); }); - pvo.setVisibleVersion(partition.getVisibleVersion()); - pvo.setVisibleVersionTime(partition.getVisibleVersionTime()); - pvo.setNextVersion(partition.getNextVersion()); + PhysicalPartition physicalPartition = partition.getDefaultPhysicalPartition(); + + pvo.setVisibleVersion(physicalPartition.getVisibleVersion()); + pvo.setVisibleVersionTime(physicalPartition.getVisibleVersionTime()); + pvo.setNextVersion(physicalPartition.getNextVersion()); PartitionType partitionType = partitionInfo.getType(); switch (partitionType) { @@ -166,7 +169,7 @@ public static PartitionView createFrom(PartitionInfo partitionInfo, Partition pa // TODO add more type support in the future } - List allIndices = partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL); + List allIndices = physicalPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL); if (CollectionUtils.isNotEmpty(allIndices)) { MaterializedIndex materializedIndex = allIndices.get(0); List tablets = materializedIndex.getTablets(); diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java index 27995d46cb83c..b343bfdfb884e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java @@ -204,7 +204,7 @@ public List> getArbitraryTabletBucketsSeq() throws DdlException { public List getShardGroupIds() { List shardGroupIds = new ArrayList<>(); for (Partition p : getAllPartitions()) { - shardGroupIds.add(p.getShardGroupId()); + shardGroupIds.add(p.getDefaultPhysicalPartition().getShardGroupId()); } return shardGroupIds; } diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java index fde1e0fb051ca..af9beb932c517 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java @@ -99,7 +99,8 @@ protected void createReplicas(OlapTable localTbl, Partition restorePart) { @Override protected void genFileMapping(OlapTable localTbl, Partition localPartition, Long remoteTblId, BackupJobInfo.BackupPartitionInfo backupPartInfo, boolean overwrite) { - for (MaterializedIndex localIdx : localPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { + for (MaterializedIndex localIdx : localPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { BackupIndexInfo backupIdxInfo = backupPartInfo.getIdx(localTbl.getIndexNameById(localIdx.getId())); Preconditions.checkState(backupIdxInfo.tablets.size() == localIdx.getTablets().size()); for (int i = 0; i < localIdx.getTablets().size(); i++) { @@ -127,7 +128,7 @@ protected void prepareAndSendSnapshotTasks(Database db) { OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() .getTable(db.getId(), idChain.getTblId()); Partition part = tbl.getPartition(idChain.getPartId()); - MaterializedIndex index = part.getIndex(idChain.getIdxId()); + MaterializedIndex index = part.getDefaultPhysicalPartition().getIndex(idChain.getIdxId()); tablet = (LakeTablet) index.getTablet(idChain.getTabletId()); Long computeNodeId = GlobalStateMgr.getCurrentState().getWarehouseMgr() .getComputeNodeId(WarehouseManager.DEFAULT_WAREHOUSE_NAME, tablet); @@ -171,7 +172,7 @@ protected void prepareDownloadTasks(List beSnapshotInfos, Database return; } - MaterializedIndex idx = part.getIndex(info.getIndexId()); + MaterializedIndex idx = part.getDefaultPhysicalPartition().getIndex(info.getIndexId()); if (idx == null) { status = new Status(Status.ErrCode.NOT_FOUND, "index " + info.getIndexId() + " does not exist in partion " + part.getName() @@ -277,7 +278,8 @@ public static LakeRestoreJob read(DataInput in) throws IOException { @Override protected void modifyInvertedIndex(OlapTable restoreTbl, Partition restorePart) { - for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { + for (MaterializedIndex restoredIdx : restorePart.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { MaterializedIndexMeta indexMeta = restoreTbl.getIndexMetaByIndexId(restoredIdx.getId()); TStorageMedium medium = restoreTbl.getPartitionInfo().getDataProperty(restorePart.getId()).getStorageMedium(); TabletMeta tabletMeta = new TabletMeta(dbId, restoreTbl.getId(), restorePart.getId(), diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java index 2d0b6c217bf78..1f3a0598326e5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java @@ -951,13 +951,16 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { TBasePartitionDesc basePartitionDesc = new TBasePartitionDesc(); // fill partition meta info for (Partition partition : olapTable.getAllPartitions()) { + + PhysicalPartition physicalPartition = partition.getDefaultPhysicalPartition(); + TPartitionMeta partitionMeta = new TPartitionMeta(); partitionMeta.setPartition_id(partition.getId()); partitionMeta.setPartition_name(partition.getName()); partitionMeta.setState(partition.getState().name()); - partitionMeta.setVisible_version(partition.getVisibleVersion()); - partitionMeta.setVisible_time(partition.getVisibleVersionTime()); - partitionMeta.setNext_version(partition.getNextVersion()); + partitionMeta.setVisible_version(physicalPartition.getVisibleVersion()); + partitionMeta.setVisible_time(physicalPartition.getVisibleVersionTime()); + partitionMeta.setNext_version(physicalPartition.getNextVersion()); partitionMeta.setIs_temp(olapTable.getPartition(partition.getName(), true) != null); tableMeta.addToPartitions(partitionMeta); short replicaNum = partitionInfo.getReplicationNum(partition.getId()); @@ -1031,7 +1034,8 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { } for (Partition partition : olapTable.getAllPartitions()) { - List indexes = partition.getMaterializedIndices(IndexExtState.ALL); + List indexes = + partition.getDefaultPhysicalPartition().getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : indexes) { TIndexMeta indexMeta = new TIndexMeta(); indexMeta.setIndex_id(index.getId()); diff --git a/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java index de472241445b8..e68ae7c58880c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java @@ -625,7 +625,7 @@ private boolean checkDelete(OlapTable table, List partitions, List> indexIdToSchema = table.getIndexIdToSchema(); - Partition partition = partitions.get(0); + PhysicalPartition partition = partitions.get(0).getDefaultPhysicalPartition(); for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { if (table.getBaseIndexId() == index.getId()) { continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java b/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java index 895746176ac94..1564b69de24c7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java @@ -364,7 +364,7 @@ private void genTaskFragments(List fragments, List scanN if (tabletMeta.isLakeTablet()) { Partition partition = exportTable.getPartition(tabletMeta.getPartitionId()); if (partition != null) { - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getIndex(tabletMeta.getIndexId()); if (index != null) { Tablet tablet = index.getTablet(tabletId); if (tablet != null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java b/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java index f8683c9ab83e2..586b547d919d2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java @@ -27,6 +27,7 @@ import com.starrocks.catalog.Partition; import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.PartitionType; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.SinglePartitionInfo; import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; @@ -386,14 +387,14 @@ private void gc(boolean isReplay) { if (job.getTmpPartitionIds() != null) { for (long pid : job.getTmpPartitionIds()) { LOG.info("drop temp partition:{}", pid); - - Partition partition = targetTable.getPartition(pid); - if (partition != null) { - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + PhysicalPartition physicalPartition = targetTable.getPhysicalPartition(pid); + if (physicalPartition != null) { + for (MaterializedIndex index : + physicalPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { // hash set is able to deduplicate the elements sourceTablets.addAll(index.getTablets()); } - targetTable.dropTempPartition(partition.getName(), true); + targetTable.dropTempPartition(physicalPartition.getName(), true); } else { LOG.warn("partition {} is null", pid); } @@ -438,7 +439,8 @@ private void doCommit(boolean isReplay) { Set sourceTablets = Sets.newHashSet(); sourcePartitionNames.forEach(name -> { Partition partition = targetTable.getPartition(name); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { sourceTablets.addAll(index.getTablets()); } }); diff --git a/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java b/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java index ecacec80acc97..05cd7a8561b45 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java @@ -153,7 +153,8 @@ public static void createAndAddTempPartitionsForTable(Database db, OlapTable tar public static void clearTabletsFromInvertedIndex(List partitions) { TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); for (Partition partition : partitions) { - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex materializedIndex : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { for (Tablet tablet : materializedIndex.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java index df06da905f646..fbf1dd6cdd848 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java @@ -506,12 +506,13 @@ private Set submitPushTasks() throws UserException { LOG.warn("partition does not exist. id: {}", partitionId); continue; } - long partitionVersion = partition.getVisibleVersion(); + long partitionVersion = partition.getDefaultPhysicalPartition().getVisibleVersion(); hasLoadPartitions = true; int quorumReplicaNum = table.getPartitionInfo().getQuorumNum(partitionId, table.writeQuorum()); - List indexes = partition.getMaterializedIndices(IndexExtState.ALL); + List indexes = partition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : indexes) { long indexId = index.getId(); int schemaHash = indexToSchemaHash.get(indexId); diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/PhysicalPartitionPersistInfoV2.java b/fe/fe-core/src/main/java/com/starrocks/persist/PhysicalPartitionPersistInfoV2.java index e3677ec0d0878..715898a7635bc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/PhysicalPartitionPersistInfoV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/PhysicalPartitionPersistInfoV2.java @@ -12,12 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.persist; import com.google.gson.annotations.SerializedName; import com.starrocks.catalog.PhysicalPartition; -import com.starrocks.catalog.PhysicalPartitionImpl; import com.starrocks.common.io.Text; import com.starrocks.common.io.Writable; import com.starrocks.persist.gson.GsonUtils; @@ -35,9 +33,9 @@ public class PhysicalPartitionPersistInfoV2 implements Writable { @SerializedName("partitionId") private Long partitionId; @SerializedName("physicalPartition") - private PhysicalPartitionImpl partition; + private PhysicalPartition partition; - public PhysicalPartitionPersistInfoV2(Long dbId, Long tableId, Long partitionId, PhysicalPartitionImpl partition) { + public PhysicalPartitionPersistInfoV2(Long dbId, Long tableId, Long partitionId, PhysicalPartition partition) { this.dbId = dbId; this.tableId = tableId; this.partitionId = partitionId; diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/MetaScanNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/MetaScanNode.java index a6909e9f65b52..70991cb054749 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/MetaScanNode.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/MetaScanNode.java @@ -67,11 +67,11 @@ public MetaScanNode(PlanNodeId id, TupleDescriptor desc, OlapTable olapTable, public void computeRangeLocations() { Collection partitions = olapTable.getPartitions(); for (Partition partition : partitions) { - MaterializedIndex index = partition.getBaseIndex(); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getBaseIndex(); int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); List tablets = index.getTablets(); - long visibleVersion = partition.getVisibleVersion(); + long visibleVersion = partition.getDefaultPhysicalPartition().getVisibleVersion(); String visibleVersionStr = String.valueOf(visibleVersion); for (Tablet tablet : tablets) { diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java index a55ec81cce053..1fc6ad8354921 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java @@ -416,7 +416,7 @@ public List updateScanRangeLocations(List listMaterializedViewStatus( // rows if (olapTable.getPartitionInfo().getType() == PartitionType.UNPARTITIONED) { Partition partition = olapTable.getPartitions().iterator().next(); - MaterializedIndex index = partition.getIndex(mvId); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getIndex(mvId); mvStatus.setRows(index.getRowCount()); } else { mvStatus.setRows(0L); diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java index bdb6f46550b97..13882445dfe0a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java @@ -1355,7 +1355,9 @@ private Map getRefreshedPartitionInf continue; } MaterializedView.BasePartitionInfo basePartitionInfo = new MaterializedView.BasePartitionInfo( - partition.getId(), partition.getVisibleVersion(), partition.getVisibleVersionTime()); + partition.getId(), + partition.getDefaultPhysicalPartition().getVisibleVersion(), + partition.getDefaultPhysicalPartition().getVisibleVersionTime()); partitionInfos.put(partition.getName(), basePartitionInfo); } LOG.info("Collect olap base table {}'s refreshed partition infos: {}", baseTable.getName(), partitionInfos); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index f8ab87e29a6f4..c366cd03f523e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -85,7 +85,6 @@ import com.starrocks.catalog.PartitionKey; import com.starrocks.catalog.PartitionType; import com.starrocks.catalog.PhysicalPartition; -import com.starrocks.catalog.PhysicalPartitionImpl; import com.starrocks.catalog.PrimitiveType; import com.starrocks.catalog.RandomDistributionInfo; import com.starrocks.catalog.RangePartitionInfo; @@ -159,7 +158,6 @@ import com.starrocks.persist.ModifyTablePropertyOperationLog; import com.starrocks.persist.MultiEraseTableInfo; import com.starrocks.persist.OperationType; -import com.starrocks.persist.PartitionPersistInfo; import com.starrocks.persist.PartitionPersistInfoV2; import com.starrocks.persist.PartitionVersionRecoveryInfo; import com.starrocks.persist.PartitionVersionRecoveryInfo.PartitionVersion; @@ -1167,11 +1165,9 @@ private void addSubPartitionLog(Database db, OlapTable olapTable, Partition part List subPartitioins) throws DdlException { List partitionInfoV2List = Lists.newArrayList(); for (PhysicalPartition subPartition : subPartitioins) { - if (subPartition instanceof PhysicalPartitionImpl) { - PhysicalPartitionPersistInfoV2 info = new PhysicalPartitionPersistInfoV2(db.getId(), olapTable.getId(), - partition.getId(), (PhysicalPartitionImpl) subPartition); - partitionInfoV2List.add(info); - } + PhysicalPartitionPersistInfoV2 info = + new PhysicalPartitionPersistInfoV2(db.getId(), olapTable.getId(), partition.getId(), subPartition); + partitionInfoV2List.add(info); } AddSubPartitionsInfoV2 infos = new AddSubPartitionsInfoV2(partitionInfoV2List); @@ -1353,7 +1349,8 @@ public void replayAddPartition(PartitionPersistInfoV2 info) throws DdlException if (!isCheckpointThread()) { // add to inverted index TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : + partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), partition.getId(), @@ -1375,51 +1372,6 @@ public void replayAddPartition(PartitionPersistInfoV2 info) throws DdlException } } - public void replayAddPartition(PartitionPersistInfo info) throws DdlException { - Database db = this.getDb(info.getDbId()); - Locker locker = new Locker(); - locker.lockDatabase(db, LockType.WRITE); - try { - OlapTable olapTable = (OlapTable) getTable(db.getId(), info.getTableId()); - Partition partition = info.getPartition(); - - PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - if (info.isTempPartition()) { - olapTable.addTempPartition(partition); - } else { - olapTable.addPartition(partition); - } - - if (partitionInfo.isRangePartition()) { - ((RangePartitionInfo) partitionInfo).unprotectHandleNewSinglePartitionDesc(partition.getId(), - info.isTempPartition(), info.getRange(), info.getDataProperty(), info.getReplicationNum(), - info.isInMemory()); - } else { - partitionInfo.addPartition( - partition.getId(), info.getDataProperty(), info.getReplicationNum(), info.isInMemory()); - } - if (!isCheckpointThread()) { - // add to inverted index - TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - long indexId = index.getId(); - int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), partition.getId(), - index.getId(), schemaHash, info.getDataProperty().getStorageMedium()); - for (Tablet tablet : index.getTablets()) { - long tabletId = tablet.getId(); - invertedIndex.addTablet(tabletId, tabletMeta); - for (Replica replica : ((LocalTablet) tablet).getImmutableReplicas()) { - invertedIndex.addReplica(tabletId, replica); - } - } - } - } - } finally { - locker.unLockDatabase(db, LockType.WRITE); - } - } - @Override public void dropPartition(Database db, Table table, DropPartitionClause clause) throws DdlException { CatalogUtils.checkTableExist(db, table.getName()); @@ -1595,12 +1547,12 @@ private PhysicalPartition createPhysicalPartition(String name, Database db, Olap Long id = GlobalStateMgr.getCurrentState().getNextId(); // physical partitions in the same logical partition use the same shard_group_id, // so that the shards of this logical partition are more evenly distributed. - long shardGroupId = partition.getShardGroupId(); + long shardGroupId = partition.getDefaultPhysicalPartition().getShardGroupId(); if (name == null) { name = partition.generatePhysicalPartitionName(id); } - PhysicalPartitionImpl physicalParition = new PhysicalPartitionImpl( + PhysicalPartition physicalParition = new PhysicalPartition( id, name, partition.getId(), shardGroupId, indexMap.get(olapTable.getBaseIndexId())); PartitionInfo partitionInfo = olapTable.getPartitionInfo(); @@ -1762,12 +1714,24 @@ Partition createPartition(Database db, OlapTable table, long partitionId, String createShardGroup(db.getId(), table.getId(), partitionId); } - Partition partition = - new Partition(partitionId, partitionName, indexMap.get(table.getBaseIndexId()), - distributionInfo, shardGroupId); + Partition logicalPartition = new Partition( + partitionId, + partitionName, + distributionInfo); + + PhysicalPartition physicalPartition = new PhysicalPartition( + partitionId, + partitionName, + partitionId, + shardGroupId, + indexMap.get(table.getBaseIndexId())); + + logicalPartition.addSubPartition(physicalPartition); + + //LogicalPartition partition = new LogicalPartition(partitionId, partitionName, indexMap.get(table.getBaseIndexId()), distributionInfo, shardGroupId); // version if (version != null) { - partition.updateVisibleVersion(version); + physicalPartition.updateVisibleVersion(version); } short replicationNum = partitionInfo.getReplicationNum(partitionId); @@ -1787,14 +1751,14 @@ Partition createPartition(Database db, OlapTable table, long partitionId, String tabletMeta, tabletIdSet, warehouseId); } else { createOlapTablets(table, index, Replica.ReplicaState.NORMAL, distributionInfo, - partition.getVisibleVersion(), replicationNum, tabletMeta, tabletIdSet); + physicalPartition.getVisibleVersion(), replicationNum, tabletMeta, tabletIdSet); } if (index.getId() != table.getBaseIndexId()) { // add rollup index to partition - partition.createRollupIndex(index); + physicalPartition.createRollupIndex(index); } } - return partition; + return logicalPartition; } void buildPartitions(Database db, OlapTable table, List partitions, long warehouseId) @@ -4766,7 +4730,7 @@ public void replayBackendTabletsInfo(BackendTabletsInfo backendTabletsInfo) { if (partition == null) { continue; } - MaterializedIndex mindex = partition.getIndex(info.getIndexId()); + MaterializedIndex mindex = partition.getDefaultPhysicalPartition().getIndex(info.getIndexId()); if (mindex == null) { continue; } @@ -4976,7 +4940,7 @@ public void setPartitionVersion(AdminSetPartitionVersionStmt stmt) { if (partition.getSubPartitions().size() >= 2) { throw ErrorReportException.report(ErrorCode.ERR_MULTI_SUB_PARTITION, stmt.getPartitionName()); } - physicalPartition = partition; + physicalPartition = partition.getDefaultPhysicalPartition(); } long visibleVersionTime = System.currentTimeMillis(); @@ -5021,10 +4985,12 @@ public void onEraseDatabase(long dbId) { public void onErasePartition(Partition partition) { // remove tablet in inverted index TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - for (Tablet tablet : index.getTablets()) { - long tabletId = tablet.getId(); - invertedIndex.deleteTablet(tabletId); + for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + for (MaterializedIndex index : physicalPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (Tablet tablet : index.getTablets()) { + long tabletId = tablet.getId(); + invertedIndex.deleteTablet(tabletId); + } } } } diff --git a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java index c981b44577b03..c45011d741679 100644 --- a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java @@ -2333,7 +2333,8 @@ private static TCreatePartitionResult buildCreatePartitionResponse(OlapTable ola buildPartitionInfo(olapTable, partitions, partition, tPartition, txnState); // tablet int quorum = olapTable.getPartitionInfo().getQuorumNum(partition.getId(), olapTable.writeQuorum()); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { if (olapTable.isCloudNativeTable()) { for (Tablet tablet : index.getTablets()) { LakeTablet cloudNativeTablet = (LakeTablet) tablet; @@ -2446,7 +2447,8 @@ private static void buildPartitionInfo(OlapTable olapTable, List createTab sb.append(entry.getValue().lowerEndpoint().toSql()); sb.append(", ").append(entry.getValue().upperEndpoint().toSql()).append(")"); sb.append("(\"version_info\" = \""); - sb.append(partition.getVisibleVersion()).append("\""); + sb.append(partition.getDefaultPhysicalPartition().getVisibleVersion()).append("\""); sb.append(");"); addPartitionStmt.add(sb.toString()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java index 3894dfb492714..17da5e38db3c0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java @@ -1644,7 +1644,8 @@ public Void visitDictQueryExpr(DictQueryExpr node, Scope context) { dictQueryExpr.setTbl_name(tableName.getTbl()); Map partitionVersion = new HashMap<>(); - dictTable.getPartitions().forEach(p -> partitionVersion.put(p.getId(), p.getVisibleVersion())); + dictTable.getPartitions().forEach(p -> + partitionVersion.put(p.getId(), p.getDefaultPhysicalPartition().getVisibleVersion())); dictQueryExpr.setPartition_version(partitionVersion); List keyFields = keyColumns.stream().map(Column::getName).collect(Collectors.toList()); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRule.java index f82b0cf3bedb9..2fba2ee45c153 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRule.java @@ -504,7 +504,7 @@ private long selectBestRowCountIndex(Set indexesMatchingBestPrefixIndex, O for (Long indexId : indexesMatchingBestPrefixIndex) { long rowCount = 0; for (Partition partition : olapTable.getPartitions()) { - rowCount += partition.getIndex(indexId).getRowCount(); + rowCount += partition.getDefaultPhysicalPartition().getIndex(indexId).getRowCount(); } if (rowCount < minRowCount) { minRowCount = rowCount; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/AddDecodeNodeForDictStringRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/AddDecodeNodeForDictStringRule.java index d9ab65247fa2e..cd7bbdac403b4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/AddDecodeNodeForDictStringRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/AddDecodeNodeForDictStringRule.java @@ -26,7 +26,6 @@ import com.starrocks.catalog.FunctionSet; import com.starrocks.catalog.KeysType; import com.starrocks.catalog.OlapTable; -import com.starrocks.catalog.Partition; import com.starrocks.catalog.Type; import com.starrocks.common.FeConstants; import com.starrocks.common.Pair; @@ -930,8 +929,8 @@ public OptExpression rewrite(OptExpression root, TaskContext taskContext) { for (PhysicalOlapScanOperator scanOperator : scanOperators) { OlapTable table = (OlapTable) scanOperator.getTable(); - long version = table.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo) - .orElse(0L); + long version = table.getPartitions().stream().map(p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()) + .max(Long::compareTo).orElse(0L); if ((table.getKeysType().equals(KeysType.PRIMARY_KEYS))) { continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/lowcardinality/DecodeCollector.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/lowcardinality/DecodeCollector.java index e98409a33246f..a7aa8681e216a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/lowcardinality/DecodeCollector.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/tree/lowcardinality/DecodeCollector.java @@ -25,7 +25,6 @@ import com.starrocks.catalog.FunctionSet; import com.starrocks.catalog.KeysType; import com.starrocks.catalog.OlapTable; -import com.starrocks.catalog.Partition; import com.starrocks.catalog.Type; import com.starrocks.common.FeConstants; import com.starrocks.qe.SessionVariable; @@ -521,8 +520,8 @@ public DecodeInfo visitPhysicalDistribution(OptExpression optExpression, DecodeI public DecodeInfo visitPhysicalOlapScan(OptExpression optExpression, DecodeInfo context) { PhysicalOlapScanOperator scan = optExpression.getOp().cast(); OlapTable table = (OlapTable) scan.getTable(); - long version = table.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo) - .orElse(0L); + long version = table.getPartitions().stream().map(p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()) + .max(Long::compareTo).orElse(0L); if ((table.getKeysType().equals(KeysType.PRIMARY_KEYS))) { return DecodeInfo.EMPTY; diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java index 69a4f2329ed1e..cefdbc23ed93f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java @@ -20,7 +20,6 @@ import com.starrocks.catalog.Database; import com.starrocks.catalog.InternalCatalog; import com.starrocks.catalog.OlapTable; -import com.starrocks.catalog.Partition; import com.starrocks.catalog.Table; import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; @@ -203,7 +202,7 @@ public static Pair, Status> queryDictSync(Long dbId, Long t } OlapTable olapTable = (OlapTable) table; - long version = olapTable.getPartitions().stream().map(Partition::getVisibleVersionTime) + long version = olapTable.getPartitions().stream().map(p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()) .max(Long::compareTo).orElse(0L); String columnName = MetaUtils.getColumnNameByColumnId(dbId, tableId, columnId); String catalogName = InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME; diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java index d58477592d2fc..bf1ddfcd77b12 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java @@ -164,7 +164,7 @@ public static void triggerCollectionOnFirstLoad(TransactionState txnState, Datab // statistic collect granularity is logic partition. PhysicalPartition physicalPartition = table.getPhysicalPartition(physicalPartitionId); if (physicalPartition != null) { - Partition partition = table.getPartition(physicalPartition.getParentId()); + PhysicalPartition partition = table.getPartition(physicalPartition.getParentId()).getDefaultPhysicalPartition(); if (partition != null && partition.isFirstLoad()) { collectPartitionIds.add(partition.getId()); } @@ -270,7 +270,7 @@ public static boolean checkStatisticTableStateNormal() { // check replicate miss for (Partition partition : table.getPartitions()) { - if (partition.getBaseIndex().getTablets().stream() + if (partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().stream() .anyMatch(t -> ((LocalTablet) t).getNormalReplicaBackendIds().isEmpty())) { return false; } @@ -282,7 +282,7 @@ public static boolean checkStatisticTableStateNormal() { public static LocalDateTime getTableLastUpdateTime(Table table) { if (table.isNativeTableOrMaterializedView()) { - long maxTime = table.getPartitions().stream().map(Partition::getVisibleVersionTime) + long maxTime = table.getPartitions().stream().map(p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()) .max(Long::compareTo).orElse(0L); return LocalDateTime.ofInstant(Instant.ofEpochMilli(maxTime), Clock.systemDefaultZone().getZone()); } else if (table.isHiveTable()) { @@ -313,7 +313,7 @@ public static LocalDateTime getTableLastUpdateTime(Table table) { } public static LocalDateTime getPartitionLastUpdateTime(Partition partition) { - long time = partition.getVisibleVersionTime(); + long time = partition.getDefaultPhysicalPartition().getVisibleVersionTime(); return LocalDateTime.ofInstant(Instant.ofEpochMilli(time), Clock.systemDefaultZone().getZone()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java index ca983ef3c1620..9a90de81299a6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java @@ -103,7 +103,7 @@ private boolean checkReplicateNormal(String tableName) { boolean check = true; for (Partition partition : table.getPartitions()) { // check replicate miss - if (partition.getBaseIndex().getTablets().stream() + if (partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().stream() .anyMatch(t -> ((LocalTablet) t).getNormalReplicaBackendIds().isEmpty())) { check = false; break; diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/sample/TabletSampleManager.java b/fe/fe-core/src/main/java/com/starrocks/statistic/sample/TabletSampleManager.java index 8b76a132674b2..85d0117871e4d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/sample/TabletSampleManager.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/sample/TabletSampleManager.java @@ -14,7 +14,9 @@ package com.starrocks.statistic.sample; +import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; import com.starrocks.common.Config; @@ -33,16 +35,12 @@ public class TabletSampleManager { private static double LOW_WEIGHT_READ_RATIO = 0.8; - private static long HIGH_WEIGHT_ROWS_THRESHOLD = 10000000L; - private static long MEDIUM_HIGH_WEIGHT_ROWS_THRESHOLD = 1000000L; - private static long MEDIUM_LOW_WEIGHT_ROWS_THRESHOLD = 100000L; - private final SampleTabletSlot highWeight; private final SampleTabletSlot mediumHighWeight; @@ -53,7 +51,6 @@ public class TabletSampleManager { private final long sampleRowsLimit; - public static TabletSampleManager init(Map properties, Table table) { double highSampleRatio = Double.parseDouble(properties.getOrDefault(StatsConstants.HIGH_WEIGHT_SAMPLE_RATIO, "0.5")); @@ -73,8 +70,9 @@ public static TabletSampleManager init(Map properties, Table tab manager.classifyTablet(table); return manager; } + private TabletSampleManager(double highSampleRatio, double mediumHighRatio, double mediumLowRatio, double lowRatio, - int maxSize, long sampleRowsLimit) { + int maxSize, long sampleRowsLimit) { this.highWeight = new SampleTabletSlot(highSampleRatio, HIGH_WEIGHT_READ_RATIO, maxSize); this.mediumHighWeight = new SampleTabletSlot(mediumHighRatio, MEDIUM_HIGH_WEIGHT_READ_RATIO, maxSize); this.mediumLowWeight = new SampleTabletSlot(mediumLowRatio, MEDIUM_LOW_WEIGHT_READ_RATIO, maxSize); @@ -83,23 +81,27 @@ private TabletSampleManager(double highSampleRatio, double mediumHighRatio, doub } private void classifyTablet(Table table) { - for (Partition p : table.getPartitions()) { - if (!p.hasData()) { - continue; - } - long partitionId = p.getId(); - for (Tablet tablet : p.getBaseIndex().getTablets()) { - long tabletId = tablet.getId(); - long rowCount = tablet.getFuzzyRowCount(); - TabletStats tabletStats = new TabletStats(tabletId, partitionId, rowCount); - if (rowCount >= HIGH_WEIGHT_ROWS_THRESHOLD) { - highWeight.addTabletStats(tabletStats); - } else if (rowCount >= MEDIUM_HIGH_WEIGHT_ROWS_THRESHOLD) { - mediumHighWeight.addTabletStats(tabletStats); - } else if (rowCount >= MEDIUM_LOW_WEIGHT_ROWS_THRESHOLD) { - mediumLowWeight.addTabletStats(tabletStats); - } else { - lowWeight.addTabletStats(tabletStats); + if (table instanceof OlapTable) { + OlapTable olapTable = (OlapTable) table; + for (Partition logicalPartition : olapTable.getPartitions()) { + if (!logicalPartition.hasData()) { + continue; + } + for (PhysicalPartition physicalPartition : logicalPartition.getSubPartitions()) { + for (Tablet tablet : physicalPartition.getBaseIndex().getTablets()) { + long tabletId = tablet.getId(); + long rowCount = tablet.getFuzzyRowCount(); + TabletStats tabletStats = new TabletStats(tabletId, physicalPartition.getId(), rowCount); + if (rowCount >= HIGH_WEIGHT_ROWS_THRESHOLD) { + highWeight.addTabletStats(tabletStats); + } else if (rowCount >= MEDIUM_HIGH_WEIGHT_ROWS_THRESHOLD) { + mediumHighWeight.addTabletStats(tabletStats); + } else if (rowCount >= MEDIUM_LOW_WEIGHT_ROWS_THRESHOLD) { + mediumLowWeight.addTabletStats(tabletStats); + } else { + lowWeight.addTabletStats(tabletStats); + } + } } } } @@ -137,7 +139,6 @@ public SampleInfo generateSampleInfo(String dbName, String tableName) { long totalRows = Math.max(highWeight.getRowCount() + mediumHighWeight.getRowCount() + mediumLowWeight.getRowCount() + lowWeight.getRowCount(), 1); - return new SampleInfo( dbName, tableName, sampleTablets * 1.0 / totalTablets, diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java index f04612e80dc1b..f6bea16d91061 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java @@ -964,22 +964,22 @@ public void testSwapTable() throws Exception { OlapTable replace2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "replace2"); Assert.assertEquals(3, - replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace1.getPartition("replace1").getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); Assert.assertEquals(1, - replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace2.getPartition("replace2").getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); alterTableWithNewParser(replaceStmt, false); replace1 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "replace1"); replace2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "replace2"); Assert.assertEquals(1, - replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace1.getPartition("replace1").getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); Assert.assertEquals(3, - replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace2.getPartition("replace2").getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); Assert.assertEquals("replace1", replace1.getIndexNameById(replace1.getBaseIndexId())); Assert.assertEquals("replace2", replace2.getIndexNameById(replace2.getBaseIndexId())); } diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java index 455a5a9cd2322..d6574ab4097a1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java @@ -119,7 +119,7 @@ public void testBatchRollup() throws Exception { } Assert.assertEquals(OlapTableState.NORMAL, tbl.getState()); for (Partition partition : tbl.getPartitions()) { - Assert.assertEquals(4, partition.getMaterializedIndices(IndexExtState.VISIBLE).size()); + Assert.assertEquals(4, partition.getDefaultPhysicalPartition().getMaterializedIndices(IndexExtState.VISIBLE).size()); } } @@ -154,7 +154,7 @@ public void testCancelBatchRollup() throws Exception { } Assert.assertEquals(OlapTableState.NORMAL, tbl.getState()); for (Partition partition : tbl.getPartitions()) { - Assert.assertEquals(1, partition.getMaterializedIndices(IndexExtState.VISIBLE).size()); + Assert.assertEquals(1, partition.getDefaultPhysicalPartition().getMaterializedIndices(IndexExtState.VISIBLE).size()); } } } diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java index 0b225ef82b230..3e459f859a7c7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java @@ -220,8 +220,8 @@ public void testReplay() { for (long partitionId : partitionIndexMap.rowKeySet()) { Partition partition = table.getPartition(partitionId); long commitVersion = commitVersionMap.get(partitionId); - Assert.assertEquals(partition.getVisibleVersion(), commitVersion); - partition.updateVisibleVersion(commitVersion - 1); + Assert.assertEquals(partition.getDefaultPhysicalPartition().getVisibleVersion(), commitVersion); + partition.getDefaultPhysicalPartition().updateVisibleVersion(commitVersion - 1); } replayAlterMetaJob.replay(job); @@ -238,7 +238,7 @@ public void testReplay() { for (long partitionId : partitionIndexMap.rowKeySet()) { Partition partition = table.getPartition(partitionId); long commitVersion = commitVersionMap.get(partitionId); - Assert.assertEquals(partition.getVisibleVersion(), commitVersion); + Assert.assertEquals(partition.getDefaultPhysicalPartition().getVisibleVersion(), commitVersion); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java index 70d7fcae77fad..68dbf7acbeac1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java @@ -175,8 +175,8 @@ public void testReplay() throws Exception { Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Partition partition = table.getPartition("t3"); long baseIndexId = table.getBaseIndexId(); - long initVisibleVersion = partition.getVisibleVersion(); - long initNextVersion = partition.getNextVersion(); + long initVisibleVersion = partition.getDefaultPhysicalPartition().getVisibleVersion(); + long initNextVersion = partition.getDefaultPhysicalPartition().getNextVersion(); Assert.assertEquals(initVisibleVersion + 1, initNextVersion); LakeTableAsyncFastSchemaChangeJob replaySourceJob = new LakeTableAsyncFastSchemaChangeJob(job); @@ -186,10 +186,11 @@ public void testReplay() throws Exception { replaySourceJob.setJobState(AlterJobV2.JobState.FINISHED_REWRITING); replaySourceJob.getCommitVersionMap().put(partition.getId(), initNextVersion); - replaySourceJob.addDirtyPartitionIndex(partition.getId(), baseIndexId, partition.getIndex(baseIndexId)); + replaySourceJob.addDirtyPartitionIndex(partition.getId(), baseIndexId, + partition.getDefaultPhysicalPartition().getIndex(baseIndexId)); job.replay(replaySourceJob); - Assert.assertEquals(initNextVersion + 1, partition.getNextVersion()); - Assert.assertEquals(initVisibleVersion, partition.getVisibleVersion()); + Assert.assertEquals(initNextVersion + 1, partition.getDefaultPhysicalPartition().getNextVersion()); + Assert.assertEquals(initVisibleVersion, partition.getDefaultPhysicalPartition().getVisibleVersion()); replaySourceJob.setJobState(AlterJobV2.JobState.FINISHED); replaySourceJob.setFinishedTimeMs(System.currentTimeMillis()); @@ -197,8 +198,9 @@ public void testReplay() throws Exception { job.replay(replaySourceJob); Assert.assertEquals(AlterJobV2.JobState.FINISHED, job.getJobState()); Assert.assertEquals(replaySourceJob.getFinishedTimeMs(), job.getFinishedTimeMs()); - Assert.assertEquals(initVisibleVersion + 1, partition.getVisibleVersion()); - Assert.assertEquals(partition.getVisibleVersion() + 1, partition.getNextVersion()); + Assert.assertEquals(initVisibleVersion + 1, partition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(partition.getDefaultPhysicalPartition().getVisibleVersion() + 1, + partition.getDefaultPhysicalPartition().getNextVersion()); Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Assert.assertEquals(2, table.getBaseSchema().size()); Assert.assertEquals(0, table.getBaseSchema().get(0).getUniqueId()); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java index f3455325b8285..ea9f76548b0aa 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java @@ -241,7 +241,8 @@ public void testCreateTabletSuccess() throws AlterCancelException { Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Partition partition = table.getPartitions().stream().findFirst().get(); - Assert.assertEquals(0, partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + Assert.assertEquals(0, partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); } @Test @@ -265,7 +266,8 @@ public boolean isPreviousLoadFinished(long dbId, long tableId, long txnId) { Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Partition partition = table.getPartitions().stream().findFirst().get(); - Assert.assertEquals(0, partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + Assert.assertEquals(0, partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); } @Test @@ -292,7 +294,8 @@ public boolean isPreviousLoadFinished(long dbId, long tableId, long txnId) throw Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Partition partition = table.getPartitions().stream().findFirst().get(); - Assert.assertEquals(0, partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + Assert.assertEquals(0, partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); } @Test @@ -323,7 +326,8 @@ public void testTableNotExistWhileWaitingTxn() throws AlterCancelException { Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Partition partition = table.getPartitions().stream().findFirst().get(); - Assert.assertEquals(0, partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + Assert.assertEquals(0, partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); } @Test @@ -356,7 +360,8 @@ public void testTableDroppedBeforeRewriting() throws AlterCancelException { Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Partition partition = table.getPartitions().stream().findFirst().get(); - Assert.assertEquals(0, partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + Assert.assertEquals(0, partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); } @Test @@ -387,7 +392,8 @@ public void sendAgentTask(AgentBatchTask batchTask) { Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); Partition partition = table.getPartitions().stream().findFirst().get(); - Assert.assertEquals(0, partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + Assert.assertEquals(0, partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); } @Test @@ -412,9 +418,9 @@ public void sendAgentTask(AgentBatchTask batchTask) { Assert.assertEquals(1, partitions.size()); Partition partition = partitions.stream().findFirst().orElse(null); Assert.assertNotNull(partition); - Assert.assertEquals(3, partition.getNextVersion()); + Assert.assertEquals(3, partition.getDefaultPhysicalPartition().getNextVersion()); List shadowIndexes = - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); + partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); Assert.assertEquals(1, shadowIndexes.size()); // Does not support cancel job in FINISHED_REWRITING state. @@ -460,16 +466,16 @@ public void sendAgentTask(AgentBatchTask batchTask) { Partition partition = partitions.stream().findFirst().orElse(null); Assert.assertNotNull(partition); - Assert.assertEquals(1, partition.getVisibleVersion()); - Assert.assertEquals(2, partition.getNextVersion()); + Assert.assertEquals(1, partition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, partition.getDefaultPhysicalPartition().getNextVersion()); // Disable send publish version - partition.setNextVersion(3); + partition.getDefaultPhysicalPartition().setNextVersion(3); schemaChangeJob.runRunningJob(); Assert.assertEquals(AlterJobV2.JobState.FINISHED_REWRITING, schemaChangeJob.getJobState()); List shadowIndexes = - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); + partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); Assert.assertEquals(1, shadowIndexes.size()); // The partition's visible version has not catch up with the commit version of this schema change job now. @@ -477,7 +483,7 @@ public void sendAgentTask(AgentBatchTask batchTask) { Assert.assertEquals(AlterJobV2.JobState.FINISHED_REWRITING, schemaChangeJob.getJobState()); // Reset partition's next version - partition.setVisibleVersion(2, System.currentTimeMillis()); + partition.getDefaultPhysicalPartition().setVisibleVersion(2, System.currentTimeMillis()); // Drop table db.dropTable(table.getName()); @@ -514,14 +520,14 @@ public void publishVersion(@NotNull List tablets, TxnInfoPB txnInfo, lon Assert.assertEquals("c1", table.getBaseSchema().get(1).getName()); Assert.assertSame(partition, table.getPartitions().stream().findFirst().get()); - Assert.assertEquals(3, partition.getVisibleVersion()); - Assert.assertEquals(4, partition.getNextVersion()); + Assert.assertEquals(3, partition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(4, partition.getDefaultPhysicalPartition().getNextVersion()); - shadowIndexes = partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); + shadowIndexes = partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); Assert.assertEquals(0, shadowIndexes.size()); List normalIndexes = - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE); + partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE); Assert.assertEquals(1, normalIndexes.size()); MaterializedIndex normalIndex = normalIndexes.get(0); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/MaterializedViewHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/MaterializedViewHandlerTest.java index cee3fca9d59db..a4f8ddc86ea42 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/MaterializedViewHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/MaterializedViewHandlerTest.java @@ -149,7 +149,7 @@ public void testRollupReplica(@Injectable CreateMaterializedViewStmt createMater result = baseIndexId; olapTable.getPhysicalPartitions(); result = Lists.newArrayList(partition); - partition.getIndex(baseIndexId); + partition.getDefaultPhysicalPartition().getIndex(baseIndexId); result = materializedIndex; materializedIndex.getState(); result = MaterializedIndex.IndexState.SHADOW; @@ -337,7 +337,7 @@ public void testCheckDropMaterializedView(@Injectable OlapTable olapTable, @Inje result = 1; olapTable.getPhysicalPartitions(); result = Lists.newArrayList(partition); - partition.getIndex(1L); + partition.getDefaultPhysicalPartition().getIndex(1L); result = materializedIndex; } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java index 6a3975c4be2d7..cf2e87c4804ba 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java @@ -177,7 +177,7 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { Assert.assertEquals(1, alterJobsV2.size()); OnlineOptimizeJobV2 optimizeJob = (OnlineOptimizeJobV2) alterJobsV2.values().stream().findAny().get(); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); + MaterializedIndex baseIndex = testPartition.getDefaultPhysicalPartition().getBaseIndex(); LocalTablet baseTablet = (LocalTablet) baseIndex.getTablets().get(0); List replicas = baseTablet.getImmutableReplicas(); Replica replica1 = replicas.get(0); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java index 15d8d060c7482..229cfb46459ab 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java @@ -259,7 +259,7 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { Assert.assertEquals(1, alterJobsV2.size()); OptimizeJobV2 optimizeJob = (OptimizeJobV2) alterJobsV2.values().stream().findAny().get(); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); + MaterializedIndex baseIndex = testPartition.getDefaultPhysicalPartition().getBaseIndex(); LocalTablet baseTablet = (LocalTablet) baseIndex.getTablets().get(0); List replicas = baseTablet.getImmutableReplicas(); Replica replica1 = replicas.get(0); @@ -420,7 +420,8 @@ public void testOptimizeFailedByVersion() throws Exception { optimizeTasks.get(1).setOptimizeTaskState(Constants.TaskRunState.SUCCESS); for (Partition p : olapTable.getPartitions()) { - p.setVisibleVersion(p.getVisibleVersion() + 1, 0); + p.getDefaultPhysicalPartition().setVisibleVersion( + p.getDefaultPhysicalPartition().getVisibleVersion() + 1, 0); } try { diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java index fc8e951b894be..8b70f990ab9c8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java @@ -163,9 +163,12 @@ public void testSchemaChange1() throws Exception { // runPendingJob rollupJob.runPendingJob(); assertEquals(AlterJobV2.JobState.WAITING_TXN, rollupJob.getJobState()); - assertEquals(2, testPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).size()); - assertEquals(1, testPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); - assertEquals(1, testPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + assertEquals(2, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).size()); + assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); // runWaitingTxnJob rollupJob.runWaitingTxnJob(); @@ -204,7 +207,7 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { assertEquals(1, alterJobsV2.size()); RollupJobV2 rollupJob = (RollupJobV2) alterJobsV2.values().stream().findAny().get(); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); + MaterializedIndex baseIndex = testPartition.getDefaultPhysicalPartition().getBaseIndex(); assertEquals(MaterializedIndex.IndexState.NORMAL, baseIndex.getState()); assertEquals(Partition.PartitionState.NORMAL, testPartition.getState()); assertEquals(OlapTableState.ROLLUP, olapTable.getState()); @@ -226,9 +229,12 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { replica1.setState(Replica.ReplicaState.NORMAL); rollupJob.runPendingJob(); assertEquals(AlterJobV2.JobState.WAITING_TXN, rollupJob.getJobState()); - assertEquals(2, testPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).size()); - assertEquals(1, testPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); - assertEquals(1, testPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); + assertEquals(2, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).size()); + assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW).size()); // runWaitingTxnJob rollupJob.runWaitingTxnJob(); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java index 091b22899e3d7..3e2b69a4972b1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java @@ -144,7 +144,7 @@ public void testSchemaChange1() throws Exception { SchemaChangeJobV2 schemaChangeJob = (SchemaChangeJobV2) alterJobsV2.values().stream().findAny().get(); alterJobsV2.clear(); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); + MaterializedIndex baseIndex = testPartition.getDefaultPhysicalPartition().getBaseIndex(); assertEquals(IndexState.NORMAL, baseIndex.getState()); assertEquals(PartitionState.NORMAL, testPartition.getState()); assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); @@ -160,9 +160,12 @@ public void testSchemaChange1() throws Exception { // runPendingJob schemaChangeJob.runPendingJob(); Assert.assertEquals(JobState.WAITING_TXN, schemaChangeJob.getJobState()); - Assert.assertEquals(2, testPartition.getMaterializedIndices(IndexExtState.ALL).size()); - Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.VISIBLE).size()); - Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.SHADOW).size()); + Assert.assertEquals(2, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.ALL).size()); + Assert.assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE).size()); + Assert.assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.SHADOW).size()); // runWaitingTxnJob schemaChangeJob.runWaitingTxnJob(); @@ -199,7 +202,7 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { SchemaChangeJobV2 schemaChangeJob = (SchemaChangeJobV2) alterJobsV2.values().stream().findAny().get(); alterJobsV2.clear(); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); + MaterializedIndex baseIndex = testPartition.getDefaultPhysicalPartition().getBaseIndex(); assertEquals(IndexState.NORMAL, baseIndex.getState()); assertEquals(PartitionState.NORMAL, testPartition.getState()); assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); @@ -221,9 +224,12 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { replica1.setState(Replica.ReplicaState.NORMAL); schemaChangeJob.runPendingJob(); Assert.assertEquals(JobState.WAITING_TXN, schemaChangeJob.getJobState()); - Assert.assertEquals(2, testPartition.getMaterializedIndices(IndexExtState.ALL).size()); - Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.VISIBLE).size()); - Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.SHADOW).size()); + Assert.assertEquals(2, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.ALL).size()); + Assert.assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE).size()); + Assert.assertEquals(1, testPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.SHADOW).size()); // runWaitingTxnJob schemaChangeJob.runWaitingTxnJob(); @@ -471,7 +477,7 @@ public void testCancelPendingJobWithFlag() throws Exception { SchemaChangeJobV2 schemaChangeJob = (SchemaChangeJobV2) alterJobsV2.values().stream().findAny().get(); alterJobsV2.clear(); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); + MaterializedIndex baseIndex = testPartition.getDefaultPhysicalPartition().getBaseIndex(); assertEquals(IndexState.NORMAL, baseIndex.getState()); assertEquals(PartitionState.NORMAL, testPartition.getState()); assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java b/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java index 0c33c0f430681..49862407084c9 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java @@ -106,11 +106,11 @@ public static OlapTable mockTable(String name) { Partition partition = Deencapsulation.newInstance(Partition.class); new Expectations(partition) { { - partition.getBaseIndex(); + partition.getDefaultPhysicalPartition().getBaseIndex(); minTimes = 0; result = index; - partition.getIndex(30000L); + partition.getDefaultPhysicalPartition().getIndex(30000L); minTimes = 0; result = index; } @@ -237,11 +237,11 @@ public static Analyzer fetchTableAnalyzer() { Partition partition = Deencapsulation.newInstance(Partition.class); new Expectations(partition) { { - partition.getBaseIndex(); + partition.getDefaultPhysicalPartition().getBaseIndex(); minTimes = 0; result = index; - partition.getIndex(30000L); + partition.getDefaultPhysicalPartition().getIndex(30000L); minTimes = 0; result = index; } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java index b990fdace4b53..aa7fa08bb7564 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java @@ -4737,7 +4737,8 @@ public void testCreateMVWithLocationAndPersist() throws Exception { String result = starRocksAssert.showCreateTable("show create table mv_with_location"); System.out.println(result); Assert.assertTrue(result.contains("rack:*")); - for (Tablet tablet : materializedView.getPartitions().iterator().next().getBaseIndex().getTablets()) { + for (Tablet tablet : materializedView.getPartitions().iterator().next() + .getDefaultPhysicalPartition().getBaseIndex().getTablets()) { Assert.assertEquals(backend.getId(), (long) tablet.getBackendIds().iterator().next()); } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java index 4751f3d69f0a8..2e4580a9c3d34 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java @@ -106,7 +106,8 @@ public void testRefreshMaterializedView() throws Exception { refreshScheme.getAsyncRefreshContext().getBaseTableVisibleVersionMap().get(table.getId()); if (partitionInfoMap.containsKey("table_name_tmp_1")) { MaterializedView.BasePartitionInfo partitionInfo = partitionInfoMap.get("table_name_tmp_1"); - Assert.assertEquals(table.getPartition("table_name_tmp_1").getVisibleVersion(), partitionInfo.getVersion()); + Assert.assertEquals(table.getPartition("table_name_tmp_1").getDefaultPhysicalPartition() + .getVisibleVersion(), partitionInfo.getVersion()); } } } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java index a8e9a75593a5b..1cc8858b4b841 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java @@ -143,7 +143,7 @@ public void testRefreshExecution() throws Exception { OlapTable table = (OlapTable) getTable("test", "tbl_with_mv"); Partition p1 = table.getPartition("p1"); Partition p2 = table.getPartition("p2"); - if (p2.getVisibleVersion() == 3) { + if (p2.getDefaultPhysicalPartition().getVisibleVersion() == 3) { MvUpdateInfo mvUpdateInfo = getMvUpdateInfo(mv1); Assert.assertTrue(mvUpdateInfo.getMvToRefreshType() == MvUpdateInfo.MvToRefreshType.FULL); Assert.assertTrue(!mvUpdateInfo.isValidRewrite()); @@ -155,8 +155,8 @@ public void testRefreshExecution() throws Exception { } else { // publish version is async, so version update may be late // for debug - System.out.println("p1 visible version:" + p1.getVisibleVersion()); - System.out.println("p2 visible version:" + p2.getVisibleVersion()); + System.out.println("p1 visible version:" + p1.getDefaultPhysicalPartition().getVisibleVersion()); + System.out.println("p2 visible version:" + p2.getDefaultPhysicalPartition().getVisibleVersion()); System.out.println("mv1 refresh context" + mv1.getRefreshScheme().getAsyncRefreshContext()); System.out.println("mv2 refresh context" + mv2.getRefreshScheme().getAsyncRefreshContext()); } @@ -264,7 +264,8 @@ public void testMaxMVRewriteStaleness2() { Table tbl1 = getTable("test", "tbl_staleness2"); Optional maxPartitionRefreshTimestamp = - tbl1.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo); + tbl1.getPartitions().stream().map( + p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()).max(Long::compareTo); Assert.assertTrue(maxPartitionRefreshTimestamp.isPresent()); MaterializedView mv1 = getMv("test", "mv_with_mv_rewrite_staleness2"); @@ -282,7 +283,8 @@ public void testMaxMVRewriteStaleness2() { Table tbl1 = getTable("test", "tbl_staleness2"); Optional maxPartitionRefreshTimestamp = - tbl1.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo); + tbl1.getPartitions().stream().map( + p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()).max(Long::compareTo); Assert.assertTrue(maxPartitionRefreshTimestamp.isPresent()); MaterializedView mv1 = getMv("test", "mv_with_mv_rewrite_staleness2"); @@ -347,7 +349,8 @@ public void testMaxMVRewriteStaleness3() { { Table tbl1 = getTable("test", "tbl_staleness3"); Optional maxPartitionRefreshTimestamp = - tbl1.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo); + tbl1.getPartitions().stream().map( + p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()).max(Long::compareTo); Assert.assertTrue(maxPartitionRefreshTimestamp.isPresent()); MaterializedView mv1 = getMv("test", "mv_with_mv_rewrite_staleness21"); @@ -362,7 +365,8 @@ public void testMaxMVRewriteStaleness3() { { Table tbl1 = getTable("test", "mv_with_mv_rewrite_staleness21"); Optional maxPartitionRefreshTimestamp = - tbl1.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo); + tbl1.getPartitions().stream().map( + p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()).max(Long::compareTo); Assert.assertTrue(maxPartitionRefreshTimestamp.isPresent()); MaterializedView mv2 = getMv("test", "mv_with_mv_rewrite_staleness22"); @@ -380,7 +384,8 @@ public void testMaxMVRewriteStaleness3() { { Table tbl1 = getTable("test", "tbl_staleness3"); Optional maxPartitionRefreshTimestamp = - tbl1.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo); + tbl1.getPartitions().stream().map( + p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()).max(Long::compareTo); Assert.assertTrue(maxPartitionRefreshTimestamp.isPresent()); MaterializedView mv1 = getMv("test", "mv_with_mv_rewrite_staleness21"); @@ -401,7 +406,8 @@ public void testMaxMVRewriteStaleness3() { { Table tbl1 = getTable("test", "mv_with_mv_rewrite_staleness21"); Optional maxPartitionRefreshTimestamp = - tbl1.getPartitions().stream().map(Partition::getVisibleVersionTime).max(Long::compareTo); + tbl1.getPartitions().stream().map( + p -> p.getDefaultPhysicalPartition().getVisibleVersionTime()).max(Long::compareTo); Assert.assertTrue(maxPartitionRefreshTimestamp.isPresent()); MaterializedView mv2 = getMv("test", "mv_with_mv_rewrite_staleness22"); @@ -465,9 +471,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { .getTable(testDb.getFullName(), tableName.getTbl())); for (Partition partition : tbl.getPartitions()) { if (insertStmt.getTargetPartitionIds().contains(partition.getId())) { - long version = partition.getVisibleVersion() + 1; - partition.setVisibleVersion(version, System.currentTimeMillis()); - MaterializedIndex baseIndex = partition.getBaseIndex(); + long version = partition.getDefaultPhysicalPartition().getVisibleVersion() + 1; + partition.getDefaultPhysicalPartition().setVisibleVersion(version, System.currentTimeMillis()); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); List tablets = baseIndex.getTablets(); for (Tablet tablet : tablets) { List replicas = ((LocalTablet) tablet).getImmutableReplicas(); @@ -569,9 +575,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { .getTable(testDb.getFullName(), tableName.getTbl())); for (Partition partition : tbl.getPartitions()) { if (insertStmt.getTargetPartitionIds().contains(partition.getId())) { - long version = partition.getVisibleVersion() + 1; - partition.setVisibleVersion(version, System.currentTimeMillis()); - MaterializedIndex baseIndex = partition.getBaseIndex(); + long version = partition.getDefaultPhysicalPartition().getVisibleVersion() + 1; + partition.getDefaultPhysicalPartition().setVisibleVersion(version, System.currentTimeMillis()); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); List tablets = baseIndex.getTablets(); for (Tablet tablet : tablets) { List replicas = ((LocalTablet) tablet).getImmutableReplicas(); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java index 8806b92658e2f..69bdecf2aea37 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java @@ -157,8 +157,8 @@ public void done() { if (rootDir != null) { try { Files.walk(Paths.get(Config.tmp_dir), - FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) - .forEach(File::delete); + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) + .forEach(File::delete); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -178,7 +178,7 @@ public void testInit(@Mocked GlobalStateMgr globalStateMgr, @Mocked BrokerMgr br @Test public void testCreateAndDropRepository( - @Mocked GlobalStateMgr globalStateMgr, @Mocked BrokerMgr brokerMgr, @Mocked EditLog editLog) throws Exception { + @Mocked GlobalStateMgr globalStateMgr, @Mocked BrokerMgr brokerMgr, @Mocked EditLog editLog) throws Exception { setUpMocker(globalStateMgr, brokerMgr, editLog); new Expectations() { { @@ -219,24 +219,25 @@ public Status listSnapshots(List snapshotNames) { @Mock public Status getSnapshotInfoFile(String label, String backupTimestamp, List infos) { OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() - .getTable(db.getFullName(), CatalogMocker.TEST_TBL_NAME); + .getTable(db.getFullName(), CatalogMocker.TEST_TBL_NAME); List tbls = Lists.newArrayList(); tbls.add(tbl); Map snapshotInfos = Maps.newHashMap(); for (Partition part : tbl.getPartitions()) { - for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex idx : part.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { List files = Lists.newArrayList(); SnapshotInfo sinfo = new SnapshotInfo(db.getId(), tbl.getId(), part.getId(), idx.getId(), - tablet.getId(), -1, 0, "./path", files); + tablet.getId(), -1, 0, "./path", files); snapshotInfos.put(tablet.getId(), sinfo); } } } BackupJobInfo info = BackupJobInfo.fromCatalog(System.currentTimeMillis(), - "ss2", CatalogMocker.TEST_DB_NAME, - CatalogMocker.TEST_DB_ID, tbls, snapshotInfos); + "ss2", CatalogMocker.TEST_DB_NAME, + CatalogMocker.TEST_DB_ID, tbls, snapshotInfos); infos.add(info); return Status.OK; } @@ -291,7 +292,7 @@ public Table getTable(String dbName, String tblName) { // add repo BackupHandler handler = new BackupHandler(globalStateMgr); CreateRepositoryStmt stmt = new CreateRepositoryStmt(false, "repo", "broker", "bos://location", - Maps.newHashMap()); + Maps.newHashMap()); try { handler.createRepository(stmt); } catch (DdlException e) { @@ -303,7 +304,7 @@ public Table getTable(String dbName, String tblName) { List tblRefs = Lists.newArrayList(); tblRefs.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME), null)); BackupStmt backupStmt = new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label1"), "repo", tblRefs, - null); + null); try { handler.process(backupStmt); } catch (DdlException e1) { @@ -314,7 +315,7 @@ public Table getTable(String dbName, String tblName) { // handleFinishedSnapshotTask BackupJob backupJob = (BackupJob) handler.getJob(CatalogMocker.TEST_DB_ID); SnapshotTask snapshotTask = new SnapshotTask(null, 0, 0, backupJob.getJobId(), CatalogMocker.TEST_DB_ID, - 0, 0, 0, 0, 0, 0, 1, false); + 0, 0, 0, 0, 0, 0, 1, false); TFinishTaskRequest request = new TFinishTaskRequest(); List snapshotFiles = Lists.newArrayList(); request.setSnapshot_files(snapshotFiles); @@ -325,7 +326,7 @@ public Table getTable(String dbName, String tblName) { // handleFinishedSnapshotUploadTask Map srcToDestPath = Maps.newHashMap(); UploadTask uploadTask = new UploadTask(null, 0, 0, backupJob.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath, null, null); + srcToDestPath, null, null); request = new TFinishTaskRequest(); Map> tabletFiles = Maps.newHashMap(); request.setTablet_files(tabletFiles); @@ -361,8 +362,8 @@ public Table getTable(String dbName, String tblName) { List tblRefs1 = Lists.newArrayList(); tblRefs1.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL3_NAME), null)); BackupStmt backupStmt1 = - new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs1, - null); + new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs1, + null); try { handler.process(backupStmt1); } catch (DdlException e1) { @@ -373,7 +374,7 @@ public Table getTable(String dbName, String tblName) { // handleFinishedSnapshotTask BackupJob backupJob1 = (BackupJob) handler.getJob(CatalogMocker.TEST_DB_ID); SnapshotTask snapshotTask1 = new SnapshotTask(null, 0, 0, backupJob1.getJobId(), CatalogMocker.TEST_DB_ID, - 0, 0, 0, 0, 0, 0, 1, false); + 0, 0, 0, 0, 0, 0, 1, false); TFinishTaskRequest request1 = new TFinishTaskRequest(); List snapshotFiles1 = Lists.newArrayList(); request1.setSnapshot_files(snapshotFiles1); @@ -384,7 +385,7 @@ public Table getTable(String dbName, String tblName) { // handleFinishedSnapshotUploadTask Map srcToDestPath1 = Maps.newHashMap(); UploadTask uploadTask1 = new UploadTask(null, 0, 0, backupJob1.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath1, null, null); + srcToDestPath1, null, null); request1 = new TFinishTaskRequest(); Map> tabletFiles1 = Maps.newHashMap(); request1.setTablet_files(tabletFiles1); @@ -422,7 +423,7 @@ public Table getTable(String dbName, String tblName) { Map properties = Maps.newHashMap(); properties.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "ss2"), "repo", tblRefs2, - properties); + properties); try { BackupRestoreAnalyzer.analyze(restoreStmt, new ConnectContext()); } catch (SemanticException e2) { @@ -440,7 +441,7 @@ public Table getTable(String dbName, String tblName) { // handleFinishedSnapshotTask RestoreJob restoreJob = (RestoreJob) handler.getJob(CatalogMocker.TEST_DB_ID); snapshotTask = new SnapshotTask(null, 0, 0, restoreJob.getJobId(), CatalogMocker.TEST_DB_ID, - 0, 0, 0, 0, 0, 0, 1, true); + 0, 0, 0, 0, 0, 0, 1, true); request = new TFinishTaskRequest(); request.setSnapshot_path("./snapshot/path"); request.setTask_status(new TStatus(TStatusCode.OK)); @@ -448,7 +449,7 @@ public Table getTable(String dbName, String tblName) { // handleDownloadSnapshotTask DownloadTask downloadTask = new DownloadTask(null, 0, 0, restoreJob.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath, null, null); + srcToDestPath, null, null); request = new TFinishTaskRequest(); List downloadedTabletIds = Lists.newArrayList(); request.setDownloaded_tablet_ids(downloadedTabletIds); @@ -457,7 +458,7 @@ public Table getTable(String dbName, String tblName) { // handleDirMoveTask DirMoveTask dirMoveTask = new DirMoveTask(null, 0, 0, restoreJob.getJobId(), CatalogMocker.TEST_DB_ID, 0, 0, 0, - 0, "", 0, true); + 0, "", 0, true); request = new TFinishTaskRequest(); request.setTask_status(new TStatus(TStatusCode.OK)); handler.handleDirMoveTask(dirMoveTask, request); @@ -500,7 +501,7 @@ BackupHandler getBackupHandler() { Map properties1 = Maps.newHashMap(); properties1.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt1 = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs3, - properties1); + properties1); try { BackupRestoreAnalyzer.analyze(restoreStmt1, new ConnectContext()); } catch (SemanticException e2) { @@ -518,7 +519,7 @@ BackupHandler getBackupHandler() { // handleFinishedSnapshotTask RestoreJob restoreJob1 = (RestoreJob) handler.getJob(CatalogMocker.TEST_DB_ID); snapshotTask1 = new SnapshotTask(null, 0, 0, restoreJob1.getJobId(), CatalogMocker.TEST_DB_ID, - 0, 0, 0, 0, 0, 0, 1, true); + 0, 0, 0, 0, 0, 0, 1, true); request1 = new TFinishTaskRequest(); request1.setSnapshot_path("./snapshot/path1"); request1.setTask_status(new TStatus(TStatusCode.OK)); @@ -526,7 +527,7 @@ BackupHandler getBackupHandler() { // handleDownloadSnapshotTask DownloadTask downloadTask1 = new DownloadTask(null, 0, 0, restoreJob1.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath1, null, null); + srcToDestPath1, null, null); request1 = new TFinishTaskRequest(); List downloadedTabletIds1 = Lists.newArrayList(); request1.setDownloaded_tablet_ids(downloadedTabletIds1); @@ -535,7 +536,7 @@ BackupHandler getBackupHandler() { // handleDirMoveTask DirMoveTask dirMoveTask1 = new DirMoveTask(null, 0, 0, restoreJob1.getJobId(), CatalogMocker.TEST_DB_ID, 0, 0, 0, - 0, "", 0, true); + 0, "", 0, true); request1 = new TFinishTaskRequest(); request1.setTask_status(new TStatus(TStatusCode.OK)); handler.handleDirMoveTask(dirMoveTask1, request1); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java index 997d26e21f9ef..baa8bf45a546e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java @@ -54,7 +54,6 @@ import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.PartitionKey; import com.starrocks.catalog.PhysicalPartition; -import com.starrocks.catalog.PhysicalPartitionImpl; import com.starrocks.catalog.PrimitiveType; import com.starrocks.catalog.RandomDistributionInfo; import com.starrocks.catalog.RangePartitionInfo; @@ -376,7 +375,7 @@ public static Database mockDb() throws AnalysisException { rollupTabletP1.addReplica(replica10); rollupTabletP1.addReplica(replica11); - partition1.createRollupIndex(rollupIndexP1); + partition1.getDefaultPhysicalPartition().createRollupIndex(rollupIndexP1); // rollup index p2 MaterializedIndex rollupIndexP2 = new MaterializedIndex(TEST_ROLLUP_ID, IndexState.NORMAL); @@ -392,7 +391,7 @@ public static Database mockDb() throws AnalysisException { rollupTabletP2.addReplica(replica13); rollupTabletP2.addReplica(replica14); - partition2.createRollupIndex(rollupIndexP2); + partition2.getDefaultPhysicalPartition().createRollupIndex(rollupIndexP2); olapTable2.setIndexMeta(TEST_ROLLUP_ID, TEST_ROLLUP_NAME, TEST_ROLLUP_SCHEMA, 0, ROLLUP_SCHEMA_HASH, (short) 1, TStorageType.COLUMN, KeysType.AGG_KEYS); @@ -475,7 +474,7 @@ public static Database mockDb() throws AnalysisException { partition1 = new Partition(TEST_PARTITION1_ID, TEST_PARTITION1_NAME, baseIndexP1, distributionInfo4); - PhysicalPartition physicalPartition2 = new PhysicalPartitionImpl( + PhysicalPartition physicalPartition2 = new PhysicalPartition( TEST_PARTITION2_ID, "", TEST_PARTITION1_ID, 0, baseIndexP2); partition1.addSubPartition(physicalPartition2); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java index d0aab1e9fa861..1897734c89a92 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java @@ -146,7 +146,7 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); private BackupMeta backupMeta; @@ -267,7 +267,7 @@ public void setUp() throws AnalysisException { Deencapsulation.setField(globalStateMgr, "backupHandler", backupHandler); systemInfoService = new SystemInfoService(); db = UnitTestUtil.createDbWithMaterializedView(dbId, tblId, partId, idxId, tabletId, - backendId, version, KeysType.DUP_KEYS); + backendId, version, KeysType.DUP_KEYS); mvRestoreContext = new MvRestoreContext(); setUpMocker(); } @@ -283,7 +283,8 @@ private BackupTableInfo mockBackupTableInfo(OlapTable olapTable) { partInfo.name = partition.getName(); tblInfo.partitions.put(partInfo.name, partInfo); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { BackupIndexInfo idxInfo = new BackupIndexInfo(); idxInfo.id = index.getId(); idxInfo.name = olapTable.getIndexNameById(index.getId()); @@ -323,8 +324,8 @@ public RestoreJob createRestoreJob(List restoreTbls) { } RestoreJob job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), - jobInfo, false, 3, 100000, - globalStateMgr, repo.getId(), backupMeta, mvRestoreContext); + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, mvRestoreContext); job.setRepo(repo); // add job into mvRestoreContext @@ -366,7 +367,7 @@ private void checkJobRun(RestoreJob job) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - task.getSignature(), taskStatus); + task.getSignature(), taskStatus); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(task, request)); } @@ -399,7 +400,7 @@ private void checkJobRun(RestoreJob job) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); request.setDownloaded_tablet_ids(downloadedTabletIds); Assert.assertTrue(job.finishTabletDownloadTask((DownloadTask) agentTask, request)); } @@ -431,7 +432,7 @@ private void checkJobRun(RestoreJob job) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); job.finishDirMoveTask((DirMoveTask) agentTask, request); } diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java index 947e2f8231bbe..c72ef8f7ca752 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java @@ -137,7 +137,7 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); private BackupMeta backupMeta; @@ -238,7 +238,8 @@ boolean await(long timeout, TimeUnit unit) { partInfo.name = partition.getName(); tblInfo.partitions.put(partInfo.name, partInfo); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { BackupIndexInfo idxInfo = new BackupIndexInfo(); idxInfo.id = index.getId(); idxInfo.name = expectedRestoreTbl.getIndexNameById(index.getId()); @@ -260,8 +261,8 @@ boolean await(long timeout, TimeUnit unit) { tbls.add(expectedRestoreTbl); backupMeta = new BackupMeta(tbls); job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), - jobInfo, false, 3, 100000, - globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); } @Ignore @@ -297,7 +298,7 @@ public void testRun() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - task.getSignature(), taskStatus); + task.getSignature(), taskStatus); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(task, request)); } @@ -330,7 +331,7 @@ public void testRun() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); request.setDownloaded_tablet_ids(downloadedTabletIds); Assert.assertTrue(job.finishTabletDownloadTask((DownloadTask) agentTask, request)); } @@ -362,7 +363,7 @@ public void testRun() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); job.finishDirMoveTask((DirMoveTask) agentTask, request); } diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java index 1ed06f0ce9df8..eb7c873759766 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java @@ -454,7 +454,8 @@ boolean await(long timeout, TimeUnit unit) { partInfo.name = partition.getName(); tblInfo.partitions.put(partInfo.name, partInfo); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE)) { BackupIndexInfo idxInfo = new BackupIndexInfo(); idxInfo.id = index.getId(); idxInfo.name = expectedRestoreTbl.getIndexNameById(index.getId()); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java index 7618278e8876e..af6f0f4e19917 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java @@ -93,7 +93,8 @@ public void testAdminSetReplicaStatus() throws Exception { // tablet id, backend id List> tabletToBackendList = Lists.newArrayList(); for (Partition partition : tbl.getPartitions()) { - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { for (Tablet tablet : index.getTablets()) { for (Replica replica : ((LocalTablet) tablet).getImmutableReplicas()) { tabletToBackendList.add(Pair.create(tablet.getId(), replica.getBackendId())); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java index 3f63a7dd47abe..db7897f59b6a0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java @@ -97,7 +97,8 @@ private static void checkTableTablet(Table table, boolean expectExist) { private static void checkPartitionTablet(Partition partition, boolean expectExist) { TabletInvertedIndex tabletIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + for (MaterializedIndex index : + partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { TabletMeta meta = tabletIndex.getTabletMeta(tablet.getId()); if (expectExist) { @@ -343,9 +344,12 @@ public void testRecycleLakePartition(@Mocked LakeService lakeService) throws Exc Partition p2 = table1.getPartition("p2"); Partition p3 = table1.getPartition("p3"); Assert.assertNotNull(p1); - Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p1, WarehouseManager.DEFAULT_WAREHOUSE_ID)); - Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p2, WarehouseManager.DEFAULT_WAREHOUSE_ID)); - Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p3, WarehouseManager.DEFAULT_WAREHOUSE_ID)); + Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p1.getDefaultPhysicalPartition(), + WarehouseManager.DEFAULT_WAREHOUSE_ID)); + Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p2.getDefaultPhysicalPartition(), + WarehouseManager.DEFAULT_WAREHOUSE_ID)); + Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p3.getDefaultPhysicalPartition(), + WarehouseManager.DEFAULT_WAREHOUSE_ID)); // Drop partition "p1" alterTable(connectContext, String.format("ALTER TABLE %s.t1 DROP PARTITION p1", dbName)); @@ -440,7 +444,8 @@ public LakeService getLakeService(TNetworkAddress address) throws RpcException { "PROPERTIES('replication_num' = '1');", dbName)); p1 = table2.getPartition("p1"); - Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p1, WarehouseManager.DEFAULT_WAREHOUSE_ID)); + Assert.assertFalse(LakeTableHelper.isSharedPartitionDirectory(p1.getDefaultPhysicalPartition(), + WarehouseManager.DEFAULT_WAREHOUSE_ID)); // Drop partition "p1" alterTable(connectContext, String.format("ALTER TABLE %s.t2 DROP PARTITION p1", dbName)); Assert.assertNull(table2.getPartition("p1")); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java index e60f249f8492e..c9c26f868ec4c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java @@ -117,7 +117,7 @@ public void testGetPhysicalPartition() throws Exception { PhysicalPartition recycledPart = bin.getPhysicalPartition(1L); Assert.assertNotNull(recycledPart); - recycledPart = bin.getPartition(2L); + recycledPart = bin.getPhysicalPartition(2L); Assert.assertEquals(2L, recycledPart.getId()); } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogUtilsTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogUtilsTest.java index 9c56dff9497b1..dcc966e94ab3f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogUtilsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogUtilsTest.java @@ -61,7 +61,7 @@ public void testCalAvgBucketNumOfRecentPartitions_CalculateByDataSize() { partitions.add(partition); when(olapTable.getPartitions()).thenReturn(partitions); when(olapTable.getRecentPartitions(anyInt())).thenReturn(partitions); - when(partition.getVisibleVersion()).thenReturn(2L); + when(partition.getDefaultPhysicalPartition().getVisibleVersion()).thenReturn(2L); when(partition.getDataSize()).thenReturn(2L * FeConstants.AUTO_DISTRIBUTION_UNIT); int bucketNum = CatalogUtils.calAvgBucketNumOfRecentPartitions(olapTable, 1, true); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java index d1809b2a25fca..8cf96d630d1ed 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java @@ -194,7 +194,7 @@ public void testCreateTableNoLocPropBackendWithLocProp() throws SQLException { Assert.assertTrue(table.getLocation().keySet().contains("*")); List partitions = new ArrayList<>(table.getAllPartitions()); - List tablets = partitions.get(0).getBaseIndex().getTablets(); + List tablets = partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets(); Set backendIdsWithLocProp = getBackendIdsWithLocProp(); Set backendIdsWithoutLocProp = getBackendIdsWithoutLocProp(); System.out.println(backendIdsWithLocProp); @@ -240,7 +240,7 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc1"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); List partitions = new ArrayList<>(table.getAllPartitions()); - List tablets = partitions.get(0).getBaseIndex().getTablets(); + List tablets = partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets(); Set backendIdsWithLocProp = getBackendIdsWithLocProp("rack"); Set backendIdsWithoutLocProp = getBackendIdsWithoutLocProp(); // test_table_explicit_loc_backend_with_loc1's replicas should only distribute on backends with rack location @@ -280,7 +280,7 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc Assert.assertTrue(table.getLocation().keySet().contains("rack")); Assert.assertTrue(table.getLocation().keySet().contains("region")); partitions = new ArrayList<>(table.getAllPartitions()); - tablets = partitions.get(0).getBaseIndex().getTablets(); + tablets = partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets(); for (Tablet tablet : tablets) { List replicas = tablet.getAllReplicas(); Set replicaBackendIds = Sets.newHashSet(); @@ -319,7 +319,7 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc Assert.assertTrue(table.getLocation().keySet().contains("rack")); Assert.assertEquals(1, table.getLocation().keySet().size()); partitions = new ArrayList<>(table.getAllPartitions()); - tablets = partitions.get(0).getBaseIndex().getTablets(); + tablets = partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets(); backendIdsWithLocProp = getBackendIdsWithLocProp("rack", "r1"); backendIdsWithLocProp.addAll(getBackendIdsWithLocProp("rack", "rack2")); backendIdsWithLocProp.addAll(getBackendIdsWithLocProp("rack", "rack3")); @@ -354,7 +354,7 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc Assert.assertTrue(table.getLocation().keySet().contains("rack")); Assert.assertTrue(table.getLocation().keySet().contains("region")); partitions = new ArrayList<>(table.getAllPartitions()); - tablets = partitions.get(0).getBaseIndex().getTablets(); + tablets = partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets(); Set allReplicasBackendIds = Sets.newHashSet(); for (Tablet tablet : tablets) { List replicas = tablet.getAllReplicas(); @@ -393,7 +393,7 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc5"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); partitions = new ArrayList<>(table.getAllPartitions()); - tablets = partitions.get(0).getBaseIndex().getTablets(); + tablets = partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets(); allReplicasBackendIds = Sets.newHashSet(); for (Tablet tablet : tablets) { List replicas = tablet.getAllReplicas(); @@ -464,7 +464,7 @@ public void testBestEffortMatchLocationProp() throws SQLException { .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc_best_effort"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); List partitions = new ArrayList<>(table.getAllPartitions()); - List tablets = partitions.get(0).getBaseIndex().getTablets(); + List tablets = partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets(); Set backendIdsWithLocProp = getBackendIdsWithLocProp("rack", "r1"); backendIdsWithLocProp.addAll(getBackendIdsWithLocProp("rack", "rack2")); for (Tablet tablet : tablets) { diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java index ab5109b561a54..e65dba6a93e67 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java @@ -100,7 +100,7 @@ public void testNormalDropPartition() throws Exception { Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Partition partition = table.getPartition("p20210201"); - long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); + long tabletId = partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().get(0).getId(); String dropPartitionSql = " alter table test.tbl1 drop partition p20210201;"; dropPartition(dropPartitionSql); List replicaList = @@ -122,7 +122,7 @@ public void testForceDropPartition() throws Exception { Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Partition partition = table.getPartition("p20210202"); - long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); + long tabletId = partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().get(0).getId(); String dropPartitionSql = " alter table test.tbl1 drop partition p20210202 force;"; dropPartition(dropPartitionSql); List replicaList; @@ -147,7 +147,7 @@ public void testDropPartitionAndReserveTablets() throws Exception { Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Partition partition = table.getPartition("p20210203"); - long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); + long tabletId = partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().get(0).getId(); table.dropPartitionAndReserveTablet("p20210203"); List replicaList = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getReplicasByTabletId(tabletId); @@ -196,7 +196,7 @@ private void checkNormalDropPartitions(String dbName, String tableName, String d for (String partitionName : partitionNames) { Table table = getTable(dbName, tableName); Partition partition = table.getPartition(partitionName); - long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); + long tabletId = partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().get(0).getId(); tabletIds.add(tabletId); } dropPartition(String.format(dropPartitionSql, dbName, tableName)); @@ -218,7 +218,7 @@ private void checkForceDropPartitions(String dropPartitionSql) throws Exception for (String partitionName : partitionNames) { Table table = getTable(dbName, tableName); Partition partition = table.getPartition(partitionName); - long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); + long tabletId = partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().get(0).getId(); tabletIds.add(tabletId); } dropPartition(dropPartitionSql); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java b/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java index 6cb7bbc568532..31078eba1b4d0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java @@ -131,13 +131,16 @@ public static boolean compareState(GlobalStateMgr masterGlobalStateMgr, GlobalSt if (masterPartition.getId() != slavePartition.getId()) { return false; } - if (masterPartition.getVisibleVersion() != slavePartition.getVisibleVersion() - || masterPartition.getNextVersion() != slavePartition.getNextVersion()) { + if (masterPartition.getDefaultPhysicalPartition().getVisibleVersion() + != slavePartition.getDefaultPhysicalPartition().getVisibleVersion() + || masterPartition.getDefaultPhysicalPartition().getNextVersion() + != slavePartition.getDefaultPhysicalPartition().getNextVersion()) { return false; } - List allMaterializedIndices = masterPartition.getMaterializedIndices(IndexExtState.ALL); + List allMaterializedIndices = masterPartition.getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex masterIndex : allMaterializedIndices) { - MaterializedIndex slaveIndex = slavePartition.getIndex(masterIndex.getId()); + MaterializedIndex slaveIndex = slavePartition.getDefaultPhysicalPartition().getIndex(masterIndex.getId()); if (slaveIndex == null) { return false; } @@ -191,8 +194,8 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, // partition RandomDistributionInfo distributionInfo = new RandomDistributionInfo(10); Partition partition = new Partition(partitionId, testPartition1, index, distributionInfo); - partition.updateVisibleVersion(testStartVersion); - partition.setNextVersion(testStartVersion + 1); + partition.getDefaultPhysicalPartition().updateVisibleVersion(testStartVersion); + partition.getDefaultPhysicalPartition().setNextVersion(testStartVersion + 1); // columns List columns = new ArrayList(); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/PhysicalPartitionImplTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/PhysicalPartitionImplTest.java index 48738cafbb8ad..9ba302b21271b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/PhysicalPartitionImplTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/PhysicalPartitionImplTest.java @@ -40,7 +40,7 @@ public void setUp() { @Test public void testPhysicalPartition() throws Exception { - PhysicalPartitionImpl p = new PhysicalPartitionImpl(1, "", 1, 0, new MaterializedIndex()); + PhysicalPartition p = new PhysicalPartition(1, "", 1, 0, new MaterializedIndex()); Assert.assertEquals(1, p.getId()); Assert.assertEquals(1, p.getParentId()); Assert.assertEquals(0, p.getShardGroupId()); @@ -107,7 +107,7 @@ public void testPhysicalPartition() throws Exception { Assert.assertTrue(p.equals(p)); Assert.assertFalse(p.equals(new Partition(0, "", null, null))); - PhysicalPartitionImpl p2 = new PhysicalPartitionImpl(1, "", 1, 0, new MaterializedIndex()); + PhysicalPartition p2 = new PhysicalPartition(1, "", 1, 0, new MaterializedIndex()); Assert.assertFalse(p.equals(p2)); p2.setBaseIndex(new MaterializedIndex(1)); @@ -144,8 +144,8 @@ public void testPhysicalPartition() throws Exception { @Test public void testPhysicalPartitionEqual() throws Exception { - PhysicalPartitionImpl p1 = new PhysicalPartitionImpl(1, "", 1, 0, new MaterializedIndex()); - PhysicalPartitionImpl p2 = new PhysicalPartitionImpl(1, "", 1, 0, new MaterializedIndex()); + PhysicalPartition p1 = new PhysicalPartition(1, "", 1, 0, new MaterializedIndex()); + PhysicalPartition p2 = new PhysicalPartition(1, "", 1, 0, new MaterializedIndex()); Assert.assertTrue(p1.equals(p2)); p1.createRollupIndex(new MaterializedIndex()); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/TabletStatMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/TabletStatMgrTest.java index 6720d574f3636..08bb5ae5b941f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/TabletStatMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/TabletStatMgrTest.java @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.catalog; import com.google.common.collect.ImmutableMap; @@ -173,7 +172,6 @@ private LakeTable createLakeTableForTest() { long tablet2Id = 11L; long tablet3Id = 12L; - // Schema List columns = Lists.newArrayList(); Column k1 = new Column("k1", Type.INT, true, null, "", ""); @@ -202,7 +200,7 @@ private LakeTable createLakeTableForTest() { PartitionInfo partitionInfo = new SinglePartitionInfo(); partitionInfo.setReplicationNum(PARTITION_ID, (short) 3); Partition partition = new Partition(PARTITION_ID, "p1", index, distributionInfo); - partition.setVisibleVersion(2L, visibleVersionTime); + partition.getDefaultPhysicalPartition().setVisibleVersion(2L, visibleVersionTime); // Lake table LakeTable table = new LakeTable(TABLE_ID, "t1", columns, KeysType.AGG_KEYS, partitionInfo, distributionInfo); @@ -219,8 +217,10 @@ public void testUpdateLakeTabletStat(@Mocked SystemInfoService systemInfoService LakeTable table = createLakeTableForTest(); - long tablet1Id = table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(0).getId(); - long tablet2Id = table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(1).getId(); + long tablet1Id = + table.getPartition(PARTITION_ID).getDefaultPhysicalPartition().getBaseIndex().getTablets().get(0).getId(); + long tablet2Id = + table.getPartition(PARTITION_ID).getDefaultPhysicalPartition().getBaseIndex().getTablets().get(1).getId(); // db Database db = new Database(DB_ID, "db"); @@ -316,8 +316,10 @@ public TabletStatResponse get(long timeout, @NotNull TimeUnit unit) { Deencapsulation.invoke(tabletStatMgr, "updateLakeTableTabletStat", db, table); long t2 = System.currentTimeMillis(); - LakeTablet tablet1 = (LakeTablet) table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(0); - LakeTablet tablet2 = (LakeTablet) table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(1); + LakeTablet tablet1 = (LakeTablet) table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(0); + LakeTablet tablet2 = (LakeTablet) table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(1); Assert.assertEquals(tablet1.getRowCount(-1), tablet1NumRows); Assert.assertEquals(tablet1.getDataSize(true), tablet1DataSize); @@ -332,8 +334,10 @@ public void testUpdateLakeTabletStat2(@Mocked SystemInfoService systemInfoServic @Mocked LakeService lakeService) { LakeTable table = createLakeTableForTest(); - long tablet1Id = table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(0).getId(); - long tablet2Id = table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(1).getId(); + long tablet1Id = table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(0).getId(); + long tablet2Id = table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(1).getId(); // db Database db = new Database(DB_ID, "db"); @@ -365,8 +369,10 @@ public ComputeNode chooseNode(LakeTablet tablet) { TabletStatMgr tabletStatMgr = new TabletStatMgr(); Deencapsulation.invoke(tabletStatMgr, "updateLakeTableTabletStat", db, table); - LakeTablet tablet1 = (LakeTablet) table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(0); - LakeTablet tablet2 = (LakeTablet) table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(1); + LakeTablet tablet1 = (LakeTablet) table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(0); + LakeTablet tablet2 = (LakeTablet) table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(1); Assert.assertEquals(0, tablet1.getRowCount(-1)); Assert.assertEquals(0, tablet1.getDataSize(true)); @@ -381,8 +387,10 @@ public void testUpdateLakeTabletStat3(@Mocked SystemInfoService systemInfoServic @Mocked LakeService lakeService) { LakeTable table = createLakeTableForTest(); - long tablet1Id = table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(0).getId(); - long tablet2Id = table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(1).getId(); + long tablet1Id = table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(0).getId(); + long tablet2Id = table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(1).getId(); // db Database db = new Database(DB_ID, "db"); @@ -457,8 +465,10 @@ public TabletStatResponse get(long timeout, @NotNull TimeUnit unit) { TabletStatMgr tabletStatMgr = new TabletStatMgr(); Deencapsulation.invoke(tabletStatMgr, "updateLakeTableTabletStat", db, table); - LakeTablet tablet1 = (LakeTablet) table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(0); - LakeTablet tablet2 = (LakeTablet) table.getPartition(PARTITION_ID).getBaseIndex().getTablets().get(1); + LakeTablet tablet1 = (LakeTablet) table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(0); + LakeTablet tablet2 = (LakeTablet) table.getPartition(PARTITION_ID).getDefaultPhysicalPartition() + .getBaseIndex().getTablets().get(1); Assert.assertEquals(0, tablet1.getRowCount(-1)); Assert.assertEquals(0, tablet1.getDataSize(true)); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java index a25ca833f465d..4c03370584f57 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java @@ -780,6 +780,7 @@ private void testSerializeTempPartitions(TempPartitions tempPartitionsInstance) TempPartitions readTempPartition = TempPartitions.read(in); List partitions = readTempPartition.getAllPartitions(); Assert.assertEquals(1, partitions.size()); - Assert.assertEquals(2, partitions.get(0).getMaterializedIndices(IndexExtState.VISIBLE).size()); + Assert.assertEquals(2, partitions.get(0).getDefaultPhysicalPartition() + .getMaterializedIndices(IndexExtState.VISIBLE).size()); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java b/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java index 5687eb8d18e61..7054125042a96 100644 --- a/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java @@ -149,7 +149,7 @@ private void addTabletsToScheduler(String dbName, String tableName, boolean setG // add its tablet to TabletScheduler TabletScheduler tabletScheduler = GlobalStateMgr.getCurrentState().getTabletScheduler(); for (Partition partition : table.getPartitions()) { - MaterializedIndex materializedIndex = partition.getBaseIndex(); + MaterializedIndex materializedIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Tablet tablet : materializedIndex.getTablets()) { TabletSchedCtx ctx = new TabletSchedCtx(TabletSchedCtx.Type.REPAIR, database.getId(), @@ -215,7 +215,7 @@ public void test3RepairWithBadReplica() throws Exception { ColocateTableIndex colocateTableIndex = GlobalStateMgr.getCurrentState().getColocateTableIndex(); List partitions = Lists.newArrayList(table.getPartitions()); - LocalTablet tablet = (LocalTablet) partitions.get(0).getBaseIndex().getTablets().get(0); + LocalTablet tablet = (LocalTablet) partitions.get(0).getDefaultPhysicalPartition().getBaseIndex().getTablets().get(0); tablet.getImmutableReplicas().get(0).setBad(true); ColocateTableBalancer colocateTableBalancer = ColocateTableBalancer.getInstance(); long oldVal = Config.tablet_sched_repair_delay_factor_second; diff --git a/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java b/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java index b7de28181515c..0cc4030ad1ec9 100644 --- a/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java @@ -60,7 +60,7 @@ public void testChooseTablets(@Mocked GlobalStateMgr globalStateMgr) { partitionInfo.addPartition(partitionId, dataProperty, (short) 3, false); DistributionInfo distributionInfo = new HashDistributionInfo(1, Lists.newArrayList()); Partition partition = new Partition(partitionId, "partition", materializedIndex, distributionInfo); - partition.setVisibleVersion(2L, System.currentTimeMillis()); + partition.getDefaultPhysicalPartition().setVisibleVersion(2L, System.currentTimeMillis()); OlapTable table = new OlapTable(tableId, "table", Lists.newArrayList(), KeysType.AGG_KEYS, partitionInfo, distributionInfo); table.addPartition(partition); diff --git a/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java b/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java index fa07ad4e1430b..396c4718790a3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java @@ -73,19 +73,20 @@ public void testRecover() throws Exception { OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() .getTable(database.getFullName(), "tbl_recover"); Partition partition = table.getPartition("tbl_recover"); - MaterializedIndex index = partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).get(0); + MaterializedIndex index = partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).get(0); for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getAllReplicas()) { Assert.assertEquals(2L, replica.getVersion()); } } - Assert.assertEquals(2L, partition.getVisibleVersion()); - Assert.assertEquals(3L, partition.getNextVersion()); + Assert.assertEquals(2L, partition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(3L, partition.getDefaultPhysicalPartition().getNextVersion()); // set partition version to a lower value - partition.setVisibleVersion(1L, System.currentTimeMillis()); - partition.setNextVersion(2L); + partition.getDefaultPhysicalPartition().setVisibleVersion(1L, System.currentTimeMillis()); + partition.getDefaultPhysicalPartition().setNextVersion(2L); for (Backend backend : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackends()) { backend.getBackendStatus().lastSuccessReportTabletsTime = TimeUtils @@ -106,7 +107,7 @@ public void testRecover() throws Exception { // recover will fail, because there is a committed txn on that partition MetaRecoveryDaemon recovery = new MetaRecoveryDaemon(); recovery.recover(); - Assert.assertEquals(1L, partition.getVisibleVersion()); + Assert.assertEquals(1L, partition.getDefaultPhysicalPartition().getVisibleVersion()); BaseProcResult baseProcResult = new BaseProcResult(); recovery.fetchProcNodeResult(baseProcResult); Assert.assertEquals(1, baseProcResult.getRows().size()); @@ -114,33 +115,35 @@ public void testRecover() throws Exception { // change the txn state to visible, recover will succeed transactionState.setTransactionStatus(TransactionStatus.VISIBLE); recovery.recover(); - Assert.assertEquals(2L, partition.getVisibleVersion()); - Assert.assertEquals(3L, partition.getNextVersion()); + Assert.assertEquals(2L, partition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(3L, partition.getDefaultPhysicalPartition().getNextVersion()); baseProcResult = new BaseProcResult(); recovery.fetchProcNodeResult(baseProcResult); Assert.assertEquals(0, baseProcResult.getRows().size()); // change replica version - LocalTablet localTablet = (LocalTablet) partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL) + LocalTablet localTablet = (LocalTablet) partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL) .get(0).getTablets().get(0); long version = 3; for (Replica replica : localTablet.getAllReplicas()) { replica.updateForRestore(++version, 10, 10); } - LocalTablet localTablet2 = (LocalTablet) partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL) + LocalTablet localTablet2 = (LocalTablet) partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL) .get(0).getTablets().get(0); for (Replica replica : localTablet2.getAllReplicas()) { replica.updateForRestore(4, 10, 10); } // set partition version to a lower value - partition.setVisibleVersion(1L, System.currentTimeMillis()); - partition.setNextVersion(2L); + partition.getDefaultPhysicalPartition().setVisibleVersion(1L, System.currentTimeMillis()); + partition.getDefaultPhysicalPartition().setNextVersion(2L); // recover will fail, because there is no common version on tablets. recovery.recover(); - Assert.assertEquals(1L, partition.getVisibleVersion()); - Assert.assertEquals(2L, partition.getNextVersion()); + Assert.assertEquals(1L, partition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2L, partition.getDefaultPhysicalPartition().getNextVersion()); baseProcResult = new BaseProcResult(); recovery.fetchProcNodeResult(baseProcResult); Assert.assertEquals(1, baseProcResult.getRows().size()); @@ -158,8 +161,8 @@ public void testRecover() throws Exception { // recover will succeed recovery.recover(); - Assert.assertEquals(2L, partition.getVisibleVersion()); - Assert.assertEquals(3L, partition.getNextVersion()); + Assert.assertEquals(2L, partition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(3L, partition.getDefaultPhysicalPartition().getNextVersion()); baseProcResult = new BaseProcResult(); recovery.fetchProcNodeResult(baseProcResult); Assert.assertEquals(0, baseProcResult.getRows().size()); diff --git a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java index 1bcc2c1ffe10e..46204508b7982 100644 --- a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java +++ b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java @@ -172,8 +172,8 @@ public static OlapTable newEmptyTable(String name) { // partition HashDistributionInfo distributionInfo = new HashDistributionInfo(10, Lists.newArrayList(k1)); Partition partition = new Partition(testPartitionId, "testPartition", baseIndex, distributionInfo); - partition.updateVisibleVersion(testStartVersion); - partition.setNextVersion(testStartVersion + 1); + partition.getDefaultPhysicalPartition().updateVisibleVersion(testStartVersion); + partition.getDefaultPhysicalPartition().setNextVersion(testStartVersion + 1); // table PartitionInfo partitionInfo = new SinglePartitionInfo(); @@ -225,8 +225,8 @@ public static OlapTable newTable(String name) { // partition HashDistributionInfo distributionInfo = new HashDistributionInfo(10, Lists.newArrayList(k1)); Partition partition = new Partition(testPartitionId, "testPartition", baseIndex, distributionInfo); - partition.updateVisibleVersion(testStartVersion); - partition.setNextVersion(testStartVersion + 1); + partition.getDefaultPhysicalPartition().updateVisibleVersion(testStartVersion); + partition.getDefaultPhysicalPartition().setNextVersion(testStartVersion + 1); // table PartitionInfo partitionInfo = new SinglePartitionInfo(); diff --git a/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java b/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java index bace9e50e5ff4..f7344f5669c93 100644 --- a/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java @@ -369,8 +369,8 @@ private static OlapTable newOlapTable(Long tableId, String tableName, int partit long partitionId = BASE_PARTITION_ID + i; Partition partition = new Partition(partitionId, "testPartition_" + i, baseIndex, distributionInfo); - partition.setVisibleVersion(testStartVersion, System.currentTimeMillis()); - partition.setNextVersion(testStartVersion + 1); + partition.getDefaultPhysicalPartition().setVisibleVersion(testStartVersion, System.currentTimeMillis()); + partition.getDefaultPhysicalPartition().setNextVersion(testStartVersion + 1); PartitionKey rangeLower = PartitionKey.createPartitionKey( Lists.newArrayList(new PartitionValue(String.valueOf(i * 10))), Lists.newArrayList(c1)); @@ -459,8 +459,8 @@ private static LakeTable newLakeTable(Long tableId, String tableName, int partit long partitionId = BASE_PARTITION_ID + i; Partition partition = new Partition(partitionId, "testPartition_" + i, baseIndex, distributionInfo); - partition.setVisibleVersion(testStartVersion, System.currentTimeMillis()); - partition.setNextVersion(testStartVersion + 1); + partition.getDefaultPhysicalPartition().setVisibleVersion(testStartVersion, System.currentTimeMillis()); + partition.getDefaultPhysicalPartition().setNextVersion(testStartVersion + 1); PartitionKey rangeLower = PartitionKey.createPartitionKey( Lists.newArrayList(new PartitionValue(String.valueOf(i * 10))), Lists.newArrayList(c1)); diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionJobTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionJobTest.java index ba64b99792b4e..07e7597314925 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionJobTest.java @@ -16,7 +16,6 @@ import com.starrocks.catalog.Database; import com.starrocks.catalog.PhysicalPartition; -import com.starrocks.catalog.PhysicalPartitionImpl; import com.starrocks.catalog.Table; import mockit.Mock; import mockit.MockUp; @@ -33,7 +32,7 @@ public class CompactionJobTest { public void testGetResult() { Database db = new Database(); Table table = new Table(Table.TableType.CLOUD_NATIVE); - PhysicalPartition partition = new PhysicalPartitionImpl(0, "", 1, 2, null); + PhysicalPartition partition = new PhysicalPartition(0, "", 1, 2, null); CompactionJob job = new CompactionJob(db, table, partition, 10010, true); Assert.assertTrue(job.getAllowPartialSuccess()); @@ -78,7 +77,7 @@ public CompactionTask.TaskResult getResult() { public void testBuildTabletCommitInfo() { Database db = new Database(); Table table = new Table(Table.TableType.CLOUD_NATIVE); - PhysicalPartition partition = new PhysicalPartitionImpl(0, "", 1, 2, null); + PhysicalPartition partition = new PhysicalPartition(0, "", 1, 2, null); CompactionJob job = new CompactionJob(db, table, partition, 10010, false); assertDoesNotThrow(() -> { job.buildTabletCommitInfo(); diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java index 357a1653774ba..3d59a41d3a17e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java @@ -16,7 +16,6 @@ import com.google.common.collect.Lists; import com.starrocks.catalog.Database; -import com.starrocks.catalog.Partition; import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Table; import com.starrocks.common.Config; @@ -143,7 +142,7 @@ public ConcurrentHashMap getRunningCompactio PartitionIdentifier partitionIdentifier = new PartitionIdentifier(1, 2, 3); Database db = new Database(); Table table = new LakeTable(); - PhysicalPartition partition = new Partition(123, "aaa", null, null); + PhysicalPartition partition = new PhysicalPartition(123, "aaa", 123, 0, null); CompactionJob job = new CompactionJob(db, table, partition, txnId, false); r.put(partitionIdentifier, job); return r; diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java index 06c8e2e261526..486111cc8a819 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java @@ -17,7 +17,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.starrocks.catalog.Database; -import com.starrocks.catalog.Partition; import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Table; import com.starrocks.common.Config; @@ -169,8 +168,8 @@ public ConcurrentHashMap getRunningCompactio Table table = new LakeTable(); PartitionIdentifier partitionIdentifier1 = new PartitionIdentifier(1, 2, 3); PartitionIdentifier partitionIdentifier2 = new PartitionIdentifier(1, 2, 4); - PhysicalPartition partition1 = new Partition(123, "aaa", null, null); - PhysicalPartition partition2 = new Partition(124, "bbb", null, null); + PhysicalPartition partition1 = new PhysicalPartition(123, "aaa", 123, 0, null); + PhysicalPartition partition2 = new PhysicalPartition(124, "bbb", 124, 0, null); CompactionJob job1 = new CompactionJob(db, table, partition1, 100, false); try { Thread.sleep(10); diff --git a/fe/fe-core/src/test/java/com/starrocks/leader/LeaderImplTest.java b/fe/fe-core/src/test/java/com/starrocks/leader/LeaderImplTest.java index 78767f2455c65..0a28a075afe64 100644 --- a/fe/fe-core/src/test/java/com/starrocks/leader/LeaderImplTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/leader/LeaderImplTest.java @@ -71,7 +71,7 @@ public void testFindRelatedReplica(@Mocked OlapTable olapTable, @Mocked LakeTabl // olap table new Expectations() { { - partition.getIndex(indexId); + partition.getDefaultPhysicalPartition().getIndex(indexId); result = index; index.getTablet(tabletId); result = new LocalTablet(tabletId); @@ -90,7 +90,7 @@ public Set getBackendIds() { new Expectations() { { - partition.getIndex(indexId); + partition.getDefaultPhysicalPartition().getIndex(indexId); result = index; index.getTablet(tabletId); result = new LakeTablet(tabletId); diff --git a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java index 5785533027497..f1fe88bda4b0f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java @@ -393,7 +393,7 @@ public void testHandleMigration() throws TException { } Partition partition = table.getPartition(tabletMeta.getPartitionId()); - MaterializedIndex idx = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex idx = partition.getDefaultPhysicalPartition().getIndex(tabletMeta.getIndexId()); LocalTablet tablet = (LocalTablet) idx.getTablet(tabletId); for (Replica replica : tablet.getImmutableReplicas()) { @@ -421,7 +421,7 @@ public void submit(AgentBatchTask task) { ListMultimap tabletMetaMigrationMap = ArrayListMultimap.create(); List allTablets = new ArrayList<>(); for (MaterializedIndex index : olapTable.getPartition("binlog_report_handler_test") - .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + .getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { tabletMetaMigrationMap.put(TStorageMedium.HDD, tablet.getId()); allTablets.add(tablet.getId()); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java index c0c5c5ce41285..5ddc6a75c79f0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java @@ -382,7 +382,7 @@ public void testUpdateEtlStatusFinishedAndCommitTransaction( result = partitionInfo; table.getSchemaByIndexId(Long.valueOf(12)); result = Lists.newArrayList(new Column("k1", Type.VARCHAR)); - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL); + partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL); result = Lists.newArrayList(index); index.getId(); result = indexId; @@ -480,7 +480,7 @@ public void testUpdateEtlStatusFinishedAndCommitTransactionForLake( result = partitionInfo; table.getSchemaByIndexId(Long.valueOf(12)); result = Lists.newArrayList(new Column("k1", Type.VARCHAR)); - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL); + partition.getDefaultPhysicalPartition().getMaterializedIndices(MaterializedIndex.IndexExtState.ALL); result = Lists.newArrayList(index); index.getId(); result = indexId; diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest.java index 32385a3539cc7..931be92193606 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest.java @@ -26,6 +26,7 @@ import com.starrocks.analysis.TupleDescriptor; import com.starrocks.catalog.AggregateType; import com.starrocks.catalog.Column; +import com.starrocks.catalog.ColumnId; import com.starrocks.catalog.DataProperty; import com.starrocks.catalog.DistributionInfo; import com.starrocks.catalog.HashDistributionInfo; @@ -38,8 +39,7 @@ import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.PartitionKey; import com.starrocks.catalog.PartitionType; -import com.starrocks.catalog.ColumnId; -import com.starrocks.catalog.PhysicalPartitionImpl; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.RandomDistributionInfo; import com.starrocks.catalog.RangePartitionInfo; import com.starrocks.catalog.Replica; @@ -465,10 +465,10 @@ public void testImmutablePartition() throws UserException { RandomDistributionInfo distInfo = new RandomDistributionInfo(3); Partition partition = new Partition(2, "p1", index, distInfo); - PhysicalPartitionImpl physicalPartition = new PhysicalPartitionImpl(3, "", 2, 0, index); + PhysicalPartition physicalPartition = new PhysicalPartition(3, "", 2, 0, index); partition.addSubPartition(physicalPartition); - physicalPartition = new PhysicalPartitionImpl(4, "", 2, 0, index); + physicalPartition = new PhysicalPartition(4, "", 2, 0, index); physicalPartition.setImmutable(true); partition.addSubPartition(physicalPartition); @@ -505,10 +505,10 @@ public void testInitialOpenPartition() throws UserException { RandomDistributionInfo distInfo = new RandomDistributionInfo(3); Partition partition = new Partition(2, "p1", index, distInfo); - PhysicalPartitionImpl physicalPartition = new PhysicalPartitionImpl(3, "", 2, 0, index); + PhysicalPartition physicalPartition = new PhysicalPartition(3, "", 2, 0, index); partition.addSubPartition(physicalPartition); - physicalPartition = new PhysicalPartitionImpl(4, "", 2, 0, index); + physicalPartition = new PhysicalPartition(4, "", 2, 0, index); physicalPartition.setImmutable(true); partition.addSubPartition(physicalPartition); diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java index 5287a5dbea28e..9d617dc257cd2 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java @@ -55,7 +55,7 @@ public static void beforeClass() throws Exception { private MVRepairHandler.PartitionRepairInfo toPartitionInfo(Partition partition, long version, long versionTime) { return new MVRepairHandler.PartitionRepairInfo(partition.getId(), partition.getName(), - partition.getVisibleVersion(), version, versionTime); + partition.getDefaultPhysicalPartition().getVisibleVersion(), version, versionTime); } @Test @@ -205,8 +205,9 @@ public void testRepairBaseTableVersionChanges3() { String baseTablePartitionName = value.keySet().iterator().next(); MaterializedView.BasePartitionInfo basePartitionInfo = value.get(baseTablePartitionName); Partition p1 = m1.getPartition("p1"); - Assert.assertEquals(basePartitionInfo.getVersion(), p1.getVisibleVersion()); - Assert.assertEquals(basePartitionInfo.getLastRefreshTime(), p1.getVisibleVersionTime()); + Assert.assertEquals(basePartitionInfo.getVersion(), p1.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(basePartitionInfo.getLastRefreshTime(), + p1.getDefaultPhysicalPartition().getVisibleVersionTime()); Partition p2 = m1.getPartition("p2"); long currentTs = System.currentTimeMillis(); @@ -269,8 +270,8 @@ public void testRepairBaseTableVersionChanges4() { String baseTablePartitionName = value.keySet().iterator().next(); MaterializedView.BasePartitionInfo basePartitionInfo = value.get(baseTablePartitionName); Partition p1 = m1.getPartition("p1"); - long lastRefreshVersion = p1.getVisibleVersion(); - long lastRefreshVersionTime = p1.getVisibleVersionTime(); + long lastRefreshVersion = p1.getDefaultPhysicalPartition().getVisibleVersion(); + long lastRefreshVersionTime = p1.getDefaultPhysicalPartition().getVisibleVersionTime(); Assert.assertEquals(basePartitionInfo.getVersion(), lastRefreshVersion); Assert.assertEquals(basePartitionInfo.getLastRefreshTime(), lastRefreshVersionTime); @@ -278,7 +279,8 @@ public void testRepairBaseTableVersionChanges4() { // p1 has been refreshed, but p2 has been compaction or fast schema changed, use curPartition as its // partition // p1 has been updated, so the version of p1 should be updated - p1.setVisibleVersion(lastRefreshVersion + 1, lastRefreshVersionTime + 1); + p1.getDefaultPhysicalPartition() + .setVisibleVersion(lastRefreshVersion + 1, lastRefreshVersionTime + 1); MVRepairHandler.PartitionRepairInfo partitionRepairInfo = toPartitionInfo(p1, 100L, currentTs); diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java index eeecf69483ac4..ad28ac13ca933 100644 --- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java @@ -160,7 +160,7 @@ public void test1BestEffortBalance() throws SQLException, InterruptedException { private void printTabletReplicaInfo(OlapTable table) { table.getPartitions().forEach(partition -> { - partition.getBaseIndex().getTablets().forEach(tablet -> { + partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().forEach(tablet -> { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append("tablet ").append(tablet.getId()).append(": "); for (Replica replica : tablet.getAllReplicas()) { diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java index 77652e2b487a7..aa1c3b99f14fb 100644 --- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java @@ -107,7 +107,7 @@ private Set getBackendIdsWithLocProp(String locationKey, String locationVa private void printTabletReplicaInfo(OlapTable table) { table.getPartitions().forEach(partition -> { - partition.getBaseIndex().getTablets().forEach(tablet -> { + partition.getDefaultPhysicalPartition().getBaseIndex().getTablets().forEach(tablet -> { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append("tablet ").append(tablet.getId()).append(": "); for (Replica replica : tablet.getAllReplicas()) { diff --git a/fe/fe-core/src/test/java/com/starrocks/qe/CoordinatorTest.java b/fe/fe-core/src/test/java/com/starrocks/qe/CoordinatorTest.java index 3929ec4960022..88ffb5333d4f0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/qe/CoordinatorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/qe/CoordinatorTest.java @@ -138,7 +138,8 @@ public void testBinlogScan() throws Exception { OlapTable olapTable = getOlapTable("t0"); List olapTableTabletIds = - olapTable.getAllPartitions().stream().flatMap(x -> x.getBaseIndex().getTabletIdsInOrder().stream()) + olapTable.getAllPartitions().stream().flatMap(x -> x.getDefaultPhysicalPartition().getBaseIndex() + .getTabletIdsInOrder().stream()) .collect(Collectors.toList()); Assert.assertFalse(olapTableTabletIds.isEmpty()); tupleDesc.setTable(olapTable); diff --git a/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java index b842143bc7161..9413eda080d23 100644 --- a/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java @@ -192,7 +192,7 @@ public void setUp() throws Exception { Partition partition = Deencapsulation.newInstance(Partition.class); new Expectations(partition) { { - partition.getBaseIndex(); + partition.getDefaultPhysicalPartition().getBaseIndex(); minTimes = 0; result = index1; } diff --git a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java index 7d31dda4894c5..1237da40506a5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java @@ -98,12 +98,12 @@ public void submit(AgentBatchTask task) { @Before public void setUp() throws Exception { - partition.updateVersionForRestore(10); - srcPartition.updateVersionForRestore(100); - partition.setDataVersion(8); - partition.setNextDataVersion(9); - srcPartition.setDataVersion(98); - srcPartition.setNextDataVersion(99); + partition.getDefaultPhysicalPartition().updateVersionForRestore(10); + srcPartition.getDefaultPhysicalPartition().updateVersionForRestore(100); + partition.getDefaultPhysicalPartition().setDataVersion(8); + partition.getDefaultPhysicalPartition().setNextDataVersion(9); + srcPartition.getDefaultPhysicalPartition().setDataVersion(98); + srcPartition.getDefaultPhysicalPartition().setNextDataVersion(99); job = new ReplicationJob(null, "test_token", db.getId(), table, srcTable, GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); @@ -163,8 +163,10 @@ public void testNormal() throws Exception { job.run(); Assert.assertEquals(ReplicationJobState.COMMITTED, job.getState()); - Assert.assertEquals(partition.getCommittedVersion(), srcPartition.getVisibleVersion()); - Assert.assertEquals(partition.getCommittedDataVersion(), srcPartition.getDataVersion()); + Assert.assertEquals(partition.getDefaultPhysicalPartition().getCommittedVersion(), + srcPartition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(partition.getDefaultPhysicalPartition().getCommittedDataVersion(), + srcPartition.getDefaultPhysicalPartition().getDataVersion()); } @Test @@ -321,14 +323,14 @@ public void testInitializedByThrift() { Partition partition = table.getPartitions().iterator().next(); Partition srcPartition = srcTable.getPartitions().iterator().next(); partitionInfo.partition_id = partition.getId(); - partitionInfo.src_version = srcPartition.getVisibleVersion(); - partitionInfo.src_version_epoch = srcPartition.getVersionEpoch(); + partitionInfo.src_version = srcPartition.getDefaultPhysicalPartition().getVisibleVersion(); + partitionInfo.src_version_epoch = srcPartition.getDefaultPhysicalPartition().getVersionEpoch(); request.partition_replication_infos.put(partitionInfo.partition_id, partitionInfo); partitionInfo.index_replication_infos = new HashMap(); TIndexReplicationInfo indexInfo = new TIndexReplicationInfo(); - MaterializedIndex index = partition.getBaseIndex(); - MaterializedIndex srcIndex = srcPartition.getBaseIndex(); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getBaseIndex(); + MaterializedIndex srcIndex = srcPartition.getDefaultPhysicalPartition().getBaseIndex(); indexInfo.index_id = index.getId(); indexInfo.src_schema_hash = srcTable.getSchemaHashByIndexId(srcIndex.getId()); partitionInfo.index_replication_infos.put(indexInfo.index_id, indexInfo); diff --git a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java index fe327de3521f4..bb2d860b7a376 100644 --- a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java @@ -100,12 +100,12 @@ public void submit(AgentBatchTask task) { @Before public void setUp() throws Exception { - partition.updateVersionForRestore(10); - srcPartition.updateVersionForRestore(100); - partition.setDataVersion(8); - partition.setNextDataVersion(9); - srcPartition.setDataVersion(98); - srcPartition.setNextDataVersion(99); + partition.getDefaultPhysicalPartition().updateVersionForRestore(10); + srcPartition.getDefaultPhysicalPartition().updateVersionForRestore(100); + partition.getDefaultPhysicalPartition().setDataVersion(8); + partition.getDefaultPhysicalPartition().setNextDataVersion(9); + srcPartition.getDefaultPhysicalPartition().setDataVersion(98); + srcPartition.getDefaultPhysicalPartition().setNextDataVersion(99); job = new ReplicationJob(null, "test_token", db.getId(), table, srcTable, GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); @@ -161,8 +161,10 @@ public void testNormal() throws Exception { replicationMgr.runAfterCatalogReady(); Assert.assertEquals(ReplicationJobState.COMMITTED, job.getState()); - Assert.assertEquals(partition.getCommittedVersion(), srcPartition.getVisibleVersion()); - Assert.assertEquals(partition.getCommittedDataVersion(), srcPartition.getDataVersion()); + Assert.assertEquals(partition.getDefaultPhysicalPartition().getCommittedVersion(), + srcPartition.getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(partition.getDefaultPhysicalPartition().getCommittedDataVersion(), + srcPartition.getDefaultPhysicalPartition().getDataVersion()); replicationMgr.replayReplicationJob(job); } @@ -327,14 +329,14 @@ public void testInitializedByThrift() { Partition partition = table.getPartitions().iterator().next(); Partition srcPartition = srcTable.getPartitions().iterator().next(); partitionInfo.partition_id = partition.getId(); - partitionInfo.src_version = srcPartition.getVisibleVersion(); - partitionInfo.src_version_epoch = srcPartition.getVersionEpoch(); + partitionInfo.src_version = srcPartition.getDefaultPhysicalPartition().getVisibleVersion(); + partitionInfo.src_version_epoch = srcPartition.getDefaultPhysicalPartition().getVersionEpoch(); request.partition_replication_infos.put(partitionInfo.partition_id, partitionInfo); partitionInfo.index_replication_infos = new HashMap(); TIndexReplicationInfo indexInfo = new TIndexReplicationInfo(); - MaterializedIndex index = partition.getBaseIndex(); - MaterializedIndex srcIndex = srcPartition.getBaseIndex(); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getBaseIndex(); + MaterializedIndex srcIndex = srcPartition.getDefaultPhysicalPartition().getBaseIndex(); indexInfo.index_id = index.getId(); indexInfo.src_schema_hash = srcTable.getSchemaHashByIndexId(srcIndex.getId()); partitionInfo.index_replication_infos.put(indexInfo.index_id, indexInfo); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java index 485895906e84c..afe911ac1cf72 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java @@ -161,11 +161,16 @@ public void testAutoRefreshPartitionLimitWithHiveTable() throws Exception { Collection partitions = materializedView.getPartitions(); Assert.assertEquals(6, partitions.size()); - Assert.assertEquals(1, materializedView.getPartition("p19980101").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p19980102").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p19980103").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980104").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980105").getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p19980101").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p19980102").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p19980103").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980104").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980105").getDefaultPhysicalPartition() + .getVisibleVersion()); MockedHiveMetadata mockedHiveMetadata = (MockedHiveMetadata) connectContext.getGlobalStateMgr().getMetadataMgr(). @@ -173,22 +178,32 @@ public void testAutoRefreshPartitionLimitWithHiveTable() throws Exception { mockedHiveMetadata.updatePartitions("partitioned_db", "lineitem_par", ImmutableList.of("l_shipdate=1998-01-02", "l_shipdate=1998-01-03")); initAndExecuteTaskRun(taskRun); - Assert.assertEquals(1, materializedView.getPartition("p19980101").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p19980102").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p19980103").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980104").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980105").getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p19980101").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p19980102").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p19980103").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980104").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980105").getDefaultPhysicalPartition() + .getVisibleVersion()); task.setType(Constants.TaskType.MANUAL); taskRun = TaskRunBuilder.newBuilder(task).build(); initAndExecuteTaskRun(taskRun); Assert.assertEquals(6, partitions.size()); - Assert.assertEquals(2, materializedView.getPartition("p19980101").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980102").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980103").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980104").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980105").getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980101").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980102").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980103").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980104").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980105").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_parttbl_mv1"); } @@ -403,7 +418,8 @@ public void testAutoPartitionRefreshWithUnPartitionedHiveTable() throws Exceptio Collection partitions = materializedView.getPartitions(); Assert.assertEquals(1, partitions.size()); - Assert.assertEquals(3, materializedView.getPartition("hive_tbl_mv1").getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("hive_tbl_mv1").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_tbl_mv1"); } @@ -451,11 +467,16 @@ public void testAutoPartitionRefreshWithPartitionedHiveTable1() throws Exception Collection partitions = materializedView.getPartitions(); Assert.assertEquals(6, partitions.size()); - Assert.assertEquals(2, materializedView.getPartition("p19980101").getVisibleVersion()); - Assert.assertEquals(3, materializedView.getPartition("p19980102").getVisibleVersion()); - Assert.assertEquals(3, materializedView.getPartition("p19980103").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980104").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980105").getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980101").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("p19980102").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("p19980103").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980104").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980105").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_parttbl_mv1"); } @@ -499,7 +520,8 @@ public void testAutoPartitionRefreshWithPartitionedHiveTable2() throws Exception Collection partitions = materializedView.getPartitions(); Assert.assertEquals(1, partitions.size()); - Assert.assertEquals(3, materializedView.getPartition("hive_tbl_mv2").getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("hive_tbl_mv2").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_tbl_mv2"); } @@ -545,7 +567,8 @@ public void testAutoPartitionRefreshWithPartitionedHiveTableJoinInternalTable() Collection partitions = materializedView.getPartitions(); Assert.assertEquals(1, partitions.size()); - Assert.assertEquals(3, materializedView.getPartition("hive_join_internal_mv").getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("hive_join_internal_mv").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_join_internal_mv"); } @@ -593,11 +616,16 @@ public void testPartitionRefreshWithUpperCaseTable() throws Exception { Collection partitions = materializedView.getPartitions(); Assert.assertEquals(6, partitions.size()); - Assert.assertEquals(2, materializedView.getPartition("p19980101").getVisibleVersion()); - Assert.assertEquals(3, materializedView.getPartition("p19980102").getVisibleVersion()); - Assert.assertEquals(3, materializedView.getPartition("p19980103").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980104").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980105").getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980101").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("p19980102").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("p19980103").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980104").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980105").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_parttbl_mv1"); } @@ -644,11 +672,16 @@ public void testPartitionRefreshWithUpperCaseDb() throws Exception { Collection partitions = materializedView.getPartitions(); Assert.assertEquals(6, partitions.size()); - Assert.assertEquals(2, materializedView.getPartition("p19980101").getVisibleVersion()); - Assert.assertEquals(3, materializedView.getPartition("p19980102").getVisibleVersion()); - Assert.assertEquals(3, materializedView.getPartition("p19980103").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980104").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980105").getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980101").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("p19980102").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("p19980103").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980104").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980105").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_parttbl_mv1"); } @@ -694,9 +727,12 @@ public void testPartitionRefreshWithLowerCase() throws Exception { Collection partitions = materializedView.getPartitions(); Assert.assertEquals(3, partitions.size()); - Assert.assertEquals(3, materializedView.getPartition("p0").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p1").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p2").getVisibleVersion()); + Assert.assertEquals(3, materializedView.getPartition("p0").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p1").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p2").getDefaultPhysicalPartition() + .getVisibleVersion()); starRocksAssert.useDatabase("test").dropMaterializedView("hive_parttbl_mv1"); } @@ -716,8 +752,10 @@ public void testRangePartitionRefreshWithHiveTable() throws Exception { Collection partitions = materializedView.getPartitions(); Assert.assertEquals(2, partitions.size()); - Assert.assertEquals(2, materializedView.getPartition("p19980101").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p19980102").getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980101").getDefaultPhysicalPartition() + .getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p19980102").getDefaultPhysicalPartition() + .getVisibleVersion()); PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java index 5faed2b0f4406..a6ba215cf80ab 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java @@ -36,6 +36,7 @@ import java.time.Instant; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -165,8 +166,11 @@ public void testCreatePartitionedMVForIceberg() throws Exception { ExecPlan execPlan = mvContext.getExecPlan(); assertPlanContains(execPlan, "3: date >= '2020-01-02', 3: date < '2020-01-03'"); - Map partitionVersionMap = partitionedMaterializedView.getPartitions().stream() - .collect(Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); + Map partitionVersionMap = new HashMap<>(); + for (Partition p : partitionedMaterializedView.getPartitions()) { + partitionVersionMap.put(p.getName(), p.getDefaultPhysicalPartition().getVisibleVersion()); + } + Assert.assertEquals( ImmutableMap.of("p20200104_20200105", 2L, "p20200101_20200102", 2L, diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java index 1e2118d3c9e41..568aafad15040 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java @@ -247,8 +247,10 @@ public void testStr2Date_DateTrunc() throws Exception { // partial range refresh 1 { - Map partitionVersionMap = materializedView.getPartitions().stream().collect( - Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); + Map partitionVersionMap = new HashMap<>(); + for (Partition p : materializedView.getPartitions()) { + partitionVersionMap.put(p.getName(), p.getDefaultPhysicalPartition().getVisibleVersion()); + } starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-02') end('2023-09-01')" + "force with sync mode"); @@ -257,13 +259,16 @@ public void testStr2Date_DateTrunc() throws Exception { .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p000101_202308", "p202308_202309"), partitions); Assert.assertEquals(partitionVersionMap.get("p202308_202309").longValue(), - materializedView.getPartition("p202308_202309").getVisibleVersion()); + materializedView.getPartition("p202308_202309") + .getDefaultPhysicalPartition().getVisibleVersion()); } // partial range refresh 2 { - Map partitionVersionMap = materializedView.getPartitions().stream().collect( - Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); + Map partitionVersionMap = new HashMap<>(); + for (Partition p : materializedView.getPartitions()) { + partitionVersionMap.put(p.getName(), p.getDefaultPhysicalPartition().getVisibleVersion()); + } starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-07-01') end('2023-08-01')" + "force with sync mode"); @@ -272,7 +277,8 @@ public void testStr2Date_DateTrunc() throws Exception { .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p000101_202308", "p202308_202309"), partitions); Assert.assertEquals(partitionVersionMap.get("p202308_202309").longValue(), - materializedView.getPartition("p202308_202309").getVisibleVersion()); + materializedView.getPartition("p202308_202309").getDefaultPhysicalPartition() + .getVisibleVersion()); } starRocksAssert.dropMaterializedView(mvName); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java index 5ab27a72662c2..730702e3f7a14 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java @@ -272,8 +272,9 @@ public void testRefreshWithCachePartitionTraits() throws Exception { executeInsertSql(connectContext, "insert into tbl1 values(\"2022-02-20\", 2, 10)"); Partition p2 = table.getPartition("p2"); - while (p2.getVisibleVersion() != 3) { - System.out.println("waiting for partition p2 to be visible:" + p2.getVisibleVersion()); + while (p2.getDefaultPhysicalPartition().getVisibleVersion() != 3) { + System.out.println("waiting for partition p2 to be visible:" + + p2.getDefaultPhysicalPartition().getVisibleVersion()); Thread.sleep(1000); } MvUpdateInfo mvUpdateInfo = getMvUpdateInfo(mv); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java index 249275dd93e63..a8bcf084ec01c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java @@ -375,18 +375,28 @@ public void testRangePartitionRefresh() throws Exception { executeInsertSql(connectContext, insertSql); refreshMVRange(materializedView.getName(), null, null, false); - Assert.assertEquals(1, materializedView.getPartition("p202112_202201").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202201_202202").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p202202_202203").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p202203_202204").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202204_202205").getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202112_202201") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202201_202202") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202202_202203") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202203_202204") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202204_202205") + .getDefaultPhysicalPartition().getVisibleVersion()); refreshMVRange(materializedView.getName(), "2021-12-03", "2022-04-05", false); - Assert.assertEquals(1, materializedView.getPartition("p202112_202201").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202201_202202").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p202202_202203").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p202203_202204").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202204_202205").getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202112_202201") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202201_202202") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202202_202203") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202203_202204") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202204_202205") + .getDefaultPhysicalPartition().getVisibleVersion()); insertSql = "insert into tbl4 partition(p3) values('2022-03-02',21,102);"; executeInsertSql(connectContext, insertSql); @@ -394,18 +404,28 @@ public void testRangePartitionRefresh() throws Exception { executeInsertSql(connectContext, insertSql); refreshMVRange(materializedView.getName(), "2021-12-03", "2022-03-01", false); - Assert.assertEquals(2, materializedView.getPartition("p202112_202201").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202201_202202").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p202202_202203").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p202203_202204").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202204_202205").getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202112_202201") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202201_202202") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202202_202203") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202203_202204") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202204_202205") + .getDefaultPhysicalPartition().getVisibleVersion()); refreshMVRange(materializedView.getName(), "2021-12-03", "2022-05-06", true); - Assert.assertEquals(2, materializedView.getPartition("p202112_202201").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202201_202202").getVisibleVersion()); - Assert.assertEquals(1, materializedView.getPartition("p202202_202203").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202203_202204").getVisibleVersion()); - Assert.assertEquals(2, materializedView.getPartition("p202204_202205").getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202112_202201") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202201_202202") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(1, materializedView.getPartition("p202202_202203") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202203_202204") + .getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, materializedView.getPartition("p202204_202205") + .getDefaultPhysicalPartition().getVisibleVersion()); } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java b/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java index 8e9712043cf28..a103c7eb15078 100644 --- a/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java @@ -120,7 +120,8 @@ public void testConcurrentCreatingColocateTables() throws InterruptedException { // check all created colocate tables has same tablet distribution as the bucket seq in colocate group for (long threadId : threadIds) { table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_tbl_" + threadId); - List tablets = table.getPartitions().stream().findFirst().get().getBaseIndex().getTabletIdsInOrder(); + List tablets = table.getPartitions().stream().findFirst().get().getDefaultPhysicalPartition() + .getBaseIndex().getTabletIdsInOrder(); List backendIdList = tablets.stream() .map(id -> GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getReplicasByTabletId(id)) .map(replicaList -> replicaList.get(0).getBackendId()) diff --git a/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java b/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java index ca7fb572536f7..1b02a6d09b8c3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java @@ -25,7 +25,7 @@ import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; import com.starrocks.catalog.PartitionInfo; -import com.starrocks.catalog.PhysicalPartitionImpl; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Table; import com.starrocks.catalog.TabletMeta; import com.starrocks.catalog.system.SystemId; @@ -177,14 +177,14 @@ public void testLoadClusterV2() throws Exception { public void testReplayAddSubPartition() throws DdlException { Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); - Partition p = table.getPartitions().stream().findFirst().get(); + PhysicalPartition p = table.getPartitions().stream().findFirst().get().getDefaultPhysicalPartition(); int schemaHash = table.getSchemaHashByIndexId(p.getBaseIndex().getId()); MaterializedIndex index = new MaterializedIndex(); TabletMeta tabletMeta = new TabletMeta(db.getId(), table.getId(), p.getId(), index.getId(), schemaHash, table.getPartitionInfo().getDataProperty(p.getId()).getStorageMedium()); index.addTablet(new LocalTablet(0), tabletMeta); PhysicalPartitionPersistInfoV2 info = new PhysicalPartitionPersistInfoV2( - db.getId(), table.getId(), p.getId(), new PhysicalPartitionImpl(123, "", p.getId(), 0, index)); + db.getId(), table.getId(), p.getId(), new PhysicalPartition(123, "", p.getId(), 0, index)); LocalMetastore localMetastore = connectContext.getGlobalStateMgr().getLocalMetastore(); localMetastore.replayAddSubPartition(info); diff --git a/fe/fe-core/src/test/java/com/starrocks/server/WarehouseManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/server/WarehouseManagerTest.java index d645c61449610..ba516a72f1996 100644 --- a/fe/fe-core/src/test/java/com/starrocks/server/WarehouseManagerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/server/WarehouseManagerTest.java @@ -279,7 +279,8 @@ public RunMode getCurrentRunMode() { Partition partition = new Partition(123, "aaa", null, null); MaterializedIndex index = new MaterializedIndex(1, MaterializedIndex.IndexState.NORMAL); ErrorReportException ex = Assert.assertThrows(ErrorReportException.class, - () -> scanNode.addScanRangeLocations(partition, partition, index, Collections.emptyList(), 1)); + () -> scanNode.addScanRangeLocations(partition, partition.getDefaultPhysicalPartition(), + index, Collections.emptyList(), 1)); Assert.assertEquals("No alive backend or compute node in warehouse null.", ex.getMessage()); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java index 7f783507de7ac..1718ce638b061 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java @@ -323,7 +323,7 @@ public void testHistogramSampleRatio() { OlapTable t0 = (OlapTable) starRocksAssert.getCtx().getGlobalStateMgr() .getLocalMetastore().getDb("db").getTable("tbl"); for (Partition partition : t0.getAllPartitions()) { - partition.getBaseIndex().setRowCount(10000); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(10000); } String sql = "analyze table db.tbl update histogram on kk1 with 256 buckets " + @@ -332,7 +332,7 @@ public void testHistogramSampleRatio() { Assert.assertEquals("1", analyzeStmt.getProperties().get(StatsConstants.HISTOGRAM_SAMPLE_RATIO)); for (Partition partition : t0.getAllPartitions()) { - partition.getBaseIndex().setRowCount(400000); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(400000); } sql = "analyze table db.tbl update histogram on kk1 with 256 buckets " + @@ -341,7 +341,7 @@ public void testHistogramSampleRatio() { Assert.assertEquals("0.5", analyzeStmt.getProperties().get(StatsConstants.HISTOGRAM_SAMPLE_RATIO)); for (Partition partition : t0.getAllPartitions()) { - partition.getBaseIndex().setRowCount(20000000); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(20000000); } sql = "analyze table db.tbl update histogram on kk1 with 256 buckets " + "properties(\"histogram_sample_ratio\"=\"0.9\")"; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java index 6697579d58ec6..791625d15c933 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java @@ -84,7 +84,7 @@ public class UtilsTest { protected static void setTableStatistics(OlapTable table, long rowCount) { for (Partition partition : table.getAllPartitions()) { - partition.getBaseIndex().setRowCount(rowCount); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(rowCount); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java index c2a3518b294e0..b3e3572168359 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java @@ -171,7 +171,7 @@ public void transform(@Mocked OlapTable olapTable, @Mocked Partition partition, partition.getSubPartitions(); result = Arrays.asList(partition); - partition.getIndex(anyLong); + partition.getDefaultPhysicalPartition().getIndex(anyLong); result = index; partition.getDistributionInfo(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePartialPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePartialPartitionTest.java index f053c9e53ea6d..22567c2be125c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePartialPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePartialPartitionTest.java @@ -740,7 +740,7 @@ public void testPartitionQueryRewriteSkipEmptyPartitions() throws Exception { " as" + " select id_date, sum(t1b) from table_with_day_partition group by id_date"); MaterializedView mv = starRocksAssert.getMv("test", "test_loose_mv"); - mv.getPartition("p19910330") + mv.getPartition("p19910330").getDefaultPhysicalPartition() .setVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis()); String query5 = "select id_date, sum(t1b) from table_with_day_partition" + " where id_date >= '1991-03-30' and id_date < '1991-04-03' group by id_date"; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java index 20132fc74f767..bff5285a9f757 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java @@ -2079,7 +2079,7 @@ public void testCandidateOrdering_HierarchyAgg() throws Exception { MaterializedView mv = starRocksAssert.getMv("test", name); int mockRows = i + 1; - mv.getPartitions().forEach(p -> p.getBaseIndex().setRowCount(mockRows)); + mv.getPartitions().forEach(p -> p.getDefaultPhysicalPartition().getBaseIndex().setRowCount(mockRows)); } for (int i = 0; i < dimensions.size(); i++) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java index 7aa6087e3bfb2..987cc90772f50 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java @@ -244,7 +244,7 @@ public void testLogicalOlapTableScan() throws Exception { List partitionIds = partitions.stream().mapToLong(partition -> partition.getId()).boxed().collect(Collectors.toList()); for (Partition partition : partitions) { - partition.getBaseIndex().setRowCount(1000); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(1000); } List columns = table.getColumns(); @@ -349,9 +349,9 @@ public void testLogicalOlapTableEmptyPartition(@Mocked CachedStatisticStorage ca Partition partition2 = partitions.get(1); Partition partition3 = partitions.get(2); // mock one empty partition - partition1.setVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis()); - partition2.setVisibleVersion(2, System.currentTimeMillis()); - partition3.setVisibleVersion(2, System.currentTimeMillis()); + partition1.getDefaultPhysicalPartition().setVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis()); + partition2.getDefaultPhysicalPartition().setVisibleVersion(2, System.currentTimeMillis()); + partition3.getDefaultPhysicalPartition().setVisibleVersion(2, System.currentTimeMillis()); List partitionIds = partitions.stream().filter(partition -> !(partition.getName().equalsIgnoreCase("p1"))). mapToLong(Partition::getId).boxed().collect(Collectors.toList()); @@ -422,7 +422,7 @@ public void testLogicalOlapTableScanPartitionPrune1(@Mocked CachedStatisticStora List partitionIds = partitions.stream().filter(partition -> partition.getName().equalsIgnoreCase("p1")). mapToLong(Partition::getId).boxed().collect(Collectors.toList()); for (Partition partition : partitions) { - partition.getBaseIndex().setRowCount(1000); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(1000); } LogicalOlapScanOperator olapScanOperator = @@ -519,7 +519,7 @@ public void testLogicalOlapTableScanPartitionPrune2(@Mocked CachedStatisticStora List partitionIds = partitions.stream().filter(partition -> partition.getName().equalsIgnoreCase("p2")). mapToLong(partition -> partition.getId()).boxed().collect(Collectors.toList()); for (Partition partition : partitions) { - partition.getBaseIndex().setRowCount(1000); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(1000); } LogicalOlapScanOperator olapScanOperator = diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java index 4060dcb388193..0b3e90066e1ce 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java @@ -239,8 +239,9 @@ public void testJoinWithMysqlTable() throws Exception { Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "jointest"); for (Partition partition : tbl.getPartitions()) { - partition.updateVisibleVersion(2); - for (MaterializedIndex mIndex : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { + partition.getDefaultPhysicalPartition().updateVisibleVersion(2); + for (MaterializedIndex mIndex : partition.getDefaultPhysicalPartition() + .getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { mIndex.setRowCount(10000); for (Tablet tablet : mIndex.getTablets()) { for (Replica replica : ((LocalTablet) tablet).getImmutableReplicas()) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java index b39d74674c052..77b0158c8880e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java @@ -788,7 +788,7 @@ public void testLimitPrune() throws Exception { // We need to let some tablets have data, some tablets don't data OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); - MaterializedIndex index = t0.getPartitions().stream().findFirst().get().getBaseIndex(); + MaterializedIndex index = t0.getPartitions().stream().findFirst().get().getDefaultPhysicalPartition().getBaseIndex(); LocalTablet tablets = (LocalTablet) index.getTablets().get(0); Replica replica = tablets.getSingleReplica(); new Expectations(replica) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java index ef79f77413be3..3ddff2d6144ec 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java @@ -886,7 +886,7 @@ public void testPushDownRuntimeFilterAcrossSetOperationNode() throws Exception { List tables = new ArrayList<>(Arrays.asList(t0, t1, t2)); List tabletIdsStrList = new ArrayList<>(); tables.forEach(olapTable -> tabletIdsStrList.add(Joiner.on(",") - .join(olapTable.getPartition(olapTable.getAllPartitionIds().get(0)) + .join(olapTable.getPartition(olapTable.getAllPartitionIds().get(0)).getDefaultPhysicalPartition() .getBaseIndex().getTablets().stream().map(t -> t.getId()).collect(Collectors.toList())))); ArrayList plans = new ArrayList<>(); @@ -1056,7 +1056,7 @@ public void testNotPushDownRuntimeFilterAcrossCTE() throws Exception { List tables = new ArrayList<>(Arrays.asList(t1, t2)); List tabletIdsStrList = new ArrayList<>(); tables.forEach(olapTable -> tabletIdsStrList.add(Joiner.on(",") - .join(olapTable.getPartition(olapTable.getAllPartitionIds().get(0)) + .join(olapTable.getPartition(olapTable.getAllPartitionIds().get(0)).getDefaultPhysicalPartition() .getBaseIndex().getTablets().stream().map(t -> t.getId()).collect(Collectors.toList())))); setTableStatistics(t1, 400000); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java index 3d63995d4bef8..c9cb1b52f954e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java @@ -245,14 +245,14 @@ public static void assertNotContains(String text, String... pattern) { public static void setTableStatistics(OlapTable table, long rowCount) { for (Partition partition : table.getAllPartitions()) { - partition.getBaseIndex().setRowCount(rowCount); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(rowCount); } } public static void setPartitionStatistics(OlapTable table, String partitionName, long rowCount) { for (Partition partition : table.getAllPartitions()) { if (partition.getName().equals(partitionName)) { - partition.getBaseIndex().setRowCount(rowCount); + partition.getDefaultPhysicalPartition().getBaseIndex().setRowCount(rowCount); } } } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ScanTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ScanTest.java index a826bf7f1b038..6931ab684e09b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ScanTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ScanTest.java @@ -468,7 +468,7 @@ public void testHint() throws Exception { Collection partitions = tb.getPartitions(); acquireReplica: for (Partition partition : partitions) { - MaterializedIndex index = partition.getIndex(tb.getBaseIndexId()); + MaterializedIndex index = partition.getDefaultPhysicalPartition().getIndex(tb.getBaseIndexId()); for (Tablet tablet : index.getTablets()) { replicaId = ((LocalTablet) tablet).getImmutableReplicas().get(0).getId(); break acquireReplica; diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java index ed74fc726c8d4..6eddfbba4059f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java @@ -98,7 +98,7 @@ public static void beforeClass() throws Exception { OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0_stats"); t0StatsTableId = t0.getId(); Partition partition = new ArrayList<>(t0.getPartitions()).get(0); - partition.updateVisibleVersion(2, t0UpdateTime + partition.getDefaultPhysicalPartition().updateVisibleVersion(2, t0UpdateTime .atZone(Clock.systemDefaultZone().getZone()).toEpochSecond() * 1000); setTableStatistics(t0, 20000000); @@ -115,7 +115,7 @@ public static void beforeClass() throws Exception { ");"); OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1_stats"); - new ArrayList<>(t1.getPartitions()).get(0).updateVisibleVersion(2); + new ArrayList<>(t1.getPartitions()).get(0).getDefaultPhysicalPartition().updateVisibleVersion(2); setTableStatistics(t1, 20000000); starRocksAssert.withTable("CREATE TABLE `t0_stats_partition` (\n" + @@ -143,7 +143,7 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); OlapTable t0p = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0_stats_partition"); - new ArrayList<>(t0p.getPartitions()).get(0).updateVisibleVersion(2); + new ArrayList<>(t0p.getPartitions()).get(0).getDefaultPhysicalPartition().updateVisibleVersion(2); setTableStatistics(t0p, 20000000); starRocksAssert.withDatabase("stats"); @@ -161,7 +161,7 @@ public static void beforeClass() throws Exception { ");"); OlapTable tps = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("tprimary_stats"); - new ArrayList<>(tps.getPartitions()).get(0).updateVisibleVersion(2); + new ArrayList<>(tps.getPartitions()).get(0).getDefaultPhysicalPartition().updateVisibleVersion(2); setTableStatistics(tps, 20000000); starRocksAssert.withTable("CREATE TABLE `tunique_stats` (\n" + @@ -177,7 +177,7 @@ public static void beforeClass() throws Exception { ");"); OlapTable tus = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("tunique_stats"); - new ArrayList<>(tus.getPartitions()).get(0).updateVisibleVersion(2); + new ArrayList<>(tus.getPartitions()).get(0).getDefaultPhysicalPartition().updateVisibleVersion(2); setTableStatistics(tps, 20000000); starRocksAssert.withTable("CREATE TABLE `tcount` (\n" + @@ -192,7 +192,7 @@ public static void beforeClass() throws Exception { ");"); OlapTable tcount = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("tcount"); - new ArrayList<>(tcount.getPartitions()).get(0).updateVisibleVersion(2); + new ArrayList<>(tcount.getPartitions()).get(0).getDefaultPhysicalPartition().updateVisibleVersion(2); setTableStatistics(tcount, 20000000); String createStructTableSql = "CREATE TABLE struct_a(\n" + @@ -207,7 +207,7 @@ public static void beforeClass() throws Exception { ");"; starRocksAssert.withTable(createStructTableSql); OlapTable structTable = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("struct_a"); - new ArrayList<>(structTable.getPartitions()).get(0).updateVisibleVersion(2); + new ArrayList<>(structTable.getPartitions()).get(0).getDefaultPhysicalPartition().updateVisibleVersion(2); setTableStatistics(structTable, 20000000); } @@ -876,8 +876,8 @@ public void testFullStatisticsBuildCollectSQLList() { .getLocalMetastore().getDb("test").getTable("t0_stats_partition"); int i = 1; for (Partition p : t0p.getAllPartitions()) { - p.updateVisibleVersion(2); - p.getBaseIndex().setRowCount(i * 100L); + p.getDefaultPhysicalPartition().updateVisibleVersion(2); + p.getDefaultPhysicalPartition().getBaseIndex().setRowCount(i * 100L); i++; } @@ -909,8 +909,8 @@ public void testFullStatisticsBuildCollectSQLList() { Assert.assertEquals(4, collectSqlList.size()); for (Partition p : t0p.getAllPartitions()) { - p.updateVisibleVersion(2); - p.getBaseIndex().setRowCount(0); + p.getDefaultPhysicalPartition().updateVisibleVersion(2); + p.getDefaultPhysicalPartition().getBaseIndex().setRowCount(0); } collectSqlList = collectJob.buildCollectSQLList(1); diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java index ee9d6fecae224..7148f50fada69 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java @@ -73,7 +73,8 @@ public static void beforeClass() throws Exception { OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0_stats"); Partition partition = new ArrayList<>(t0.getPartitions()).get(0); - partition.updateVisibleVersion(2, LocalDateTime.of(2022, 1, 1, 1, 1, 1) + partition.getDefaultPhysicalPartition() + .updateVisibleVersion(2, LocalDateTime.of(2022, 1, 1, 1, 1, 1) .atZone(Clock.systemDefaultZone().getZone()).toEpochSecond() * 1000); setTableStatistics(t0, 20000000); } diff --git a/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java index 9bc3051327df9..a1fa3f166a742 100644 --- a/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java @@ -116,14 +116,14 @@ public class GlobalTransactionMgrTest { private static GlobalStateMgr slaveGlobalStateMgr; private TransactionState.TxnCoordinator transactionSource = - new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, "localfe"); + new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, "localfe"); private TransactionState.TxnCoordinator transactionSourceBe = - new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.BE, "localbe"); + new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.BE, "localbe"); @Before public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { + InvocationTargetException, NoSuchMethodException, SecurityException { fakeEditLog = new FakeEditLog(); fakeGlobalStateMgr = new FakeGlobalStateMgr(); fakeTransactionIDGenerator = new FakeTransactionIDGenerator(); @@ -143,15 +143,15 @@ public void tearDown() { @Test public void testBeginTransaction() throws LabelAlreadyUsedException, AnalysisException, - RunningTxnExceedException, DuplicatedRequestException { + RunningTxnExceedException, DuplicatedRequestException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); TransactionState transactionState = - masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); + masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); assertNotNull(transactionState); assertEquals(transactionId, transactionState.getTransactionId()); assertEquals(TransactionStatus.PREPARE, transactionState.getTransactionStatus()); @@ -161,21 +161,21 @@ public void testBeginTransaction() throws LabelAlreadyUsedException, AnalysisExc @Test public void testBeginTransactionWithSameLabel() throws LabelAlreadyUsedException, AnalysisException, - RunningTxnExceedException, DuplicatedRequestException { + RunningTxnExceedException, DuplicatedRequestException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = 0; try { transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, - Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, + Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); } catch (AnalysisException | LabelAlreadyUsedException e) { e.printStackTrace(); } TransactionState transactionState = - masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); + masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); assertNotNull(transactionState); assertEquals(transactionId, transactionState.getTransactionId()); assertEquals(TransactionStatus.PREPARE, transactionState.getTransactionStatus()); @@ -184,11 +184,11 @@ public void testBeginTransactionWithSameLabel() throws LabelAlreadyUsedException try { transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, - Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, + Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); } catch (Exception e) { // TODO: handle exception } @@ -199,36 +199,38 @@ public void testBeginTransactionWithSameLabel() throws LabelAlreadyUsedException public void testCommitTransaction1() throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); TransactionState transactionState = fakeEditLog.getTransaction(transactionId); // check status is committed assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); // check replica version Partition testPartition = masterGlobalStateMgr.getLocalMetastore() - .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, + testPartition.getDefaultPhysicalPartition().getNextVersion()); // check partition next version - LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + LocalTablet tablet = + (LocalTablet) testPartition.getDefaultPhysicalPartition().getIndex(GlobalStateMgrTestUtil.testIndexId1) + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); } @@ -244,20 +246,20 @@ public void testCommitTransactionWithOneFailed() throws UserException { TransactionState transactionState = null; FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction with 1,2 success TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); // follower globalStateMgr replay the transaction transactionState = fakeEditLog.getTransaction(transactionId); @@ -268,19 +270,19 @@ public void testCommitTransactionWithOneFailed() throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); // commit another transaction with 1,3 success long transactionId2 = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable2, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable2, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo3); try { masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); Assert.fail(); } catch (TabletQuorumFailedException e) { transactionState = masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId2); @@ -289,15 +291,16 @@ public void testCommitTransactionWithOneFailed() throws UserException { } // check replica version Partition testPartition = - masterGlobalStateMgr.getLocalMetastore() - .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getDefaultPhysicalPartition().getNextVersion()); // check partition next version - LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + LocalTablet tablet = (LocalTablet) testPartition.getDefaultPhysicalPartition() + .getIndex(GlobalStateMgrTestUtil.testIndexId1) + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); } @@ -313,20 +316,20 @@ public void testCommitTransactionWithOneFailed() throws UserException { transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); transactionState = fakeEditLog.getTransaction(transactionId2); // check status is commit assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); // check replica version testPartition = masterGlobalStateMgr.getLocalMetastore() - .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getDefaultPhysicalPartition().getNextVersion()); // check partition next version - tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + tablet = (LocalTablet) testPartition.getDefaultPhysicalPartition().getIndex(GlobalStateMgrTestUtil.testIndexId1) + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); } @@ -345,8 +348,8 @@ public void testCommitTransactionWithOneFailed() throws UserException { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replcia2.getLastSuccessVersion()); assertEquals(GlobalStateMgrTestUtil.testStartVersion, replcia3.getLastSuccessVersion()); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getDefaultPhysicalPartition().getNextVersion()); transactionState = fakeEditLog.getTransaction(transactionId2); FakeGlobalStateMgr.setGlobalStateMgr(slaveGlobalStateMgr); @@ -357,35 +360,35 @@ public void testCommitTransactionWithOneFailed() throws UserException { @Test public void testCommitRoutineLoadTransaction(@Injectable TabletCommitInfo tabletCommitInfo, @Mocked EditLog editLog) - throws UserException { + throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); TabletCommitInfo tabletCommitInfo1 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "test", 1L, 1L, "host:port", - "topic"); + "topic"); List routineLoadTaskInfoList = - Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); + Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); Map partitionIdToOffset = Maps.newHashMap(); partitionIdToOffset.put(1, 0L); KafkaTaskInfo routineLoadTaskInfo = - new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), - partitionIdToOffset, Config.routine_load_task_timeout_second); + new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), + partitionIdToOffset, Config.routine_load_task_timeout_second); Deencapsulation.setField(routineLoadTaskInfo, "txnId", 1L); routineLoadTaskInfoList.add(routineLoadTaskInfo); TransactionState transactionState = new TransactionState(1L, Lists.newArrayList(1L), 1L, "label", null, - LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), - routineLoadJob.getId(), - Config.stream_load_default_timeout_second); + LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), + routineLoadJob.getId(), + Config.stream_load_default_timeout_second); transactionState.setTransactionStatus(TransactionStatus.PREPARE); masterTransMgr.getCallbackFactory().addCallback(routineLoadJob); // Deencapsulation.setField(transactionState, "txnStateChangeListener", routineLoadJob); @@ -433,9 +436,9 @@ LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1") routineLoadManager.addRoutineLoadJob(routineLoadJob, "db"); Deencapsulation.setField(masterTransMgr.getDatabaseTransactionMgr(GlobalStateMgrTestUtil.testDbId1), - "idToRunningTransactionState", idToTransactionState); + "idToRunningTransactionState", idToTransactionState); masterTransMgr.commitTransaction(1L, 1L, transTablets, Lists.newArrayList(), - txnCommitAttachment); + txnCommitAttachment); Assert.assertEquals(Long.valueOf(101), Deencapsulation.getField(routineLoadJob, "currentTotalRows")); Assert.assertEquals(Long.valueOf(1), Deencapsulation.getField(routineLoadJob, "currentErrorRows")); @@ -452,31 +455,31 @@ public void testCommitRoutineLoadTransactionWithErrorMax(@Injectable TabletCommi FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); TabletCommitInfo tabletCommitInfo1 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); KafkaRoutineLoadJob routineLoadJob = - new KafkaRoutineLoadJob(1L, "test", 1L, 1L, "host:port", "topic"); + new KafkaRoutineLoadJob(1L, "test", 1L, 1L, "host:port", "topic"); List routineLoadTaskInfoList = - Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); + Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); Map partitionIdToOffset = Maps.newHashMap(); partitionIdToOffset.put(1, 0L); KafkaTaskInfo routineLoadTaskInfo = - new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), - partitionIdToOffset, Config.routine_load_task_timeout_second); + new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), + partitionIdToOffset, Config.routine_load_task_timeout_second); Deencapsulation.setField(routineLoadTaskInfo, "txnId", 1L); routineLoadTaskInfoList.add(routineLoadTaskInfo); TransactionState transactionState = new TransactionState(1L, Lists.newArrayList(1L), 1L, "label", null, - LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), - routineLoadJob.getId(), - Config.stream_load_default_timeout_second); + LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), + routineLoadJob.getId(), + Config.stream_load_default_timeout_second); transactionState.setTransactionStatus(TransactionStatus.PREPARE); masterTransMgr.getCallbackFactory().addCallback(routineLoadJob); Map idToTransactionState = Maps.newHashMap(); @@ -523,14 +526,14 @@ LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1") routineLoadManager.addRoutineLoadJob(routineLoadJob, "db"); Deencapsulation.setField(masterTransMgr.getDatabaseTransactionMgr(GlobalStateMgrTestUtil.testDbId1), - "idToRunningTransactionState", idToTransactionState); + "idToRunningTransactionState", idToTransactionState); masterTransMgr.commitTransaction(1L, 1L, transTablets, Lists.newArrayList(), - txnCommitAttachment); + txnCommitAttachment); Assert.assertEquals(Long.valueOf(0), Deencapsulation.getField(routineLoadJob, "currentTotalRows")); Assert.assertEquals(Long.valueOf(0), Deencapsulation.getField(routineLoadJob, "currentErrorRows")); Assert.assertEquals(Long.valueOf(111L), - ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); + ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); // todo(ml): change to assert queue // Assert.assertEquals(0, routineLoadManager.getNeedScheduleTasksQueue().size()); Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, routineLoadJob.getState()); @@ -569,13 +572,15 @@ public void testFinishTransaction() throws UserException { // check replica version Partition testPartition = masterGlobalStateMgr.getLocalMetastore() .getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, + testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getDefaultPhysicalPartition().getNextVersion()); // check partition next version - LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + LocalTablet tablet = + (LocalTablet) testPartition.getDefaultPhysicalPartition().getIndex(GlobalStateMgrTestUtil.testIndexId1) + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { if (replica.getId() == GlobalStateMgrTestUtil.testReplicaId1) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); @@ -593,27 +598,28 @@ public void testFinishTransaction() throws UserException { public void testFinishTransactionWithOneFailed() throws UserException { TransactionState transactionState = null; Partition testPartition = - masterGlobalStateMgr.getLocalMetastore() - .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); - LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); + LocalTablet tablet = + (LocalTablet) testPartition.getDefaultPhysicalPartition().getIndex(GlobalStateMgrTestUtil.testIndexId1) + .getTablet(GlobalStateMgrTestUtil.testTabletId1); FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction with 1,2 success TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); // follower globalStateMgr replay the transaction transactionState = fakeEditLog.getTransaction(transactionId); @@ -661,19 +667,19 @@ public void testFinishTransactionWithOneFailed() throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); // commit another transaction with 1,3 success long transactionId2 = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable2, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable2, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo3); try { masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); Assert.fail(); } catch (TabletQuorumFailedException e) { transactionState = masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId2); @@ -690,17 +696,18 @@ public void testFinishTransactionWithOneFailed() throws UserException { transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); transactionState = fakeEditLog.getTransaction(transactionId2); // check status is commit assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); // check replica version testPartition = masterGlobalStateMgr.getLocalMetastore() - .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, + testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getDefaultPhysicalPartition().getNextVersion()); // follower globalStateMgr replay the transaction transactionState = fakeEditLog.getTransaction(transactionId2); @@ -726,8 +733,9 @@ public void testFinishTransactionWithOneFailed() throws UserException { assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, replcia2.getLastSuccessVersion()); assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, replcia3.getLastSuccessVersion()); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, + testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getDefaultPhysicalPartition().getNextVersion()); transactionState = fakeEditLog.getTransaction(transactionId2); FakeGlobalStateMgr.setGlobalStateMgr(slaveGlobalStateMgr); @@ -743,8 +751,8 @@ public void replayWithExpiredJob() throws Exception { // 1. replay a normal finished transaction TransactionState state = - new TransactionState(dbId, new ArrayList<>(), 1, "label_a", null, LoadJobSourceType.BACKEND_STREAMING, - transactionSource, -1, -1); + new TransactionState(dbId, new ArrayList<>(), 1, "label_a", null, LoadJobSourceType.BACKEND_STREAMING, + transactionSource, -1, -1); state.setTransactionStatus(TransactionStatus.ABORTED); state.setReason("fake reason"); state.setFinishTime(System.currentTimeMillis() - 2000); @@ -753,8 +761,8 @@ public void replayWithExpiredJob() throws Exception { // 2. replay a expired transaction TransactionState state2 = - new TransactionState(dbId, new ArrayList<>(), 2, "label_b", null, LoadJobSourceType.BACKEND_STREAMING, - transactionSource, -1, -1); + new TransactionState(dbId, new ArrayList<>(), 2, "label_b", null, LoadJobSourceType.BACKEND_STREAMING, + transactionSource, -1, -1); state2.setTransactionStatus(TransactionStatus.ABORTED); state2.setReason("fake reason"); state2.setFinishTime(System.currentTimeMillis()); @@ -764,8 +772,8 @@ public void replayWithExpiredJob() throws Exception { Thread.sleep(2000); // 3. replay a valid transaction, let state expire TransactionState state3 = - new TransactionState(dbId, new ArrayList<>(), 3, "label_c", null, LoadJobSourceType.BACKEND_STREAMING, - transactionSource, -1, -1); + new TransactionState(dbId, new ArrayList<>(), 3, "label_c", null, LoadJobSourceType.BACKEND_STREAMING, + transactionSource, -1, -1); state3.setTransactionStatus(TransactionStatus.ABORTED); state3.setReason("fake reason"); state3.setFinishTime(System.currentTimeMillis()); @@ -799,30 +807,30 @@ public void testPrepareTransaction() throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.prepareTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); TransactionState transactionState = fakeEditLog.getTransaction(transactionId); assertEquals(TransactionStatus.PREPARED, transactionState.getTransactionStatus()); try { masterTransMgr.commitPreparedTransaction( - masterGlobalStateMgr.getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDbId1), transactionId, - (long) 1000); + masterGlobalStateMgr.getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDbId1), transactionId, + (long) 1000); Assert.fail("should throw publish timeout exception"); } catch (UserException e) { } @@ -837,15 +845,17 @@ public void testPrepareTransaction() throws UserException { assertEquals(TransactionStatus.VISIBLE, transactionState.getTransactionStatus()); // check replica version Partition testPartition = - masterGlobalStateMgr.getLocalMetastore() - .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, testPartition.getVisibleVersion()); - assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getNextVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, + testPartition.getDefaultPhysicalPartition().getVisibleVersion()); + assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getDefaultPhysicalPartition().getNextVersion()); // check partition next version - LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + LocalTablet tablet = + (LocalTablet) testPartition.getDefaultPhysicalPartition().getIndex(GlobalStateMgrTestUtil.testIndexId1) + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { if (replica.getId() == GlobalStateMgrTestUtil.testReplicaId1) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); @@ -862,10 +872,10 @@ public void testPrepareTransaction() throws UserException { @Test public void testSaveLoadJsonFormatImage() throws Exception { long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); UtFrameUtils.PseudoImage pseudoImage = new UtFrameUtils.PseudoImage(); masterTransMgr.saveTransactionStateV2(pseudoImage.getImageWriter()); @@ -881,7 +891,7 @@ public void testSaveLoadJsonFormatImage() throws Exception { @Test public void testRetryCommitOnRateLimitExceededTimeout() - throws UserException { + throws UserException { Database db = new Database(10, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); DatabaseTransactionMgr dbTransactionMgr = spy(new DatabaseTransactionMgr(10L, GlobalStateMgr.getCurrentState())); @@ -891,15 +901,15 @@ public void testRetryCommitOnRateLimitExceededTimeout() doReturn(transactionState).when(globalTransactionMgr).getTransactionState(db.getId(), 1001); doReturn(dbTransactionMgr).when(globalTransactionMgr).getDatabaseTransactionMgr(db.getId()); doThrow(new CommitRateExceededException(1001, now + 60 * 1000L)) - .when(dbTransactionMgr) - .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); + .when(dbTransactionMgr) + .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); Assert.assertThrows(CommitRateExceededException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001, - Collections.emptyList(), Collections.emptyList(), 10, null)); + Collections.emptyList(), Collections.emptyList(), 10, null)); } @Test public void testPublishVersionTimeout() - throws UserException, LockTimeoutException { + throws UserException, LockTimeoutException { Database db = new Database(10, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); DatabaseTransactionMgr dbTransactionMgr = spy(new DatabaseTransactionMgr(10L, GlobalStateMgr.getCurrentState())); @@ -909,52 +919,52 @@ public void testPublishVersionTimeout() doReturn(dbTransactionMgr).when(globalTransactionMgr).getDatabaseTransactionMgr(db.getId()); doReturn(transactionState).when(globalTransactionMgr).getTransactionState(db.getId(), 1001); doReturn(new VisibleStateWaiter(new TransactionState())) - .when(dbTransactionMgr) - .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); + .when(dbTransactionMgr) + .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); Assert.assertFalse(globalTransactionMgr.commitAndPublishTransaction(db, 1001, - Collections.emptyList(), Collections.emptyList(), 2, null)); + Collections.emptyList(), Collections.emptyList(), 2, null)); } @Test public void testRetryCommitOnRateLimitExceededThrowUnexpectedException() - throws UserException { + throws UserException { Database db = new Database(10, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); DatabaseTransactionMgr dbTransactionMgr = spy(new DatabaseTransactionMgr(10L, GlobalStateMgr.getCurrentState())); doReturn(dbTransactionMgr).when(globalTransactionMgr).getDatabaseTransactionMgr(db.getId()); doThrow(NullPointerException.class) - .when(dbTransactionMgr) - .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); + .when(dbTransactionMgr) + .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); Assert.assertThrows(UserException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001, - Collections.emptyList(), Collections.emptyList(), 10, null)); + Collections.emptyList(), Collections.emptyList(), 10, null)); } @Test public void testRetryCommitOnRateLimitExceededThrowLockTimeoutException() - throws UserException, LockTimeoutException { + throws UserException, LockTimeoutException { Database db = new Database(10L, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); TransactionState transactionState = new TransactionState(); doReturn(transactionState) - .when(globalTransactionMgr) - .getTransactionState(10L, 1001L); + .when(globalTransactionMgr) + .getTransactionState(10L, 1001L); doThrow(LockTimeoutException.class) - .when(globalTransactionMgr) - .retryCommitOnRateLimitExceeded(db, 1001L, Collections.emptyList(), Collections.emptyList(), null, 10L); + .when(globalTransactionMgr) + .retryCommitOnRateLimitExceeded(db, 1001L, Collections.emptyList(), Collections.emptyList(), null, 10L); Assert.assertThrows(LockTimeoutException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001L, - Collections.emptyList(), Collections.emptyList(), 10L, null)); + Collections.emptyList(), Collections.emptyList(), 10L, null)); } @Test public void testGetTransactionNumByCoordinateBe() throws LabelAlreadyUsedException, AnalysisException, - RunningTxnExceedException, DuplicatedRequestException { + RunningTxnExceedException, DuplicatedRequestException { masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSourceBe, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSourceBe, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); long res = masterTransMgr.getTransactionNumByCoordinateBe("localbe"); assertEquals(1, res); } @@ -1006,16 +1016,16 @@ public void testCommitLockTimeout() throws UserException, LockTimeoutException { Database db = new Database(10L, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); doThrow(LockTimeoutException.class) - .when(globalTransactionMgr) - .commitAndPublishTransaction(db, 1001L, Collections.emptyList(), Collections.emptyList(), 10L, null); + .when(globalTransactionMgr) + .commitAndPublishTransaction(db, 1001L, Collections.emptyList(), Collections.emptyList(), 10L, null); Assert.assertThrows(ErrorReportException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001L, - Collections.emptyList(), Collections.emptyList(), 10L)); + Collections.emptyList(), Collections.emptyList(), 10L)); } @Test public void testCheckValidTimeoutSecond() { ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, - "Invalid timeout: '1'. Expected values should be between 2 and 3 seconds", - () -> GlobalTransactionMgr.checkValidTimeoutSecond(1, 3, 2)); + "Invalid timeout: '1'. Expected values should be between 2 and 3 seconds", + () -> GlobalTransactionMgr.checkValidTimeoutSecond(1, 3, 2)); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java b/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java index de9f838446e43..0dfb21c8dc0bd 100644 --- a/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java @@ -98,7 +98,7 @@ public void testNormal() throws Exception { int num = 0; for (Partition partition : table.getPartitions()) { - MaterializedIndex baseIndex = partition.getBaseIndex(); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Long tabletId : baseIndex.getTabletIdsInOrder()) { for (Long backendId : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackendIds()) { TabletCommitInfo tabletCommitInfo = new TabletCommitInfo(tabletId, backendId); @@ -165,7 +165,7 @@ public void testPublishTransactionState() throws Exception { List transTablets = Lists.newArrayList(); for (Partition partition : table.getPartitions()) { - MaterializedIndex baseIndex = partition.getBaseIndex(); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Long tabletId : baseIndex.getTabletIdsInOrder()) { for (Long backendId : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackendIds()) { TabletCommitInfo tabletCommitInfo = new TabletCommitInfo(tabletId, backendId); @@ -197,7 +197,7 @@ public void testPublishDbDroped() throws Exception { Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), TABLE); List transTablets = Lists.newArrayList(); for (Partition partition : table.getPartitions()) { - MaterializedIndex baseIndex = partition.getBaseIndex(); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Long tabletId : baseIndex.getTabletIdsInOrder()) { for (Long backendId : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackendIds()) { TabletCommitInfo tabletCommitInfo = new TabletCommitInfo(tabletId, backendId); @@ -253,7 +253,7 @@ public void testPublishTableDropped() throws Exception { Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), TABLE); List transTablets = Lists.newArrayList(); for (Partition partition : table.getPartitions()) { - MaterializedIndex baseIndex = partition.getBaseIndex(); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Long tabletId : baseIndex.getTabletIdsInOrder()) { for (Long backendId : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackendIds()) { TabletCommitInfo tabletCommitInfo = new TabletCommitInfo(tabletId, backendId); @@ -312,7 +312,7 @@ public void testTransformBatchToSingle() throws Exception { int num = 0; for (Partition partition : table.getPartitions()) { - MaterializedIndex baseIndex = partition.getBaseIndex(); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); for (Long tabletId : baseIndex.getTabletIdsInOrder()) { for (Long backendId : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackendIds()) { TabletCommitInfo tabletCommitInfo = new TabletCommitInfo(tabletId, backendId); diff --git a/fe/fe-core/src/test/java/com/starrocks/transaction/LakeTableTxnLogApplierTest.java b/fe/fe-core/src/test/java/com/starrocks/transaction/LakeTableTxnLogApplierTest.java index a3da2b240bd72..58d27805758be 100644 --- a/fe/fe-core/src/test/java/com/starrocks/transaction/LakeTableTxnLogApplierTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/transaction/LakeTableTxnLogApplierTest.java @@ -30,15 +30,16 @@ public void testCommitAndApply() { tableCommitInfo.addPartitionCommitInfo(partitionCommitInfo); applier.applyCommitLog(state, tableCommitInfo); - Assert.assertEquals(1, table.getPartition(partitionId).getVisibleVersion()); - Assert.assertEquals(3, table.getPartition(partitionId).getNextVersion()); + Assert.assertEquals(1, table.getPartition(partitionId).getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(3, table.getPartition(partitionId).getDefaultPhysicalPartition().getNextVersion()); state.setTransactionStatus(TransactionStatus.VISIBLE); partitionCommitInfo.setVersionTime(System.currentTimeMillis()); applier.applyVisibleLog(state, tableCommitInfo, /*unused*/null); - Assert.assertEquals(2, table.getPartition(partitionId).getVisibleVersion()); - Assert.assertEquals(3, table.getPartition(partitionId).getNextVersion()); - Assert.assertEquals(partitionCommitInfo.getVersionTime(), table.getPartition(partitionId).getVisibleVersionTime()); + Assert.assertEquals(2, table.getPartition(partitionId).getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(3, table.getPartition(partitionId).getDefaultPhysicalPartition().getNextVersion()); + Assert.assertEquals(partitionCommitInfo.getVersionTime(), table.getPartition(partitionId).getDefaultPhysicalPartition() + .getVisibleVersionTime()); } @Test @@ -51,13 +52,13 @@ public void testApplyCommitLogWithDroppedPartition() { tableCommitInfo.addPartitionCommitInfo(partitionCommitInfo); applier.applyCommitLog(state, tableCommitInfo); - Assert.assertEquals(1, table.getPartition(partitionId).getVisibleVersion()); - Assert.assertEquals(2, table.getPartition(partitionId).getNextVersion()); + Assert.assertEquals(1, table.getPartition(partitionId).getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, table.getPartition(partitionId).getDefaultPhysicalPartition().getNextVersion()); state.setTransactionStatus(TransactionStatus.VISIBLE); partitionCommitInfo.setVersionTime(System.currentTimeMillis()); applier.applyVisibleLog(state, tableCommitInfo, /*unused*/null); - Assert.assertEquals(1, table.getPartition(partitionId).getVisibleVersion()); - Assert.assertEquals(2, table.getPartition(partitionId).getNextVersion()); + Assert.assertEquals(1, table.getPartition(partitionId).getDefaultPhysicalPartition().getVisibleVersion()); + Assert.assertEquals(2, table.getPartition(partitionId).getDefaultPhysicalPartition().getNextVersion()); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java b/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java index 561b0ca9c6a60..9e0c5e4f3f58c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java @@ -634,9 +634,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { .getTable(testDb.getFullName(), tableName.getTbl())); for (Partition partition : tbl.getPartitions()) { if (insertStmt.getTargetPartitionIds().contains(partition.getId())) { - long version = partition.getVisibleVersion() + 1; - partition.setVisibleVersion(version, System.currentTimeMillis()); - MaterializedIndex baseIndex = partition.getBaseIndex(); + long version = partition.getDefaultPhysicalPartition().getVisibleVersion() + 1; + partition.getDefaultPhysicalPartition().setVisibleVersion(version, System.currentTimeMillis()); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); List tablets = baseIndex.getTablets(); for (Tablet tablet : tablets) { List replicas = ((LocalTablet) tablet).getImmutableReplicas(); diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java index 315b3ad1e0a36..6ce82406f3c26 100644 --- a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java @@ -1229,8 +1229,8 @@ public static boolean matchPlanWithoutId(String expect, String actual) { } public static void setPartitionVersion(Partition partition, long version) { - partition.setVisibleVersion(version, System.currentTimeMillis()); - MaterializedIndex baseIndex = partition.getBaseIndex(); + partition.getDefaultPhysicalPartition().setVisibleVersion(version, System.currentTimeMillis()); + MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getBaseIndex(); List tablets = baseIndex.getTablets(); for (Tablet tablet : tablets) { List replicas = ((LocalTablet) tablet).getImmutableReplicas(); @@ -1344,7 +1344,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { if (tbl != null) { for (Long partitionId : insertStmt.getTargetPartitionIds()) { Partition partition = tbl.getPartition(partitionId); - setPartitionVersion(partition, partition.getVisibleVersion() + 1); + setPartitionVersion(partition, partition.getDefaultPhysicalPartition().getVisibleVersion() + 1); } } } else if (stmt instanceof DeleteStmt) {