From c6101215f9343376066152c7a602cce53bdcda4e Mon Sep 17 00:00:00 2001 From: Harbor Liu <460660596@qq.com> Date: Thu, 29 Aug 2024 18:22:56 +0800 Subject: [PATCH] [Refactor] Unify the getTable interface to LocalMetastore (#50206) Signed-off-by: HangyuanLiu <460660596@qq.com> --- .../com/starrocks/alter/AlterJobExecutor.java | 27 +- .../java/com/starrocks/alter/AlterJobMgr.java | 51 +- .../java/com/starrocks/alter/AlterJobV2.java | 2 +- .../starrocks/alter/AlterMVJobExecutor.java | 4 +- .../alter/LakeTableAlterMetaJobBase.java | 24 +- .../alter/LakeTableSchemaChangeJob.java | 11 +- .../alter/MaterializedViewHandler.java | 21 +- .../starrocks/alter/OnlineOptimizeJobV2.java | 24 +- .../com/starrocks/alter/OptimizeJobV2.java | 26 +- .../java/com/starrocks/alter/RollupJobV2.java | 30 +- .../starrocks/alter/SchemaChangeHandler.java | 24 +- .../starrocks/alter/SchemaChangeJobV2.java | 30 +- .../com/starrocks/alter/SystemHandler.java | 2 +- .../java/com/starrocks/analysis/Analyzer.java | 4 +- .../com/starrocks/analysis/TableName.java | 34 +- .../com/starrocks/backup/BackupHandler.java | 6 +- .../java/com/starrocks/backup/BackupJob.java | 10 +- .../java/com/starrocks/backup/RestoreJob.java | 64 +- .../starrocks/backup/mv/MVRestoreUpdater.java | 9 +- .../com/starrocks/binlog/BinlogManager.java | 14 +- .../starrocks/catalog/CatalogRecycleBin.java | 4 +- .../com/starrocks/catalog/CatalogUtils.java | 6 +- .../starrocks/catalog/ColocateTableIndex.java | 17 +- .../java/com/starrocks/catalog/Database.java | 35 +- .../catalog/ForeignKeyConstraint.java | 36 +- .../java/com/starrocks/catalog/HiveTable.java | 2 +- .../starrocks/catalog/MaterializedView.java | 10 +- .../com/starrocks/catalog/MetadataViewer.java | 8 +- .../java/com/starrocks/catalog/OlapTable.java | 4 +- .../catalog/ResourceGroupClassifier.java | 2 +- .../catalog/TabletInvertedIndex.java | 8 +- .../com/starrocks/catalog/TabletStatMgr.java | 8 +- .../starrocks/catalog/UniqueConstraint.java | 6 +- .../catalog/system/sys/GrantsTo.java | 12 +- .../system/sys/SysObjectDependencies.java | 2 +- .../clone/DynamicPartitionScheduler.java | 12 +- .../com/starrocks/clone/TabletChecker.java | 8 +- .../com/starrocks/clone/TabletSchedCtx.java | 4 +- .../com/starrocks/clone/TabletScheduler.java | 4 +- .../com/starrocks/common/proc/DbsProcDir.java | 8 +- .../starrocks/common/proc/JobsDbProcDir.java | 4 +- .../com/starrocks/common/proc/ProcUtils.java | 2 +- .../common/proc/StatisticProcDir.java | 6 +- .../starrocks/common/proc/TablesProcDir.java | 7 +- .../starrocks/common/util/SmallFileMgr.java | 6 +- .../connector/elasticsearch/EsRepository.java | 5 +- .../hive/ConnectorTableMetadataProcessor.java | 6 +- .../consistency/CheckConsistencyJob.java | 8 +- .../consistency/ConsistencyChecker.java | 9 +- .../consistency/MetaRecoveryDaemon.java | 9 +- .../starrocks/http/rest/CancelStreamLoad.java | 2 +- .../starrocks/http/rest/GetDdlStmtAction.java | 4 +- .../http/rest/GetStreamLoadState.java | 2 +- .../starrocks/http/rest/MigrationAction.java | 6 +- .../starrocks/http/rest/RowCountAction.java | 4 +- .../starrocks/http/rest/ShowDataAction.java | 2 +- .../http/rest/ShowMetaInfoAction.java | 4 +- .../http/rest/StorageTypeCheckAction.java | 5 +- .../http/rest/TableQueryPlanAction.java | 4 +- .../http/rest/TableRowCountAction.java | 4 +- .../http/rest/TableSchemaAction.java | 4 +- .../http/rest/TransactionLoadAction.java | 2 +- .../starrocks/http/rest/TriggerAction.java | 4 +- .../BypassWriteTransactionHandler.java | 7 +- .../TransactionWithoutChannelHandler.java | 2 +- .../http/rest/v2/TableBaseAction.java | 4 +- .../com/starrocks/lake/StarMgrMetaSyncer.java | 14 +- .../starrocks/lake/backup/LakeBackupJob.java | 2 +- .../starrocks/lake/backup/LakeRestoreJob.java | 9 +- .../lake/compaction/CompactionScheduler.java | 7 +- .../lake/vacuum/AutovacuumDaemon.java | 4 +- .../java/com/starrocks/leader/LeaderImpl.java | 21 +- .../com/starrocks/leader/ReportHandler.java | 39 +- .../starrocks/listener/LoadJobMVListener.java | 7 +- .../listener/LoadJobStatsListener.java | 2 +- .../com/starrocks/load/BrokerFileGroup.java | 6 +- .../java/com/starrocks/load/DeleteMgr.java | 10 +- .../java/com/starrocks/load/ExportJob.java | 13 +- .../java/com/starrocks/load/ExportMgr.java | 4 +- .../load/InsertOverwriteJobRunner.java | 16 +- .../main/java/com/starrocks/load/Load.java | 2 +- .../com/starrocks/load/OlapDeleteJob.java | 2 +- .../com/starrocks/load/PartitionUtils.java | 2 +- .../starrocks/load/loadv2/BrokerLoadJob.java | 2 +- .../starrocks/load/loadv2/BulkLoadJob.java | 14 +- .../starrocks/load/loadv2/InsertLoadJob.java | 14 +- .../com/starrocks/load/loadv2/LoadJob.java | 2 +- .../load/loadv2/LoadLoadingTask.java | 2 +- .../com/starrocks/load/loadv2/LoadMgr.java | 8 +- .../load/loadv2/LoadsHistorySyncer.java | 7 +- .../starrocks/load/loadv2/SparkLoadJob.java | 5 +- .../load/loadv2/SparkLoadPendingTask.java | 9 +- .../java/com/starrocks/load/pipe/Pipe.java | 4 +- .../com/starrocks/load/pipe/PipeManager.java | 2 +- .../load/pipe/filelist/RepoCreator.java | 5 +- .../load/routineload/KafkaRoutineLoadJob.java | 4 +- .../load/routineload/KafkaTaskInfo.java | 4 +- .../routineload/PulsarRoutineLoadJob.java | 4 +- .../load/routineload/PulsarTaskInfo.java | 4 +- .../load/routineload/RoutineLoadJob.java | 26 +- .../load/routineload/RoutineLoadMgr.java | 2 +- .../load/streamload/StreamLoadMgr.java | 13 +- .../load/streamload/StreamLoadTask.java | 4 +- .../memory/InternalCatalogMemoryTracker.java | 4 +- .../metric/MaterializedViewMetricsEntity.java | 25 +- .../java/com/starrocks/metric/MetricRepo.java | 6 +- .../starrocks/mv/MVMetaVersionRepairer.java | 3 +- .../com/starrocks/mv/MVRepairHandler.java | 2 +- .../java/com/starrocks/persist/EditLog.java | 2 +- .../com/starrocks/planner/BinlogScanNode.java | 2 +- .../com/starrocks/planner/OlapTableSink.java | 14 +- .../starrocks/privilege/DbPEntryObject.java | 2 +- .../privilege/FunctionPEntryObject.java | 2 +- .../privilege/NativeAccessController.java | 9 +- .../starrocks/privilege/PipePEntryObject.java | 8 +- .../privilege/TablePEntryObject.java | 4 +- .../RangerStarRocksAccessController.java | 12 +- .../com/starrocks/qe/DDLStmtExecutor.java | 4 +- .../java/com/starrocks/qe/ShowExecutor.java | 46 +- .../java/com/starrocks/qe/StmtExecutor.java | 43 +- .../starrocks/replication/ReplicationJob.java | 8 +- .../starrocks/scheduler/MVActiveChecker.java | 5 +- .../PartitionBasedMvRefreshProcessor.java | 6 +- .../java/com/starrocks/scheduler/TaskRun.java | 4 +- .../scheduler/history/TableKeeper.java | 8 +- .../scheduler/mv/MVMaintenanceJob.java | 6 +- .../scheduler/mv/MVMaintenanceTask.java | 2 +- .../scheduler/mv/MVPCTRefreshPartitioner.java | 2 +- .../mv/TxnBasedEpochCoordinator.java | 2 +- .../com/starrocks/server/GlobalStateMgr.java | 27 +- .../com/starrocks/server/LocalMetastore.java | 154 +- .../com/starrocks/server/MetadataMgr.java | 12 +- .../service/FrontendServiceImpl.java | 54 +- .../service/InformationSchemaDataSource.java | 10 +- .../com/starrocks/sql/StatementPlanner.java | 16 +- .../analyzer/AlterTableStatementAnalyzer.java | 8 +- .../sql/analyzer/AnalyzeStmtAnalyzer.java | 19 +- .../starrocks/sql/analyzer/AnalyzerUtils.java | 12 +- .../sql/analyzer/AuthorizationAnalyzer.java | 11 +- .../sql/analyzer/AuthorizerStmtVisitor.java | 14 +- .../sql/analyzer/BackupRestoreAnalyzer.java | 8 +- .../starrocks/sql/analyzer/CTASAnalyzer.java | 3 +- .../CancelAlterTableStatementAnalyzer.java | 5 +- .../sql/analyzer/CreateTableAnalyzer.java | 10 +- .../sql/analyzer/CreateTableLikeAnalyzer.java | 13 +- .../sql/analyzer/DeleteAnalyzer.java | 10 +- .../sql/analyzer/DropStmtAnalyzer.java | 8 +- .../sql/analyzer/ExportStmtAnalyzer.java | 2 +- .../sql/analyzer/ExpressionAnalyzer.java | 4 +- .../sql/analyzer/InsertAnalyzer.java | 8 +- .../sql/analyzer/LoadStmtAnalyzer.java | 5 +- .../analyzer/MaterializedViewAnalyzer.java | 36 +- .../starrocks/sql/analyzer/QueryAnalyzer.java | 4 +- .../analyzer/RecoverPartitionAnalyzer.java | 3 +- .../sql/analyzer/RecoverTableAnalyzer.java | 3 +- .../RefreshTableStatementAnalyzer.java | 3 +- .../sql/analyzer/ResourceGroupAnalyzer.java | 2 +- .../sql/analyzer/ShowAlterStmtAnalyzer.java | 4 +- .../sql/analyzer/ShowStmtAnalyzer.java | 6 +- .../sql/analyzer/ShowTabletStmtAnalyzer.java | 4 +- .../sql/analyzer/TruncateTableAnalyzer.java | 3 +- .../sql/analyzer/UpdateAnalyzer.java | 10 +- .../starrocks/sql/analyzer/ViewAnalyzer.java | 8 +- .../starrocks/sql/ast/DataDescription.java | 10 +- .../com/starrocks/sql/ast/ExportStmt.java | 4 +- .../sql/ast/ShowAnalyzeStatusStmt.java | 9 +- .../sql/ast/ShowBasicStatsMetaStmt.java | 4 +- .../sql/ast/ShowHistogramStatsMetaStmt.java | 4 +- .../starrocks/sql/ast/pipe/ShowPipeStmt.java | 2 +- .../com/starrocks/sql/common/MetaUtils.java | 136 +- .../starrocks/sql/common/SqlWithIdUtils.java | 4 +- .../sql/common/SyncPartitionUtils.java | 4 +- .../sql/optimizer/MvRewritePreprocessor.java | 4 +- .../sql/optimizer/function/MetaFunctions.java | 10 +- ...aterializedViewTransparentRewriteRule.java | 4 +- .../materialization/MvUtils.java | 7 +- .../statistics/CacheDictManager.java | 4 +- .../ColumnBasicStatsCacheLoader.java | 4 +- .../ColumnHistogramStatsCacheLoader.java | 4 +- .../sql/plan/PlanFragmentBuilder.java | 2 +- .../com/starrocks/statistic/AnalyzeMgr.java | 49 +- .../starrocks/statistic/BasicStatsMeta.java | 4 +- .../starrocks/statistic/NativeAnalyzeJob.java | 6 +- .../statistic/NativeAnalyzeStatus.java | 15 +- .../statistic/StatisticExecutor.java | 20 +- .../starrocks/statistic/StatisticUtils.java | 8 +- .../StatisticsCollectJobFactory.java | 13 +- .../statistic/StatisticsMetaManager.java | 10 +- .../starrocks/system/SystemInfoService.java | 6 +- .../com/starrocks/task/AlterReplicaTask.java | 4 +- .../com/starrocks/task/ExportPendingTask.java | 2 +- .../starrocks/task/PublishVersionTask.java | 2 +- .../transaction/DatabaseTransactionMgr.java | 42 +- .../transaction/GlobalTransactionMgr.java | 4 +- .../transaction/OlapTableTxnLogApplier.java | 2 +- .../OlapTableTxnStateListener.java | 2 +- .../transaction/PublishVersionDaemon.java | 8 +- .../transaction/TransactionChecker.java | 4 +- .../transaction/TransactionState.java | 2 +- .../UpdateDbUsedDataQuotaDaemon.java | 2 +- .../com/starrocks/alter/AlterJobV2Test.java | 91 +- .../com/starrocks/alter/AlterTableTest.java | 400 ++-- .../java/com/starrocks/alter/AlterTest.java | 1940 +++++++-------- .../starrocks/alter/BatchRollupJobTest.java | 8 +- .../alter/CompactionHandlerTest.java | 35 +- ...leAlterDataCachePartitionDurationTest.java | 20 +- .../alter/LakeTableAlterMetaJobTest.java | 21 +- ...LakeTableAsyncFastSchemaChangeJobTest.java | 31 +- .../alter/LakeTableSchemaChangeJobTest.java | 29 +- .../alter/OnlineOptimizeJobV2Test.java | 37 +- .../starrocks/alter/OptimizeJobV2Test.java | 42 +- .../com/starrocks/alter/RollupJobV2Test.java | 52 +- .../alter/SchemaChangeHandlerTest.java | 79 +- .../alter/SchemaChangeJobV2Test.java | 54 +- .../starrocks/analysis/AccessTestUtil.java | 11 +- .../analysis/CTASAutoTabletTest.java | 8 +- .../analysis/CreateMaterializedViewTest.java | 78 +- .../CreateSyncMaterializedViewTest.java | 6 +- .../analysis/CreateTableAutoTabletTest.java | 20 +- .../MaterializedViewAutoTabletTest.java | 6 +- .../RefreshMaterializedViewStatementTest.java | 6 +- .../analysis/RefreshMaterializedViewTest.java | 160 +- .../analysis/ResourceGroupStmtTest.java | 10 +- .../ShowCreateMaterializedViewStmtTest.java | 6 +- .../analysis/ShowCreateViewStmtTest.java | 10 +- .../starrocks/analysis/ShowDbStmtTest.java | 25 +- .../starrocks/analysis/ShowDeleteTest.java | 25 +- .../analysis/UseMaterializedViewTest.java | 11 +- .../starrocks/backup/BackupHandlerTest.java | 127 +- .../backup/BackupJobMaterializedViewTest.java | 46 +- .../backup/BackupJobPrimaryKeyTest.java | 20 +- .../com/starrocks/backup/BackupJobTest.java | 25 +- .../com/starrocks/backup/CatalogMocker.java | 8 +- .../RestoreJobMaterializedViewTest.java | 136 +- .../backup/RestoreJobPrimaryKeyTest.java | 16 +- .../com/starrocks/backup/RestoreJobTest.java | 63 +- .../benchmark/MvRefreshConcurrencyTest.java | 4 +- .../starrocks/binlog/BinlogManagerTest.java | 15 +- .../com/starrocks/catalog/AdminStmtTest.java | 4 +- .../CatalogRecycleBinLakeTableTest.java | 6 +- .../catalog/ColocateTableIndexTest.java | 99 +- .../starrocks/catalog/ColocateTableTest.java | 10 +- .../catalog/CreateTableLikeTest.java | 219 +- .../starrocks/catalog/CreateTableTest.java | 56 +- .../catalog/CreateTableWithAggStateTest.java | 24 +- .../catalog/CreateTableWithLocationTest.java | 251 +- .../com/starrocks/catalog/CreateViewTest.java | 4 +- .../starrocks/catalog/DropPartitionTest.java | 20 +- .../catalog/DynamicPartitionTableTest.java | 574 ++--- .../ExpressionRangePartitionInfoTest.java | 248 +- .../catalog/ForeignKeyConstraintTest.java | 21 +- .../catalog/GlobalStateMgrTestUtil.java | 6 +- .../starrocks/catalog/InfoSchemaDbTest.java | 2 +- .../catalog/MVPartitionExprResolverTest.java | 96 +- .../catalog/MaterializedViewTest.java | 635 ++--- .../starrocks/catalog/MetadataViewerTest.java | 6 +- .../catalog/StorageCoolDownTest.java | 10 +- .../catalog/StorageMediumInferTest.java | 14 +- .../starrocks/catalog/TempPartitionTest.java | 6 +- .../clone/ColocateTableBalancerTest.java | 156 +- .../clone/DynamicPartitionSchedulerTest.java | 277 +-- .../starrocks/clone/TabletSchedCtxTest.java | 2 +- .../cluster/SystemInfoServiceTest.java | 10 +- .../starrocks/common/proc/DbsProcDirTest.java | 24 +- .../common/proc/TablesProcDirTest.java | 26 + .../common/util/SmallFileMgrTest.java | 2 +- .../elasticsearch/EsShardPartitionsTest.java | 4 +- .../consistency/ConsistencyCheckerTest.java | 6 +- .../consistency/MetaRecoveryDaemonTest.java | 35 +- .../starrocks/TableMetaSyncerTest.java | 2 +- .../starrocks/http/ShowDataActionTest.java | 4 + .../starrocks/http/StarRocksHttpTestCase.java | 53 +- .../rest/v2/TablePartitionActionTest.java | 6 +- .../http/rest/v2/TableSchemaActionTest.java | 2 +- .../java/com/starrocks/lake/AlterTest.java | 145 +- .../starrocks/lake/CreateLakeTableTest.java | 10 +- .../lake/LakeMaterializedViewTest.java | 145 +- .../com/starrocks/lake/delete/DeleteTest.java | 10 +- .../starrocks/lake/qe/ShowExecutorTest.java | 4 +- .../starrocks/leader/ReportHandlerTest.java | 52 +- .../starrocks/load/BrokerFileGroupTest.java | 10 + .../com/starrocks/load/DeleteHandlerTest.java | 36 +- .../com/starrocks/load/DeletePruneTest.java | 95 +- .../load/InsertOverwriteJobRunnerTest.java | 8 +- .../load/loadv2/BrokerLoadJobTest.java | 24 +- .../load/loadv2/InsertLoadJobTest.java | 8 +- .../starrocks/load/loadv2/LoadJobTest.java | 2 +- .../starrocks/load/loadv2/LoadMgrTest.java | 16 +- .../load/loadv2/SparkLoadJobTest.java | 16 +- .../load/loadv2/SparkLoadPendingTaskTest.java | 14 +- .../starrocks/load/pipe/PipeManagerTest.java | 8 +- .../routineload/KafkaRoutineLoadJobTest.java | 8 +- .../routineload/RoutineLoadJobMetaTest.java | 6 +- .../load/routineload/RoutineLoadJobTest.java | 10 +- .../routineload/RoutineLoadManagerTest.java | 14 +- .../routineload/RoutineLoadSchedulerTest.java | 8 +- .../streamload/StreamLoadManagerTest.java | 6 +- .../starrocks/persist/ColumnRenameTest.java | 10 +- .../RenameMaterializedViewLogTest.java | 4 +- .../MaterializedViewLowCardTPCHTest.java | 4 +- .../planner/MaterializedViewTPCHTest.java | 4 +- .../planner/MaterializedViewTest.java | 4 +- .../planner/MaterializedViewTestBase.java | 4 +- ...erializedViewTextBasedRewriteTPCHTest.java | 4 +- .../starrocks/planner/OlapTableSinkTest2.java | 4 +- .../planner/mv/MVMetaVersionRepairerTest.java | 8 +- .../privilege/AuthorizationMgrTest.java | 14 + .../privilege/RBACMockedMetadataMgr.java | 11 - .../pseudocluster/BeRestartTest.java | 2 +- .../LocationLabeledTableBalanceTest.java | 4 +- .../LocationMismatchRepairTest.java | 142 +- .../pseudocluster/PseudoCluster.java | 4 +- .../ReplicaMinReadableVersionTest.java | 2 +- .../com/starrocks/qe/ShowExecutorTest.java | 210 +- .../com/starrocks/qe/ShowTableMockMeta.java | 11 - .../replication/ReplicationJobTest.java | 23 +- .../replication/ReplicationMgrTest.java | 5 +- .../scheduler/MVRefreshTestBase.java | 8 +- .../PCTRefreshListPartitionOlapTest.java | 1534 ++++++------ ...titionBasedMvRefreshProcessorHiveTest.java | 100 +- ...ionBasedMvRefreshProcessorIcebergTest.java | 271 ++- ...titionBasedMvRefreshProcessorJdbcTest.java | 275 +-- ...nBasedMvRefreshProcessorOlapPart2Test.java | 24 +- ...titionBasedMvRefreshProcessorOlapTest.java | 292 ++- ...tionBasedMvRefreshProcessorPaimonTest.java | 10 +- .../PartitionBasedMvRefreshTest.java | 430 ++-- .../scheduler/mv/MVMaintenanceJobTest.java | 2 +- .../starrocks/server/ConcurrentDDLTest.java | 47 +- .../starrocks/server/LocalMetaStoreTest.java | 79 +- .../starrocks/server/MVRepairHandlerTest.java | 2 +- .../service/FrontendServiceImplTest.java | 651 ++--- .../analyzer/AnalyzeCreateAnalyzeJobTest.java | 7 +- .../sql/analyzer/AnalyzeInsertTest.java | 37 +- .../sql/analyzer/AnalyzeStmtTest.java | 44 +- .../sql/analyzer/CTASAnalyzerTest.java | 4 +- .../MaterializedViewAnalyzerTest.java | 5 +- .../sql/analyzer/PrivilegeCheckerTest.java | 18 +- .../starrocks/sql/common/MetaUtilTest.java | 4 +- .../sql/common/SqlWithIdUtilsTest.java | 6 +- .../starrocks/sql/optimizer/UtilsTest.java | 4 +- .../operator/ColumnFilterConverterTest.java | 2 +- .../rewrite/PredicateReorderRuleTest.java | 10 +- .../rule/mv/MaterializedViewRuleTest.java | 22 +- .../MVRewriteWithSchemaChangeTest.java | 7 +- .../MvRefreshAndRewriteIcebergTest.java | 2084 +++++++++-------- .../MvRefreshAndRewriteJDBCTest.java | 281 +-- .../materialization/MvRefreshTest.java | 226 +- .../materialization/MvRewriteTestBase.java | 4 +- .../materialization/MvUtilsTest.java | 7 +- .../CachedStatisticStorageTest.java | 16 +- .../statistics/StatisticsCalculatorTest.java | 361 +-- .../com/starrocks/sql/plan/CTEPlanTest.java | 4 +- .../sql/plan/DistributedEnvPlanTestBase.java | 26 +- .../plan/DistributedEnvPlanWithCostTest.java | 2 +- .../sql/plan/DistributedEnvTPCDSPlanTest.java | 10 +- .../starrocks/sql/plan/ExternalTableTest.java | 4 +- .../starrocks/sql/plan/GroupingSetsTest.java | 2 +- .../java/com/starrocks/sql/plan/JoinTest.java | 8 +- .../com/starrocks/sql/plan/LimitTest.java | 2 +- .../sql/plan/MockTpchStatisticStorage.java | 24 +- .../sql/plan/MultiJoinReorderTest.java | 8 +- .../sql/plan/PlanFragmentWithCostTest.java | 78 +- .../com/starrocks/sql/plan/PlanTestBase.java | 2 +- .../sql/plan/PlanTestNoneDBBase.java | 2 +- .../com/starrocks/sql/plan/SkewJoinTest.java | 16 +- .../com/starrocks/sql/plan/TPCHPlanTest.java | 2 +- .../plan/TPCHPlanWithHistogramCostTest.java | 16 +- .../starrocks/statistic/CacheLoaderTest.java | 11 +- .../statistic/StatisticExecutorTest.java | 6 +- .../statistic/StatisticsCollectJobTest.java | 90 +- .../statistic/StatisticsExecutorTest.java | 7 +- .../statistic/StatisticsSQLTest.java | 28 +- .../system/SystemInfoServiceTest.java | 2 +- .../transaction/GlobalTransactionMgrTest.java | 311 +-- .../transaction/LakePublishBatchTest.java | 23 +- .../starrocks/utframe/StarRocksAssert.java | 116 +- .../com/starrocks/utframe/UtFrameUtils.java | 202 +- 377 files changed, 9264 insertions(+), 8523 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java index ec50c6b387080..3409e8c29a037 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java @@ -89,7 +89,6 @@ import com.starrocks.sql.ast.TableRenameClause; import com.starrocks.sql.ast.TruncatePartitionClause; import com.starrocks.sql.ast.TruncateTableStmt; -import com.starrocks.sql.common.MetaUtils; import com.starrocks.thrift.TStorageMedium; import com.starrocks.thrift.TTabletMetaType; import com.starrocks.thrift.TTabletType; @@ -136,8 +135,11 @@ public Void visitAlterTableStatement(AlterTableStmt statement, ConnectContext co TableName tableName = statement.getTbl(); this.tableName = tableName; - Database db = MetaUtils.getDatabase(context, tableName); - Table table = MetaUtils.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tableName.getDb()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(tableName.getDb(), tableName.getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName); + } if (!(table.isOlapOrCloudNativeTable() || table.isMaterializedView())) { throw new SemanticException("Do not support alter non-native table/materialized-view[" + tableName + "]"); @@ -190,8 +192,11 @@ public Void visitAlterTableStatement(AlterTableStmt statement, ConnectContext co @Override public Void visitAlterViewStatement(AlterViewStmt statement, ConnectContext context) { TableName tableName = statement.getTableName(); - Database db = MetaUtils.getDatabase(context, tableName); - Table table = MetaUtils.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tableName.getDb()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(tableName.getDb(), tableName.getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName); + } if (table.getType() != Table.TableType.VIEW) { throw new SemanticException("The specified table [" + tableName + "] is not a view"); @@ -208,7 +213,10 @@ public Void visitAlterViewStatement(AlterViewStmt statement, ConnectContext cont public Void visitAlterMaterializedViewStatement(AlterMaterializedViewStmt stmt, ConnectContext context) { // check db final TableName mvName = stmt.getMvName(); - Database db = MetaUtils.getDatabase(context, mvName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvName.getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", mvName.getCatalogAndDb()); + } Locker locker = new Locker(); if (!locker.lockDatabaseAndCheckExist(db, LockType.WRITE)) { @@ -216,7 +224,10 @@ public Void visitAlterMaterializedViewStatement(AlterMaterializedViewStmt stmt, } try { - Table table = MetaUtils.getTable(mvName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(mvName.getDb(), mvName.getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", mvName); + } if (!table.isMaterializedView()) { throw new SemanticException("The specified table [" + mvName + "] is not a view"); } @@ -288,7 +299,7 @@ public Void visitSwapTableClause(SwapTableClause clause, ConnectContext context) OlapTable origTable = (OlapTable) table; String origTblName = origTable.getName(); String newTblName = clause.getTblName(); - Table newTbl = db.getTable(newTblName); + Table newTbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), newTblName); if (newTbl == null || !(newTbl.isOlapOrCloudNativeTable() || newTbl.isMaterializedView())) { throw new AlterJobException("Table " + newTblName + " does not exist or is not OLAP/LAKE table"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java index 13d2126f989c7..836ebb0f054fd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java @@ -133,7 +133,7 @@ public void stop() { public void processDropMaterializedView(DropMaterializedViewStmt stmt) throws DdlException, MetaNotFoundException { // check db String dbName = stmt.getDbName(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } @@ -144,7 +144,7 @@ public void processDropMaterializedView(DropMaterializedViewStmt stmt) throws Dd try { Table table = null; boolean hasfindTable = false; - for (Table t : db.getTables()) { + for (Table t : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (t instanceof OlapTable) { OlapTable olapTable = (OlapTable) t; for (MaterializedIndexMeta mvMeta : olapTable.getVisibleIndexMetas()) { @@ -231,7 +231,7 @@ public static QueryStatement recreateMVQuery(MaterializedView materializedView, String createMvSql) { // If we could parse the MV sql successfully, and the schema of mv does not change, // we could reuse the existing MV - Optional mayDb = GlobalStateMgr.getCurrentState().mayGetDb(materializedView.getDbId()); + Optional mayDb = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(materializedView.getDbId()); // check database existing String dbName = mayDb.orElseThrow(() -> @@ -271,8 +271,8 @@ public static QueryStatement recreateMVQuery(MaterializedView materializedView, public void replayAlterMaterializedViewBaseTableInfos(AlterMaterializedViewBaseTableInfosLog log) { long dbId = log.getDbId(); long mvId = log.getMvId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - MaterializedView mv = (MaterializedView) db.getTable(mvId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId); if (mv == null) { return; } @@ -292,8 +292,9 @@ public void replayAlterMaterializedViewBaseTableInfos(AlterMaterializedViewBaseT public void replayAlterMaterializedViewStatus(AlterMaterializedViewStatusLog log) { long dbId = log.getDbId(); long tableId = log.getTableId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - MaterializedView mv = (MaterializedView) db.getTable(tableId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (mv == null) { return; } @@ -314,8 +315,9 @@ public void replayRenameMaterializedView(RenameMaterializedViewLog log) { long dbId = log.getDbId(); long materializedViewId = log.getId(); String newMaterializedViewName = log.getNewMaterializedViewName(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - MaterializedView oldMaterializedView = (MaterializedView) db.getTable(materializedViewId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + MaterializedView oldMaterializedView = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), materializedViewId); if (oldMaterializedView != null) { try (AutoCloseableLock ignore = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(oldMaterializedView.getId()), LockType.WRITE)) { @@ -344,12 +346,13 @@ private void updateTaskDefinition(MaterializedView materializedView) { public void replayChangeMaterializedViewRefreshScheme(ChangeMaterializedViewRefreshSchemeLog log) { long dbId = log.getDbId(); long id = log.getId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { return; } - MaterializedView oldMaterializedView = (MaterializedView) db.getTable(id); + MaterializedView oldMaterializedView = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), id); if (oldMaterializedView == null) { LOG.warn("Ignore change materialized view refresh scheme log because table:" + id + "is null"); return; @@ -393,8 +396,9 @@ public void replayAlterMaterializedViewProperties(short opCode, ModifyTablePrope long tableId = log.getTableId(); Map properties = log.getProperties(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - MaterializedView mv = (MaterializedView) db.getTable(tableId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (mv == null) { LOG.warn("Ignore change materialized view properties og because table:" + tableId + "is null"); return; @@ -428,9 +432,9 @@ public void replaySwapTable(SwapTableOperationLog log) { long dbId = log.getDbId(); long origTblId = log.getOrigTblId(); long newTblId = log.getNewTblId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - OlapTable origTable = (OlapTable) db.getTable(origTblId); - OlapTable newTbl = (OlapTable) db.getTable(newTblId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + OlapTable origTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), origTblId); + OlapTable newTbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), newTblId); LOG.debug("finish replay swap table {}-{} with table {}-{}", origTblId, origTable.getName(), newTblId, newTbl.getName()); } @@ -444,9 +448,9 @@ public void swapTableInternal(SwapTableOperationLog log) throws DdlException { long dbId = log.getDbId(); long origTblId = log.getOrigTblId(); long newTblId = log.getNewTblId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - OlapTable origTable = (OlapTable) db.getTable(origTblId); - OlapTable newTbl = (OlapTable) db.getTable(newTblId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + OlapTable origTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), origTblId); + OlapTable newTbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), newTblId); String origTblName = origTable.getName(); String newTblName = newTbl.getName(); @@ -479,8 +483,8 @@ public void alterView(AlterViewInfo alterViewInfo) { List newFullSchema = alterViewInfo.getNewFullSchema(); String comment = alterViewInfo.getComment(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - View view = (View) db.getTable(tableId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + View view = (View) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(view.getId()), LockType.WRITE); @@ -506,8 +510,9 @@ public void alterView(AlterViewInfo alterViewInfo) { } public void replayModifyPartition(ModifyPartitionInfo info) { - Database db = GlobalStateMgr.getCurrentState().getDb(info.getDbId()); - OlapTable olapTable = (OlapTable) db.getTable(info.getTableId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(info.getDbId()); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), info.getTableId()); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(olapTable.getId()), LockType.WRITE); diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java index add80cae8f6de..12c356238170a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java @@ -262,7 +262,7 @@ public boolean cancel(String errMsg) { */ protected boolean checkTableStable(Database db) throws AlterCancelException { long unHealthyTabletId = TabletInvertedIndex.NOT_EXIST_VALUE; - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterMVJobExecutor.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterMVJobExecutor.java index 489426f0fb5af..404b81c508b21 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterMVJobExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterMVJobExecutor.java @@ -73,7 +73,7 @@ public Void visitTableRenameClause(TableRenameClause clause, ConnectContext cont String newMvName = clause.getNewTableName(); String oldMvName = table.getName(); - if (db.getTable(newMvName) != null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), newMvName) != null) { throw new SemanticException("Materialized view [" + newMvName + "] is already used"); } table.setName(newMvName); @@ -349,7 +349,7 @@ public Void visitRefreshSchemeClause(RefreshSchemeClause refreshSchemeDesc, Conn } try { // check - Table mv = db.getTable(materializedView.getId()); + Table mv = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), materializedView.getId()); if (mv == null) { throw new DmlException( "update meta failed. materialized view:" + materializedView.getName() + " not exist"); diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java index c21ce78583482..4c0effe5a1a5c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableAlterMetaJobBase.java @@ -85,13 +85,13 @@ protected void runPendingJob() throws AlterCancelException { // send task to be GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); List partitions = Lists.newArrayList(); - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("database does not exist, dbId:" + dbId); } - LakeTable table = (LakeTable) db.getTable(tableName); + LakeTable table = (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new AlterCancelException("table does not exist, tableName:" + tableName); } @@ -135,13 +135,13 @@ protected void runWaitingTxnJob() throws AlterCancelException { @Override protected void runRunningJob() throws AlterCancelException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database has been dropped throw new AlterCancelException("database does not exist, dbId:" + dbId); } - LakeTable table = (LakeTable) db.getTable(tableId); + LakeTable table = (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { // table has been dropped throw new AlterCancelException("table does not exist, tableId:" + tableId); @@ -187,14 +187,14 @@ protected void runFinishedRewritingJob() throws AlterCancelException { return; } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database has been dropped LOG.warn("database does not exist, dbId:" + dbId); throw new AlterCancelException("database does not exist, dbId:" + dbId); } - LakeTable table = (LakeTable) db.getTable(tableId); + LakeTable table = (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { // table has been dropped LOG.warn("table does not exist, tableId:" + tableId); @@ -221,12 +221,12 @@ protected void runFinishedRewritingJob() throws AlterCancelException { } boolean readyToPublishVersion() throws AlterCancelException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database has been dropped throw new AlterCancelException("database does not exist, dbId:" + dbId); } - LakeTable table = (LakeTable) db.getTable(tableId); + LakeTable table = (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { // table has been dropped throw new AlterCancelException("table does not exist, tableId:" + tableId); @@ -422,9 +422,9 @@ protected boolean cancelImpl(String errMsg) { return false; } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { - LakeTable table = (LakeTable) db.getTable(tableId); + LakeTable table = (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table != null) { Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(table.getId()), LockType.WRITE); @@ -480,14 +480,14 @@ public void replay(AlterJobV2 replayedJob) { restoreState(other); } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database has been dropped LOG.warn("database does not exist, dbId:" + dbId); return; } - LakeTable table = (LakeTable) db.getTable(tableId); + LakeTable table = (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { return; } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java index 456a4d3b3b21f..eda28f210ff83 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java @@ -697,9 +697,10 @@ private void inactiveRelatedMv(Set modifiedColumns, @NotNull OlapTable t if (modifiedColumns.isEmpty()) { return; } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); for (MvId mvId : tbl.getRelatedMaterializedViews()) { - MaterializedView mv = (MaterializedView) db.getTable(mvId.getId()); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), mvId.getId()); if (mv == null) { LOG.warn("Ignore materialized view {} does not exists", mvId); continue; @@ -1055,13 +1056,13 @@ public void write(DataOutput out) throws IOException { @Nullable ReadLockedDatabase getReadLockedDatabase(long dbId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); return db != null ? new ReadLockedDatabase(db) : null; } @Nullable WriteLockedDatabase getWriteLockedDatabase(long dbId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); return db != null ? new WriteLockedDatabase(db) : null; } @@ -1088,7 +1089,7 @@ private abstract static class LockedDatabase implements AutoCloseable { @Nullable OlapTable getTable(long tableId) { - return (OlapTable) db.getTable(tableId); + return (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java index 313906699e5ae..366a01a3a6145 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java @@ -458,12 +458,13 @@ private List checkAndPrepareMaterializedView(CreateMaterializedViewStmt } // check if mv index already exists in db - if (db.tryGetTable(mvName).isPresent()) { + + if (GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetTable(db.getFullName(), mvName).isPresent()) { throw new DdlException("Table [" + mvName + "] already exists in the db " + db.getFullName()); } // check if mv index already exists in other table's materialized indexes - for (Table tbl : db.getTables()) { + for (Table tbl : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (tbl.isOlapTable()) { OlapTable otherOlapTable = (OlapTable) tbl; if (otherOlapTable.getIndexNameToId().size() > 1 && otherOlapTable.hasMaterializedIndex(mvName)) { @@ -836,10 +837,10 @@ private long dropMaterializedView(String mvName, OlapTable olapTable) { } public void replayDropRollup(DropInfo dropInfo, GlobalStateMgr globalStateMgr) { - Database db = globalStateMgr.getDb(dropInfo.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(dropInfo.getDbId()); long tableId = dropInfo.getTableId(); long rollupIndexId = dropInfo.getIndexId(); - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); try (AutoCloseableLock ignore = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(tableId), LockType.WRITE)) { TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); @@ -884,13 +885,13 @@ private void removeJobFromRunningQueue(AlterJobV2 alterJob) { } private void changeTableStatus(long dbId, long tableId, OlapTableState olapTableState) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.warn("db {} has been dropped when changing table {} status after rollup job done", dbId, tableId); return; } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { return; } @@ -1054,13 +1055,13 @@ public void cancel(CancelStmt stmt, String reason) throws DdlException { Preconditions.checkState(!Strings.isNullOrEmpty(dbName)); Preconditions.checkState(!Strings.isNullOrEmpty(tableName)); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } List rollupJobV2List = new ArrayList<>(); - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); } @@ -1114,7 +1115,7 @@ public void cancelMV(CancelStmt stmt) throws DdlException { Preconditions.checkState(!Strings.isNullOrEmpty(dbName)); Preconditions.checkState(!Strings.isNullOrEmpty(tableName)); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } @@ -1126,7 +1127,7 @@ public void cancelMV(CancelStmt stmt) throws DdlException { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table instanceof OlapTable) { List rollupJobV2List = getUnfinishedAlterJobV2ByTableId(table.getId()); for (AlterJobV2 alterJobV2 : rollupJobV2List) { diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java index 5d92818ee5eac..696faf87385ee 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/OnlineOptimizeJobV2.java @@ -155,7 +155,7 @@ public List getOptimizeTasks() { } private OlapTable checkAndGetTable(Database db, long tableId) throws AlterCancelException { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new AlterCancelException("table: " + tableId + " does not exist in database: " + db.getFullName()); } @@ -173,7 +173,7 @@ protected void runPendingJob() throws AlterCancelException { Preconditions.checkState(jobState == JobState.PENDING, jobState); LOG.info("begin to send create temp partitions. job: {}", jobId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Database " + dbId + " does not exist"); } @@ -240,7 +240,7 @@ protected void runWaitingTxnJob() throws AlterCancelException { // must check if db or table still exist first. // or if table is dropped, the tasks will never be finished, // and the job will be in RUNNING state forever. - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Database " + dbId + " does not exist"); } @@ -320,7 +320,7 @@ private void disableDoubleWritePartition(Database db, OlapTable tbl) { protected void runRunningJob() throws AlterCancelException { Preconditions.checkState(jobState == JobState.RUNNING, jobState); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Database " + dbId + " does not exist"); } @@ -517,7 +517,7 @@ private void cancelInternal() { Database db = null; Locker locker = new Locker(); try { - db = GlobalStateMgr.getCurrentState().getDb(dbId); + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("database id:" + dbId + " does not exist"); } @@ -532,7 +532,7 @@ private void cancelInternal() { } try { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new AlterCancelException("table:" + tableId + " does not exist in database:" + db.getFullName()); } @@ -580,7 +580,7 @@ protected boolean isPreviousLoadFinished() throws AnalysisException { * Should replay all changes before this job's state transfer to PENDING. */ private void replayPending(OnlineOptimizeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; @@ -588,7 +588,7 @@ private void replayPending(OnlineOptimizeJobV2 replayedJob) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -611,7 +611,7 @@ private void replayPending(OnlineOptimizeJobV2 replayedJob) { * Should replay all changes in runPendingJob() */ private void replayWaitingTxn(OnlineOptimizeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; @@ -620,7 +620,7 @@ private void replayWaitingTxn(OnlineOptimizeJobV2 replayedJob) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - tbl = (OlapTable) db.getTable(tableId); + tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -677,12 +677,12 @@ private void onReplayFinished(OnlineOptimizeJobV2 replayedJob, OlapTable targetT * Should replay all changes in runRuningJob() */ private void replayFinished(OnlineOptimizeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl != null) { onReplayFinished(replayedJob, tbl); } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java index 91c95f7626c16..229fa73f0c23c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/OptimizeJobV2.java @@ -140,7 +140,7 @@ public List getOptimizeTasks() { } private OlapTable checkAndGetTable(Database db, long tableId) throws AlterCancelException { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new AlterCancelException("table: " + tableId + " does not exist in database: " + db.getFullName()); } @@ -158,7 +158,7 @@ protected void runPendingJob() throws AlterCancelException { Preconditions.checkState(jobState == JobState.PENDING, jobState); LOG.info("begin to send create temp partitions. job: {}", jobId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Database " + dbId + " does not exist"); } @@ -233,7 +233,7 @@ protected void runWaitingTxnJob() throws AlterCancelException { List partitionLastVersion = Lists.newArrayList(); List tableColumnNames = Lists.newArrayList(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("database id: " + dbId + " does not exist"); } @@ -314,12 +314,12 @@ protected void runRunningJob() throws AlterCancelException { // must check if db or table still exist first. // or if table is dropped, the tasks will never be finished, // and the job will be in RUNNING state forever. - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Databasee " + dbId + " does not exist"); } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -545,7 +545,7 @@ private void cancelInternal() { Database db = null; Locker locker = new Locker(); try { - db = GlobalStateMgr.getCurrentState().getDb(dbId); + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("database id:" + dbId + " does not exist"); } @@ -560,7 +560,7 @@ private void cancelInternal() { } try { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new AlterCancelException("table:" + tableId + " does not exist in database:" + db.getFullName()); } @@ -605,12 +605,12 @@ protected boolean isPreviousLoadFinished() throws AnalysisException { * Should replay all changes before this job's state transfer to PENDING. */ private void replayPending(OptimizeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -633,12 +633,12 @@ private void replayPending(OptimizeJobV2 replayedJob) { * Should replay all changes in runPendingJob() */ private void replayWaitingTxn(OptimizeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -691,9 +691,9 @@ private void onReplayFinished(OptimizeJobV2 replayedJob, OlapTable targetTable) * Should replay all changes in runRuningJob() */ private void replayFinished(OptimizeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl != null) { try (AutoCloseableLock ignore = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(tbl.getId()), LockType.WRITE)) { diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java index 0fab5eafbe5cd..c5fa2e1b5b0d0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java @@ -252,7 +252,7 @@ protected void runPendingJob() throws AlterCancelException { Preconditions.checkState(jobState == JobState.PENDING, jobState); LOG.info("begin to send create rollup replica tasks. job: {}", jobId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Database " + dbId + " does not exist"); } @@ -272,7 +272,7 @@ protected void runPendingJob() throws AlterCancelException { } MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalReplicaNum); createReplicaLatch = countDownLatch; - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -380,7 +380,7 @@ protected void runPendingJob() throws AlterCancelException { // create all rollup replicas success. // add rollup index to globalStateMgr - tbl = (OlapTable) db.getTable(tableId); + tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -478,11 +478,11 @@ protected void runWaitingTxnJob() throws AlterCancelException { } LOG.info("previous transactions are all finished, begin to send rollup tasks. job: {}", jobId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Databasee " + dbId + " does not exist"); } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -641,12 +641,12 @@ protected void runRunningJob() throws AlterCancelException { // must check if db or table still exist first. // or if table is dropped, the tasks will never be finished, // and the job will be in RUNNING state forever. - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Databasee " + dbId + " does not exist"); } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -776,9 +776,9 @@ private void cancelInternal() { AgentTaskQueue.removeBatchTask(rollupBatchTask, TTaskType.ALTER); // remove all rollup indexes, and set state to NORMAL TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl != null) { try (AutoCloseableLock ignore = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(tbl.getId()), LockType.WRITE)) { @@ -808,13 +808,13 @@ protected boolean isPreviousLoadFinished() throws AnalysisException { * These changes should be same as changes in RollupHander.processAddRollup() */ private void replayPending(RollupJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -856,12 +856,12 @@ private void addTabletToInvertedIndex(OlapTable tbl) { * Should replay all changes in runPendingJob() */ private void replayWaitingTxn(RollupJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -884,9 +884,9 @@ private void replayWaitingTxn(RollupJobV2 replayedJob) { * Should replay all changes in runRuningJob() */ private void replayFinished(RollupJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl != null) { Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); try (AutoCloseableLock ignore diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java index 460f1a157f335..a76da34b75b03 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java @@ -2057,7 +2057,8 @@ public void updateTableMeta(Database db, String tableName, Map p TTabletMetaType metaType) throws DdlException { List partitions = Lists.newArrayList(); - OlapTable olapTable = (OlapTable) db.getTable(tableName); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(olapTable.getId()), LockType.READ); @@ -2134,7 +2135,7 @@ public boolean updateBinlogConfigMeta(Database db, Long tableId, Map partitionNames, Map properties) throws DdlException { - OlapTable olapTable = (OlapTable) db.getTable(tableName); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(olapTable.getId()), LockType.READ); try { @@ -2283,7 +2285,8 @@ public void updateBinlogPartitionTabletMeta(Database db, TTabletMetaType metaType) throws DdlException { // be id -> Set Map> beIdToTabletId = Maps.newHashMap(); - OlapTable olapTable = (OlapTable) db.getTable(tableName); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(olapTable.getId()), LockType.READ); @@ -2369,7 +2372,8 @@ public void updatePartitionTabletMeta(Database db, TTabletMetaType metaType) throws DdlException { // be id -> Map> beIdToTabletSet = Maps.newHashMap(); - OlapTable olapTable = (OlapTable) db.getTable(tableName); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(olapTable.getId()), LockType.READ); @@ -2456,7 +2460,7 @@ public void updateTableConstraint(Database db, String tableName, Map indexes = info.getIndexes(); long jobId = info.getJobId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - Table table = db.getTable(tableId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); Preconditions.checkArgument(table instanceof OlapTable, "Target of light schema change must be olap table"); OlapTable olapTable = (OlapTable) table; diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java index ee7548839c131..cb14e7e8ae80d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java @@ -293,7 +293,7 @@ private void pruneMeta() { protected void runPendingJob() throws AlterCancelException { Preconditions.checkState(jobState == JobState.PENDING, jobState); LOG.info("begin to send create replica tasks. job: {}", jobId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Databasee " + dbId + " does not exist"); } @@ -314,7 +314,7 @@ protected void runPendingJob() throws AlterCancelException { MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch<>(totalReplicaNum); createReplicaLatch = countDownLatch; - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -434,7 +434,7 @@ protected void runPendingJob() throws AlterCancelException { // create all replicas success. // add all shadow indexes to globalStateMgr - tbl = (OlapTable) db.getTable(tableId); + tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -510,11 +510,11 @@ protected void runWaitingTxnJob() throws AlterCancelException { } LOG.info("previous transactions are all finished, begin to send schema change tasks. job: {}", jobId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Databasee " + dbId + " does not exist"); } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -702,12 +702,12 @@ protected void runRunningJob() throws AlterCancelException { // must check if db or table still exist first. // or if table is dropped, the tasks will never be finished, // and the job will be in RUNNING state forever. - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new AlterCancelException("Database " + dbId + " does not exist"); } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { throw new AlterCancelException("Table " + tableId + " does not exist"); } @@ -974,9 +974,9 @@ private void cancelInternal() { AgentTaskQueue.removeBatchTask(schemaChangeBatchTask, TTaskType.ALTER); // remove all shadow indexes, and set state to NORMAL TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl != null) { Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(tbl.getId()), LockType.WRITE); @@ -1019,13 +1019,13 @@ protected boolean isPreviousLoadFinished() throws AnalysisException { * These changes should be same as changes in SchemaChangeHandler.createJob() */ private void replayPending(SchemaChangeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -1070,12 +1070,12 @@ private void replayPending(SchemaChangeJobV2 replayedJob) { * Should replay all changes in runPendingJob() */ private void replayWaitingTxn(SchemaChangeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return return; } - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl == null) { // table may be dropped before replaying this log. just return return; @@ -1100,10 +1100,10 @@ private void replayWaitingTxn(SchemaChangeJobV2 replayedJob) { * Should replay all changes in runRuningJob() */ private void replayFinished(SchemaChangeJobV2 replayedJob) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (tbl != null) { Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(tbl.getId()), LockType.WRITE); diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SystemHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SystemHandler.java index a48b3e9e1ae05..0172fd150c4dc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SystemHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SystemHandler.java @@ -240,7 +240,7 @@ public Void visitDecommissionBackendClause(DecommissionBackendClause clause, Voi Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table instanceof OlapTable) { OlapTable olapTable = (OlapTable) table; PartitionInfo partitionInfo = olapTable.getPartitionInfo(); diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java b/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java index ddaa29fb417fc..c3d9f52668157 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java @@ -229,11 +229,11 @@ public TupleDescriptor registerTableRef(TableRef ref) throws AnalysisException { } public Table getTable(TableName tblName) { - Database db = globalState.globalStateMgr.getDb(tblName.getDb()); + Database db = globalState.globalStateMgr.getLocalMetastore().getDb(tblName.getDb()); if (db == null) { return null; } - return db.getTable(tblName.getTbl()); + return GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName.getTbl()); } public TupleDescriptor getTupleDesc(TupleId id) { diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java b/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java index ee71e5276fc65..a838c98fd0f18 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java @@ -126,26 +126,22 @@ public void analyze(Analyzer analyzer) throws AnalysisException { } public void normalization(ConnectContext connectContext) { - try { - if (Strings.isNullOrEmpty(catalog)) { - if (Strings.isNullOrEmpty(connectContext.getCurrentCatalog())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_CATALOG_ERROR, catalog); - } - catalog = connectContext.getCurrentCatalog(); + if (Strings.isNullOrEmpty(catalog)) { + if (Strings.isNullOrEmpty(connectContext.getCurrentCatalog())) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_CATALOG_ERROR, catalog); } + catalog = connectContext.getCurrentCatalog(); + } + if (Strings.isNullOrEmpty(db)) { + db = connectContext.getDatabase(); if (Strings.isNullOrEmpty(db)) { - db = connectContext.getDatabase(); - if (Strings.isNullOrEmpty(db)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } + ErrorReport.reportSemanticException(ErrorCode.ERR_NO_DB_ERROR); } + } - if (Strings.isNullOrEmpty(tbl)) { - throw new SemanticException("Table name is null"); - } - } catch (AnalysisException e) { - throw new SemanticException(e.getMessage()); + if (Strings.isNullOrEmpty(tbl)) { + throw new SemanticException("Table name is null"); } } @@ -251,8 +247,12 @@ public void gsonPreProcess() throws IOException { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } TableName tableName = (TableName) o; return Objects.equals(catalog, tableName.catalog) && Objects.equals(tbl, tableName.tbl) diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java index 62fcd7fbb28af..574a9e453c728 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java @@ -262,7 +262,7 @@ public void process(AbstractBackupStmt stmt) throws DdlException { // check if db exist String dbName = stmt.getDbName(); - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } @@ -319,7 +319,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws List backupTbls = Lists.newArrayList(); for (TableRef tblRef : tblRefs) { String tblName = tblRef.getName().getTbl(); - Table tbl = db.getTable(tblName); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); if (tbl == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tblName); return; @@ -519,7 +519,7 @@ private void checkAndFilterRestoreObjsExistInSnapshot(BackupJobInfo jobInfo, Lis } public AbstractJob getAbstractJobByDbName(String dbName) throws DdlException { - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java index 632087a0502e7..6c699655f8ee8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java @@ -393,7 +393,7 @@ public synchronized boolean isDone() { protected void checkBackupTables(Database db) { for (TableRef tableRef : tableRefs) { String tblName = tableRef.getName().getTbl(); - Table tbl = db.getTable(tblName); + Table tbl = globalStateMgr.getLocalMetastore().getTable(db.getFullName(), tblName); if (tbl == null) { status = new Status(ErrCode.NOT_FOUND, "table " + tblName + " does not exist"); return; @@ -449,7 +449,7 @@ protected void sendSnapshotRequests() { private void prepareAndSendSnapshotTask() { MetricRepo.COUNTER_UNFINISHED_BACKUP_JOB.increase(1L); - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); return; @@ -473,7 +473,8 @@ private void prepareAndSendSnapshotTask() { // create snapshot tasks for (TableRef tblRef : tableRefs) { String tblName = tblRef.getName().getTbl(); - OlapTable tbl = (OlapTable) db.getTable(tblName); + OlapTable tbl = (OlapTable) globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), tblName); List partitions = Lists.newArrayList(); if (tblRef.getPartitionNames() == null) { partitions.addAll(tbl.getPartitions()); @@ -508,7 +509,8 @@ private void prepareAndSendSnapshotTask() { List
copiedTables = Lists.newArrayList(); for (TableRef tableRef : tableRefs) { String tblName = tableRef.getName().getTbl(); - OlapTable tbl = (OlapTable) db.getTable(tblName); + OlapTable tbl = (OlapTable) globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), tblName); // only copy visible indexes List reservedPartitions = tableRef.getPartitionNames() == null ? null : tableRef.getPartitionNames().getPartitionNames(); diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java index e48eb52fe916a..fd092d2c12d13 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java @@ -414,7 +414,7 @@ private void checkIfNeedCancel() { return; } - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " has been dropped"); return; @@ -424,7 +424,8 @@ private void checkIfNeedCancel() { locker.lockDatabase(db, LockType.READ); try { for (IdChain idChain : fileMapping.getMapping().keySet()) { - OlapTable tbl = (OlapTable) db.getTable(idChain.getTblId()); + OlapTable tbl = (OlapTable) globalStateMgr.getLocalMetastore() + .getTable(db.getId(), idChain.getTblId()); if (tbl == null) { status = new Status(ErrCode.NOT_FOUND, "table " + idChain.getTblId() + " has been dropped"); return; @@ -468,7 +469,7 @@ private void checkIfNeedCancel() { */ private void checkAndPrepareMeta() { MetricRepo.COUNTER_UNFINISHED_RESTORE_JOB.increase(1L); - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); return; @@ -488,7 +489,8 @@ private void checkAndPrepareMeta() { locker.lockDatabase(db, LockType.WRITE); try { for (BackupTableInfo tblInfo : jobInfo.tables.values()) { - Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + Table tbl = globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); if (tbl == null) { continue; } @@ -523,7 +525,8 @@ private void checkAndPrepareMeta() { for (BackupTableInfo tblInfo : jobInfo.tables.values()) { Table remoteTbl = backupMeta.getTable(tblInfo.name); Preconditions.checkNotNull(remoteTbl); - Table localTbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + Table localTbl = globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); if (localTbl != null) { if (localTbl instanceof OlapTable && localTbl.hasAutoIncrementColumn()) { // it must be !isReplay == true @@ -707,7 +710,7 @@ private void checkAndPrepareMeta() { return; } - ColocateTableIndex colocateTableIndex = GlobalStateMgr.getCurrentState().getColocateTableIndex(); + ColocateTableIndex colocateTableIndex = globalStateMgr.getColocateTableIndex(); if (enableColocateRestore && colocateTableIndex.isColocateTable(remoteOlapTbl.getId())) { ColocateTableIndex.GroupId groupId = colocateTableIndex.getGroup(remoteOlapTbl.getId()); List> backendsPerBucketSeq = colocateTableIndex.getBackendsPerBucketSeq(groupId); @@ -731,7 +734,8 @@ private void checkAndPrepareMeta() { // generate create replica tasks for all restored partitions for (Pair entry : restoredPartitions) { - OlapTable localTbl = (OlapTable) db.getTable(entry.first); + OlapTable localTbl = (OlapTable) globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), entry.first); Preconditions.checkNotNull(localTbl, localTbl.getName()); Partition restorePart = entry.second; OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); @@ -862,7 +866,8 @@ protected void prepareAndSendSnapshotTasks(Database db) { locker.lockDatabase(db, LockType.READ); try { for (IdChain idChain : fileMapping.getMapping().keySet()) { - OlapTable tbl = (OlapTable) db.getTable(idChain.getTblId()); + OlapTable tbl = (OlapTable) globalStateMgr.getLocalMetastore() + .getTable(db.getId(), idChain.getTblId()); PhysicalPartition part = tbl.getPhysicalPartition(idChain.getPartId()); MaterializedIndex index = part.getIndex(idChain.getIdxId()); LocalTablet tablet = (LocalTablet) index.getTablet(idChain.getTabletId()); @@ -884,7 +889,7 @@ protected void prepareAndSendSnapshotTasks(Database db) { // check disk capacity com.starrocks.common.Status st = - GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().checkExceedDiskCapacityLimit(bePathsMap, true); + globalStateMgr.getNodeMgr().getClusterInfo().checkExceedDiskCapacityLimit(bePathsMap, true); if (!st.ok()) { status = new Status(ErrCode.COMMON_ERROR, st.getErrorMsg()); return; @@ -940,9 +945,9 @@ protected void createReplicas(OlapTable localTbl, Partition restorePart) { .setIndexes(localTbl.getCopiedIndexes()) .build().toTabletSchema(); for (Tablet restoreTablet : restoredIdx.getTablets()) { - GlobalStateMgr.getCurrentState().getTabletInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); + globalStateMgr.getTabletInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); for (Replica restoreReplica : ((LocalTablet) restoreTablet).getImmutableReplicas()) { - GlobalStateMgr.getCurrentState().getTabletInvertedIndex() + globalStateMgr.getTabletInvertedIndex() .addReplica(restoreTablet.getId(), restoreReplica); LOG.info("tablet {} physical partition {} index {} replica {}", restoreTablet.getId(), physicalPartition.getId(), restoredIdx.getId(), @@ -1087,13 +1092,14 @@ protected void genFileMappingWithSubPartition(OlapTable localTbl, Partition loca } private void replayCheckAndPrepareMeta() { - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { // replay set all existing tables's state to RESTORE for (BackupTableInfo tblInfo : jobInfo.tables.values()) { - Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + Table tbl = globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); if (tbl == null) { continue; } @@ -1121,7 +1127,8 @@ private void replayCheckAndPrepareMeta() { protected void addRestoredPartitions(Database db, boolean modify) { for (Pair entry : restoredPartitions) { - OlapTable localTbl = (OlapTable) db.getTable(entry.first); + OlapTable localTbl = (OlapTable) globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), entry.first); Partition restorePart = entry.second; OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); RangePartitionInfo localPartitionInfo = (RangePartitionInfo) localTbl.getPartitionInfo(); @@ -1148,9 +1155,9 @@ protected void modifyInvertedIndex(OlapTable restoreTbl, Partition restorePart) TabletMeta tabletMeta = new TabletMeta(dbId, restoreTbl.getId(), restorePart.getId(), restoreIdx.getId(), schemaHash, TStorageMedium.HDD); for (Tablet restoreTablet : restoreIdx.getTablets()) { - GlobalStateMgr.getCurrentState().getTabletInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); + globalStateMgr.getTabletInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); for (Replica restoreReplica : ((LocalTablet) restoreTablet).getImmutableReplicas()) { - GlobalStateMgr.getCurrentState().getTabletInvertedIndex() + globalStateMgr.getTabletInvertedIndex() .addReplica(restoreTablet.getId(), restoreReplica); } } @@ -1164,7 +1171,7 @@ private void waitingAllSnapshotsFinished() { globalStateMgr.getEditLog().logRestoreJob(this); for (ColocatePersistInfo colocatePersistInfo : colocatePersistInfos) { - GlobalStateMgr.getCurrentState().getEditLog().logColocateAddTable(colocatePersistInfo); + globalStateMgr.getEditLog().logColocateAddTable(colocatePersistInfo); } LOG.info("finished making snapshots. {}", this); return; @@ -1191,7 +1198,7 @@ private void downloadSnapshots() { for (long dbId : dbToSnapshotInfos.keySet()) { List infos = dbToSnapshotInfos.get(dbId); - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "db " + dbId + " does not exist"); return; @@ -1280,7 +1287,7 @@ protected void prepareDownloadTasks(List beSnapshotInfos, Database int currentBatchTaskNum = (batch == batchNum - 1) ? totalNum - index : taskNumPerBatch; for (int j = 0; j < currentBatchTaskNum; j++) { SnapshotInfo info = beSnapshotInfos.get(index++); - Table tbl = db.getTable(info.getTblId()); + Table tbl = globalStateMgr.getLocalMetastore().getTable(db.getId(), info.getTblId()); if (tbl == null) { status = new Status(ErrCode.NOT_FOUND, "restored table " + info.getTabletId() + " does not exist"); @@ -1416,7 +1423,7 @@ protected void waitingAllTabletsCommitted() { } private Status allTabletCommitted(boolean isReplay) { - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { return new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); } @@ -1429,7 +1436,7 @@ private Status allTabletCommitted(boolean isReplay) { setTableStateToNormal(db); for (long tblId : restoredVersionInfo.rowKeySet()) { - Table tbl = db.getTable(tblId); + Table tbl = globalStateMgr.getLocalMetastore().getTable(db.getId(), tblId); if (tbl == null) { continue; } @@ -1475,7 +1482,8 @@ private Status allTabletCommitted(boolean isReplay) { locker.lockDatabase(db, LockType.READ); try { for (BackupTableInfo tblInfo : jobInfo.tables.values()) { - Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + Table tbl = globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); // skip to restore table ids if (skipRestoreRemoteTableIds.contains(tblInfo.id)) { continue; @@ -1643,7 +1651,7 @@ public void cancelInternal(boolean isReplay) { } // clean restored objs - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); @@ -1657,7 +1665,7 @@ public void cancelInternal(boolean isReplay) { for (Partition part : restoreTbl.getPartitions()) { for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { - GlobalStateMgr.getCurrentState().getTabletInvertedIndex().deleteTablet(tablet.getId()); + globalStateMgr.getTabletInvertedIndex().deleteTablet(tablet.getId()); } } } @@ -1666,7 +1674,8 @@ public void cancelInternal(boolean isReplay) { // remove restored partitions for (Pair entry : restoredPartitions) { - OlapTable restoreTbl = (OlapTable) db.getTable(entry.first); + OlapTable restoreTbl = (OlapTable) globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), entry.first); if (restoreTbl == null) { continue; } @@ -1683,7 +1692,7 @@ public void cancelInternal(boolean isReplay) { for (ColocatePersistInfo colocatePersistInfo : colocatePersistInfos) { for (Table restoreTbl : restoredTbls) { if (restoreTbl instanceof OlapTable && restoreTbl.getId() == colocatePersistInfo.getTableId()) { - GlobalStateMgr.getCurrentState().getColocateTableIndex() + globalStateMgr.getColocateTableIndex() .removeTable(restoreTbl.getId(), (OlapTable) restoreTbl, isReplay); } } @@ -1715,7 +1724,8 @@ private void setTableStateToNormal(Database db) { if (skipRestoreRemoteTableIds.contains(tblInfo.id)) { continue; } - Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + Table tbl = globalStateMgr.getLocalMetastore() + .getTable(db.getFullName(), jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); if (tbl == null) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/mv/MVRestoreUpdater.java b/fe/fe-core/src/main/java/com/starrocks/backup/mv/MVRestoreUpdater.java index b49d695f1c0c8..4d3ca893389a9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/mv/MVRestoreUpdater.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/mv/MVRestoreUpdater.java @@ -213,13 +213,14 @@ public static boolean restoreBaseTableInfoIfNoRestored(MaterializedView mv, String remoteDbName = baseTableInfo.getDbName(); String remoteTableName = baseTableInfo.getTableName(); - Database baseTableDb = GlobalStateMgr.getCurrentState().getDb(remoteDbName); + Database baseTableDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(remoteDbName); if (baseTableDb == null) { LOG.warn(String.format("Materialized view %s can not find old base table's db name:%s.%s", mv.getName(), remoteDbName, remoteTableName)); return false; } - Table baseTable = baseTableDb.getTable(remoteTableName); + Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(baseTableDb.getFullName(), remoteTableName); if (baseTable == null) { LOG.warn(String.format("Materialized view %s can not find old base table:%s.%s", mv.getName(), remoteDbName, remoteTableName)); @@ -249,14 +250,14 @@ public static Pair> restoreBaseTableInfoIfRestored(MvRes } String localDbName = mvBaseTableBackupInfo.getLocalDbName(); - Database db = GlobalStateMgr.getCurrentState().getDb(localDbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(localDbName); String localTableName = mvBaseTableBackupInfo.getLocalTableName(); if (db == null) { LOG.warn("BaseTable(local) {}'s db {} is not found, remote db/table: {}/{}", localTableName, localDbName, remoteDbName, remoteTableName); return Pair.create(false, Optional.empty()); } - Table localTable = db.getTable(localTableName); + Table localTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), localTableName); remoteToLocalTableName.put(remoteDbTblName, new TableName(db.getFullName(), localTableName)); if (localTable == null) { LOG.warn("Materialized view {} can not find the base table {}, old base table name:{}", diff --git a/fe/fe-core/src/main/java/com/starrocks/binlog/BinlogManager.java b/fe/fe-core/src/main/java/com/starrocks/binlog/BinlogManager.java index 14b097ec8b0c6..f530d32196790 100644 --- a/fe/fe-core/src/main/java/com/starrocks/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/com/starrocks/binlog/BinlogManager.java @@ -226,12 +226,13 @@ public boolean tryDisableBinlog(Database db, long tableId) { public boolean isBinlogAvailable(long dbId, long tableId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (olapTable != null) { return olapTable.getBinlogAvailableVersion().size() != 0; } @@ -246,12 +247,13 @@ public boolean isBinlogAvailable(long dbId, long tableId) { // the binlog is available // result : partitionId -> binlogAvailableVersion, null indicates the db or table is dropped public Map getBinlogAvailableVersion(long dbId, long tableId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (olapTable != null) { return olapTable.getBinlogAvailableVersion(); } @@ -268,12 +270,12 @@ public HashMap showAllBinlog() { HashMap allTablesWithBinlogConfigMap = new HashMap<>(); List allDbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : allDbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - List
tables = db.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (Table table : tables) { if (table.isOlapTable() && ((OlapTable) table).isBinlogEnabled()) { allTablesWithBinlogConfigMap.put(table.getId(), ((OlapTable) table).getCurBinlogConfig()); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java index 1f0d53dd8c376..0c81586fe5959 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java @@ -832,7 +832,7 @@ public void addTabletToInvertedIndex() { // we need to get olap table to get schema hash info // first find it in globalStateMgr. if not found, it should be in recycle bin OlapTable olapTable = null; - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // just log. db should be in recycle bin if (!idToDatabase.containsKey(dbId)) { @@ -842,7 +842,7 @@ public void addTabletToInvertedIndex() { continue; } } else { - olapTable = (OlapTable) db.getTable(tableId); + olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); } if (olapTable == null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java index bbfde75f96b9d..14d5194d64e45 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogUtils.java @@ -57,7 +57,7 @@ public static String normalizeTableName(String dbName, String tableName) { // check table exist public static void checkTableExist(Database db, String tableName) throws DdlException { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); } @@ -115,12 +115,12 @@ public static Set getPartitionNamesFromAddPartitionClause(AddPartitionCl // Used to temporarily disable some command on lake table and remove later. public static void checkIsLakeTable(String dbName, String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { return; } - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { return; } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ColocateTableIndex.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ColocateTableIndex.java index 9e5eaa6e98684..5d4956060c780 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ColocateTableIndex.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ColocateTableIndex.java @@ -492,14 +492,15 @@ public List getAllTableIds(GroupId groupId) { public int getNumOfTabletsPerBucket(GroupId groupId) { List allTableIds = getAllTableIds(groupId); - Database db = GlobalStateMgr.getCurrentState().getDb(groupId.dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(groupId.dbId); int numOfTablets = 0; if (db != null && !allTableIds.isEmpty()) { Locker locker = new Locker(); try { locker.lockDatabase(db, LockType.READ); for (long tableId : allTableIds) { - OlapTable tbl = (OlapTable) db.getTable(tableId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (tbl != null) { numOfTablets += tbl.getNumberOfPartitions(); } @@ -608,9 +609,9 @@ public GroupId changeGroup(long dbId, OlapTable tbl, String oldGroup, String new } public void replayAddTableToGroup(ColocatePersistInfo info) { - Database db = GlobalStateMgr.getCurrentState().getDb(info.getGroupId().dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(info.getGroupId().dbId); Preconditions.checkNotNull(db); - OlapTable tbl = (OlapTable) db.getTable(info.getTableId()); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), info.getTableId()); Preconditions.checkNotNull(tbl); writeLock(); @@ -661,11 +662,11 @@ public void clear() { protected Optional getTableName(long dbId, long tableId) { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { return Optional.empty(); } - Table table = database.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { return Optional.empty(); @@ -986,9 +987,9 @@ public void replayModifyTableColocate(TablePropertyInfo info) { long tableId = info.getTableId(); Map properties = info.getPropertyMap(); - Database db = GlobalStateMgr.getCurrentState().getDb(info.getGroupId().dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(info.getGroupId().dbId); try (AutoCloseableLock ignore = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(tableId), LockType.WRITE)) { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); modifyTableColocate(db, table, properties.get(PropertyAnalyzer.PROPERTIES_COLOCATE_WITH), true, info.getGroupId()); } catch (DdlException e) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java index b7e33a13536f0..3c014993468a3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java @@ -72,7 +72,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -317,7 +316,7 @@ public void dropTable(String tableName, boolean isSetIfExists, boolean isForce) Locker locker = new Locker(); locker.lockDatabase(this, LockType.WRITE); try { - table = getTable(tableName); + table = nameToTable.get(tableName); if (table == null && isSetIfExists) { return; } @@ -352,7 +351,7 @@ public void dropTemporaryTable(long tableId, String tableName, boolean isSetIfEx Locker locker = new Locker(); locker.lockDatabase(this, LockType.WRITE); try { - table = getTable(tableId); + table = idToTable.get(tableId); if (table == null) { if (isSetIfExists) { return; @@ -469,22 +468,17 @@ public Set getTableNamesViewWithLock() { } } - public Optional
tryGetTable(String tableName) { - return Optional.ofNullable(nameToTable.get(tableName)); - } - - public Optional
tryGetTable(long tableId) { - return Optional.ofNullable(idToTable.get(tableId)); + /** + * This is a thread-safe method when idToTable is a concurrent hash map + */ + public Table getTable(long tableId) { + return idToTable.get(tableId); } public Table getTable(String tableName) { return nameToTable.get(tableName); } - public Optional
mayGetTable(String tableName) { - return Optional.ofNullable(nameToTable.get(tableName)); - } - public Pair getMaterializedViewIndex(String mvName) { // TODO: add an index to speed it up. for (Table table : idToTable.values()) { @@ -504,17 +498,6 @@ public Pair getMaterializedViewIndex(String mvName return null; } - /** - * This is a thread-safe method when idToTable is a concurrent hash map - */ - public Table getTable(long tableId) { - return idToTable.get(tableId); - } - - public Optional
mayGetTable(long tableId) { - return Optional.ofNullable(getTable(tableId)); - } - @Override public int getSignature(int signatureVersion) { Adler32 adler32 = new Adler32(); @@ -630,7 +613,7 @@ public synchronized void replayAddFunction(Function function) { public static void replayCreateFunctionLog(Function function) { String dbName = function.getFunctionName().getDb(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new Error("unknown database when replay log, db=" + dbName); } @@ -666,7 +649,7 @@ public synchronized void replayDropFunction(FunctionSearchDesc functionSearchDes public static void replayDropFunctionLog(FunctionSearchDesc functionSearchDesc) { String dbName = functionSearchDesc.getName().getDb(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new Error("unknown database when replay log, db=" + dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ForeignKeyConstraint.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ForeignKeyConstraint.java index 250955161308c..b086982eb27d5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ForeignKeyConstraint.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ForeignKeyConstraint.java @@ -22,7 +22,7 @@ import com.google.common.collect.Streams; import com.starrocks.common.Pair; import com.starrocks.server.GlobalStateMgr; -import com.starrocks.sql.common.MetaUtils; +import com.starrocks.sql.analyzer.SemanticException; import org.apache.commons.lang.math.NumberUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -107,19 +107,37 @@ public List> getColumnNameRefPairs(Table defaultChildTable) private Table getParentTable() { if (parentTableInfo.isInternalCatalog()) { - return MetaUtils.getTable(parentTableInfo.getDbId(), parentTableInfo.getTableId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(parentTableInfo.getDbId(), parentTableInfo.getTableId()); + if (table == null) { + throw new SemanticException("Table %s is not found", parentTableInfo.getTableId()); + } + return table; } else { - return MetaUtils.getTable(parentTableInfo.getCatalogName(), parentTableInfo.getDbName(), - parentTableInfo.getTableName()); + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr() + .getTable(parentTableInfo.getCatalogName(), parentTableInfo.getDbName(), parentTableInfo.getTableName()); + if (table == null) { + throw new SemanticException("Table %s is not found", parentTableInfo.getTableName()); + } + return table; } } private Table getChildTable() { if (childTableInfo.isInternalCatalog()) { - return MetaUtils.getTable(childTableInfo.getDbId(), childTableInfo.getTableId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(childTableInfo.getDbId(), childTableInfo.getTableId()); + if (table == null) { + throw new SemanticException("Table %s is not found", childTableInfo.getTableId()); + } + return table; } else { - return MetaUtils.getTable(childTableInfo.getCatalogName(), childTableInfo.getDbName(), - childTableInfo.getTableName()); + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr() + .getTable(childTableInfo.getCatalogName(), childTableInfo.getDbName(), childTableInfo.getTableName()); + if (table == null) { + throw new SemanticException("Table %s is not found", childTableInfo.getTableName()); + } + return table; } } @@ -221,12 +239,12 @@ private static BaseTableInfo getTableBaseInfo(String catalogName, String db, Str "BaseTableInfo table %s should be tableId for internal catalog", table); long dbId = Long.parseLong(db); long tableId = Long.parseLong(table); - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { throw new IllegalArgumentException(String.format("BaseInfo's db %s should not be null in the foreign key " + "constraint, please drop foreign key constraints and retry", dbId)); } - Table baseTable = database.getTable(tableId); + Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (baseTable == null) { throw new IllegalArgumentException(String.format("BaseInfo' base table %s should not be null in the foreign kee" + " constraint, please drop foreign key constraints and retry", diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java index fddedd3cc6096..acf233b1da8bc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java @@ -288,7 +288,7 @@ public void modifyTableSchema(String dbName, String tableName, HiveTable updated fullSchemaTemp.addAll(updatedTable.fullSchema); dataColumnNamesTemp.addAll(updatedTable.dataColumnNames); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new StarRocksConnectorException("Not found database " + dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java index 295d1518cbac4..7b12549953960 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java @@ -992,7 +992,7 @@ public void fixRelationship() { * @return active or not */ private boolean onReloadImpl() { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.warn("db:{} do not exist. materialized view id:{} name:{} should not exist", dbId, id, name); setInactiveAndReason(MaterializedViewExceptions.inactiveReasonForDbNotExists(dbId)); @@ -1003,7 +1003,7 @@ private boolean onReloadImpl() { if (baseTableIds != null) { // for compatibility for (long tableId : baseTableIds) { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { setInactiveAndReason(MaterializedViewExceptions.inactiveReasonForBaseTableNotExists(tableId)); return false; @@ -1079,7 +1079,7 @@ private boolean onReloadImpl() { } private void analyzePartitionInfo() { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (partitionInfo.isUnPartitioned()) { return; @@ -1840,7 +1840,7 @@ public Status doAfterRestore(MvRestoreContext mvRestoreContext) throws DdlExcept String.format("Materialized view %s's base info is not found", this.name)); } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { return new Status(Status.ErrCode.NOT_FOUND, String.format("Materialized view %s's db %s is not found", this.name, this.dbId)); @@ -1989,7 +1989,7 @@ public void replayAlterMaterializedViewBaseTableInfos(AlterMaterializedViewBaseT */ public synchronized ParseNode getDefineQueryParseNode() { if (this.defineQueryParseNode == null) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { return null; } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MetadataViewer.java index 659a1125815e6..86293d2bf6a96 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MetadataViewer.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MetadataViewer.java @@ -73,7 +73,7 @@ private static List> getTabletStatus(String dbName, String tblName, GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); SystemInfoService infoService = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exsit"); } @@ -81,7 +81,7 @@ private static List> getTabletStatus(String dbName, String tblName, Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table tbl = db.getTable(tblName); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); if (tbl == null || tbl.getType() != TableType.OLAP) { throw new DdlException("Table does not exist or is not OLAP table: " + tblName); } @@ -202,7 +202,7 @@ private static List> getTabletDistribution(String dbName, String tb List> result = Lists.newArrayList(); GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exsit"); } @@ -210,7 +210,7 @@ private static List> getTabletDistribution(String dbName, String tb Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table tbl = db.getTable(tblName); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); if (tbl == null || !tbl.isNativeTableOrMaterializedView()) { throw new DdlException("Table does not exist or is not native table: " + tblName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index 2e9bfce16dd22..985f81adea8c9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -1016,11 +1016,11 @@ public Status doAfterRestore(MvRestoreContext mvRestoreContext) throws DdlExcept if (localMvId == null) { continue; } - Database mvDb = GlobalStateMgr.getCurrentState().getDb(localMvId.getDbId()); + Database mvDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(localMvId.getDbId()); if (mvDb == null) { continue; } - Table mvTable = mvDb.getTable(localMvId.getId()); + Table mvTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(mvDb.getId(), localMvId.getId()); if (mvTable == null) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ResourceGroupClassifier.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ResourceGroupClassifier.java index 19e7f84612c81..dc43505256add 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ResourceGroupClassifier.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ResourceGroupClassifier.java @@ -209,7 +209,7 @@ public String toString() { if (CollectionUtils.isNotEmpty(databaseIds)) { String str = databaseIds.stream() .map(id -> - Optional.ofNullable(GlobalStateMgr.getCurrentState().getDb(id)) + Optional.ofNullable(GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(id)) .map(Database::getFullName) .orElse("unknown")) .collect(Collectors.joining(",")); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java index 0fcb769d8562b..78bd25a9f1e19 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java @@ -394,7 +394,8 @@ public void checkTabletMetaConsistency(Map creatingTableIds) { if (creatingTableIds.containsKey(tableId)) { continue; } - com.starrocks.catalog.Table table = db.getTable(tableId); + com.starrocks.catalog.Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (table == null) { table = recycleBin.getTable(dbId, tableId); if (table != null) { @@ -544,10 +545,11 @@ private boolean isRestoreReplica(Replica replica, Map replicaToTable long dbId = tabletMeta.getDbId(); long tableId = tabletMeta.getTableId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { // getTable is thread-safe for caller, lock free - com.starrocks.catalog.Table tbl = db.getTable(tableId); + com.starrocks.catalog.Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (tbl != null && tbl instanceof OlapTable) { OlapTable olapTable = (OlapTable) tbl; if (olapTable.getState() == OlapTable.OlapTableState.RESTORE) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java index 2f7f4f7f7fc11..16057bb879d9f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletStatMgr.java @@ -102,14 +102,14 @@ protected void runAfterCatalogReady() { long start = System.currentTimeMillis(); List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { long totalRowCount = 0L; if (!table.isNativeTableOrMaterializedView()) { continue; @@ -199,12 +199,12 @@ private void updateLakeTabletStat() { List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } - List
tables = db.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (Table table : tables) { if (table.isCloudNativeTableOrMaterializedView()) { updateLakeTableTabletStat(db, (OlapTable) table); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/UniqueConstraint.java b/fe/fe-core/src/main/java/com/starrocks/catalog/UniqueConstraint.java index 36bee578e307e..8b43f6bc219fd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/UniqueConstraint.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/UniqueConstraint.java @@ -22,7 +22,6 @@ import com.starrocks.common.Pair; import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.SemanticException; -import com.starrocks.sql.common.MetaUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -65,7 +64,10 @@ public List getUniqueColumnNames() { if (referencedTable != null) { targetTable = referencedTable; } else { - targetTable = MetaUtils.getTable(catalogName, dbName, tableName); + targetTable = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(catalogName, dbName, tableName); + if (targetTable == null) { + throw new SemanticException("Table %s is not found", tableName); + } } List result = new ArrayList<>(uniqueColumns.size()); for (ColumnId columnId : uniqueColumns) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/GrantsTo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/GrantsTo.java index 8bfe2cab8e06a..7dceee92b4737 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/GrantsTo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/GrantsTo.java @@ -193,7 +193,8 @@ private static Set getGrantItems( } else { Database database; if (CatalogMgr.isInternalCatalog(catalogName)) { - database = GlobalStateMgr.getCurrentState().getDb(Long.parseLong(dbPEntryObject.getUUID())); + database = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getDb(Long.parseLong(dbPEntryObject.getUUID())); } else { String dbName = ExternalCatalog.getDbNameFromUUID(dbPEntryObject.getUUID()); database = metadataMgr.getDb(catalogName, dbName); @@ -241,7 +242,7 @@ private static Set getGrantItems( } else { Database database; if (CatalogMgr.isInternalCatalog(tablePEntryObject.getCatalogId())) { - database = GlobalStateMgr.getCurrentState() + database = GlobalStateMgr.getCurrentState().getLocalMetastore() .getDb(Long.parseLong(tablePEntryObject.getDatabaseUUID())); } else { String dbName = ExternalCatalog.getDbNameFromUUID(tablePEntryObject.getDatabaseUUID()); @@ -261,7 +262,8 @@ private static Set getGrantItems( objects.addAll(expandAllTables(metadataMgr, catalogName, dbName, privEntry.getKey())); } else { if (CatalogMgr.isInternalCatalog(tablePEntryObject.getCatalogId())) { - Table table = database.getTable((Long.parseLong(tablePEntryObject.getTableUUID()))); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(database.getId(), (Long.parseLong(tablePEntryObject.getTableUUID()))); if (table == null) { continue; } @@ -319,7 +321,7 @@ private static Set getGrantItems( if (databaseId == PrivilegeBuiltinConstants.ALL_DATABASE_ID) { List dbNames = metadataMgr.listDbNames(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME); for (String dbName : dbNames) { - Database database = GlobalStateMgr.getCurrentState().getDb(dbName); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (database == null) { continue; } @@ -333,7 +335,7 @@ private static Set getGrantItems( } } } else { - Database database = GlobalStateMgr.getCurrentState().getDb(databaseId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(databaseId); if (database == null) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/SysObjectDependencies.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/SysObjectDependencies.java index 3cb8800a47fa2..493572393ad09 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/SysObjectDependencies.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/sys/SysObjectDependencies.java @@ -85,7 +85,7 @@ public static TObjectDependencyRes listObjectDependencies(TObjectDependencyReq r .orElse(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME); locker.lockDatabase(db, LockType.READ); try { - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { // If it is not a materialized view, we do not need to verify permissions if (!table.isMaterializedView()) { continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java index 31f9e28384e80..8043546a9c66b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java @@ -376,7 +376,7 @@ private void scheduleDynamicPartition() { } public boolean executeDynamicPartitionForTable(Long dbId, Long tableId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.warn("Automatically removes the schedule because database does not exist, dbId: {}", dbId); return true; @@ -387,7 +387,7 @@ public boolean executeDynamicPartitionForTable(Long dbId, Long tableId) { String tableName; boolean skipAddPartition = false; OlapTable olapTable; - olapTable = (OlapTable) db.getTable(tableId); + olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (olapTable == null) { LOG.warn("Automatically removes the schedule because table does not exist, " + "tableId: {}", tableId); @@ -488,13 +488,13 @@ private void scheduleTTLPartition() { Pair tableInfo = iterator.next(); Long dbId = tableInfo.first; Long tableId = tableInfo.second; - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { iterator.remove(); LOG.warn("Could not get database={} info. remove it from scheduler", dbId); continue; } - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); OlapTable olapTable; if (table instanceof OlapTable) { olapTable = (OlapTable) table; @@ -686,7 +686,7 @@ private void findSchedulableTables() { Map> ttlPartitionTables = new HashMap<>(); long start = System.currentTimeMillis(); for (Long dbId : GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds()) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } @@ -697,7 +697,7 @@ private void findSchedulableTables() { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - for (Table table : GlobalStateMgr.getCurrentState().getDb(dbId).getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(dbId)) { if (DynamicPartitionUtil.isDynamicPartitionTable(table)) { registerDynamicPartitionTable(db.getId(), table.getId()); dynamicPartitionTables.computeIfAbsent(db.getFullName(), k -> new ArrayList<>()) diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java index 9bcb5417424c7..55f3ce0920044 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java @@ -507,7 +507,7 @@ private void cleanInvalidUrgentTable() { while (iter.hasNext()) { Map.Entry>> dbEntry = iter.next(); long dbId = dbEntry.getKey(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { iter.remove(); continue; @@ -518,7 +518,7 @@ private void cleanInvalidUrgentTable() { try { for (Map.Entry> tblEntry : dbEntry.getValue().entrySet()) { long tblId = tblEntry.getKey(); - OlapTable tbl = (OlapTable) db.getTable(tblId); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tblId); if (tbl == null) { deletedUrgentTable.add(Pair.create(dbId, tblId)); continue; @@ -609,7 +609,7 @@ public List> getUrgentTableInfo() { public static RepairTabletInfo getRepairTabletInfo(String dbName, String tblName, List partitions) throws DdlException { GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exist"); } @@ -620,7 +620,7 @@ public static RepairTabletInfo getRepairTabletInfo(String dbName, String tblName Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table tbl = db.getTable(tblName); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); if (tbl == null || tbl.getType() != TableType.OLAP) { throw new DdlException("Table does not exist or is not OLAP table: " + tblName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java index 366be12efd64b..4cf74fff3f3c1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java @@ -1283,7 +1283,7 @@ public boolean checkPrivForCurrUser(UserIdentity currentUser) { return true; } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { return true; } @@ -1291,7 +1291,7 @@ public boolean checkPrivForCurrUser(UserIdentity currentUser) { Locker locker = new Locker(); try { locker.lockDatabase(db, LockType.READ); - Table table = db.getTable(tblId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tblId); if (table == null) { return true; } else { diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java index 16b124b84ef64..422dcae37c416 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java @@ -1521,7 +1521,7 @@ private List filterUnschedulableTablets(List alt long tabletId = schedCtx.getTabletId(); long indexId = schedCtx.getIndexId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } @@ -1530,7 +1530,7 @@ private List filterUnschedulableTablets(List alt Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - tbl = db.getTable(tableId); + tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); } finally { locker.unLockDatabase(db, LockType.READ); } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/DbsProcDir.java index 730876f6650f2..6711281ca855d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/DbsProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/DbsProcDir.java @@ -83,9 +83,9 @@ public ProcNodeInterface lookup(String dbIdOrName) throws AnalysisException { Database db; try { - db = globalStateMgr.getDb(Long.parseLong(dbIdOrName)); + db = globalStateMgr.getLocalMetastore().getDb(Long.parseLong(dbIdOrName)); } catch (NumberFormatException e) { - db = globalStateMgr.getDb(dbIdOrName); + db = globalStateMgr.getLocalMetastore().getDb(dbIdOrName); } if (db == null) { @@ -110,7 +110,7 @@ public ProcResult fetchResult() throws AnalysisException { // get info List> dbInfos = new ArrayList>(); for (String dbName : dbNames) { - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { continue; } @@ -118,7 +118,7 @@ public ProcResult fetchResult() throws AnalysisException { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - int tableNum = db.getTables().size(); + int tableNum = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()).size(); dbInfo.add(db.getId()); dbInfo.add(dbName); dbInfo.add(tableNum); diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/JobsDbProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/JobsDbProcDir.java index 15964d022653c..ac337b04ea97c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/JobsDbProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/JobsDbProcDir.java @@ -76,7 +76,7 @@ public ProcNodeInterface lookup(String dbIdStr) throws AnalysisException { throw new AnalysisException("Invalid db id format: " + dbIdStr); } - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { throw new AnalysisException("Database[" + dbId + "] does not exist."); } @@ -98,7 +98,7 @@ public ProcResult fetchResult() throws AnalysisException { } for (String name : names) { - Database db = globalStateMgr.getDb(name); + Database db = globalStateMgr.getLocalMetastore().getDb(name); result.addRow(Lists.newArrayList(String.valueOf(db.getId()), name)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/ProcUtils.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/ProcUtils.java index bc4f73bc151ac..290c1f37ea233 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/ProcUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/ProcUtils.java @@ -24,7 +24,7 @@ static long getDbId(String dbIdOrName) throws AnalysisException { try { return Long.parseLong(dbIdOrName); } catch (NumberFormatException e) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbIdOrName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbIdOrName); if (db == null) { throw new AnalysisException("Unknown database id or name \"" + dbIdOrName + "\""); } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java index 64e79600ddabd..2a246ba4faf6c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java @@ -125,7 +125,7 @@ public ProcResult fetchResult() throws AnalysisException { // skip information_schema database continue; } - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { continue; } @@ -141,7 +141,7 @@ public ProcResult fetchResult() throws AnalysisException { int dbTabletNum = 0; int dbReplicaNum = 0; - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (!table.isNativeTableOrMaterializedView()) { continue; } @@ -259,7 +259,7 @@ public ProcNodeInterface lookup(String dbIdStr) throws AnalysisException { throw new AnalysisException("Invalid db id format: " + dbIdStr); } - if (globalStateMgr.getDb(dbId) == null) { + if (globalStateMgr.getLocalMetastore().getDb(dbId) == null) { throw new AnalysisException("Invalid db id: " + dbIdStr); } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/TablesProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/TablesProcDir.java index adb5fa8b4ebcf..8af4afb983ab5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/TablesProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/TablesProcDir.java @@ -52,6 +52,7 @@ import com.starrocks.common.util.TimeUtils; import com.starrocks.common.util.concurrent.lock.LockType; import com.starrocks.common.util.concurrent.lock.Locker; +import com.starrocks.server.GlobalStateMgr; import java.util.ArrayList; import java.util.Collections; @@ -95,9 +96,9 @@ public ProcNodeInterface lookup(String tableIdOrName) throws AnalysisException { locker.lockDatabase(db, LockType.READ); try { try { - table = db.getTable(Long.parseLong(tableIdOrName)); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), Long.parseLong(tableIdOrName)); } catch (NumberFormatException e) { - table = db.getTable(tableIdOrName); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableIdOrName); } } finally { locker.unLockDatabase(db, LockType.READ); @@ -119,7 +120,7 @@ public ProcResult fetchResult() throws AnalysisException { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { List tableInfo = new ArrayList(); TableType tableType = table.getType(); tableInfo.add(table.getId()); diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/SmallFileMgr.java b/fe/fe-core/src/main/java/com/starrocks/common/util/SmallFileMgr.java index c8c08f5f9d55c..084327f1eac1c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/SmallFileMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/SmallFileMgr.java @@ -197,7 +197,7 @@ public SmallFileMgr() { public void createFile(CreateFileStmt stmt) throws DdlException { String dbName = stmt.getDbName(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exist"); } @@ -207,7 +207,7 @@ public void createFile(CreateFileStmt stmt) throws DdlException { public void dropFile(DropFileStmt stmt) throws DdlException { String dbName = stmt.getDbName(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exist"); } @@ -496,7 +496,7 @@ private File getAbsoluteFile(long dbId, String catalog, String fileName) { } public List> getInfo(String dbName) throws DdlException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/elasticsearch/EsRepository.java b/fe/fe-core/src/main/java/com/starrocks/connector/elasticsearch/EsRepository.java index 235c95a253e33..da7e5eceaaeab 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/elasticsearch/EsRepository.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/elasticsearch/EsRepository.java @@ -35,7 +35,6 @@ package com.starrocks.connector.elasticsearch; import com.google.common.collect.Maps; -import com.starrocks.catalog.Database; import com.starrocks.catalog.EsTable; import com.starrocks.catalog.Table; import com.starrocks.catalog.Table.TableType; @@ -119,9 +118,7 @@ public void loadTableFromCatalog() { } List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : dbIds) { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); - - List
tables = database.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(dbId); for (Table table : tables) { if (table.getType() == TableType.ELASTICSEARCH) { registerTable((EsTable) table); diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java index 5b0752e9cf328..e79d76c371150 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java @@ -184,12 +184,12 @@ private void refreshResourceHiveTable() { GlobalStateMgr gsm = GlobalStateMgr.getCurrentState(); MetadataMgr metadataMgr = gsm.getMetadataMgr(); List databases = gsm.getLocalMetastore().getDbIds().stream() - .map(gsm::getDb) + .map(dbId -> GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId)) .filter(Objects::nonNull) .filter(db -> !db.isSystemDatabase()) .collect(Collectors.toList()); for (Database db : databases) { - List tables = db.getTables().stream() + List tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()).stream() .filter(tbl -> tbl.getType() == Table.TableType.HIVE) .map(tbl -> (HiveTable) tbl) .collect(Collectors.toList()); @@ -199,7 +199,7 @@ private void refreshResourceHiveTable() { "in the background", db.getFullName(), table.getName(), table.getDbName(), table.getTableName()); // we didn't use db locks to prevent background tasks from affecting the query. // So we need to check if the table to be refreshed exists. - if (db.getTable(table.getId()) != null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), table.getId()) != null) { metadataMgr.refreshTable(table.getCatalogName(), db.getFullName(), table, Lists.newArrayList(), false); } diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java index 7e42d9d2512ff..21bc56346f029 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java @@ -128,13 +128,13 @@ public boolean sendTasks() { return false; } - Database db = GlobalStateMgr.getCurrentState().getDb(tabletMeta.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tabletMeta.getDbId()); if (db == null) { LOG.debug("db[{}] does not exist", tabletMeta.getDbId()); return false; } - Table table = db.getTable(tabletMeta.getTableId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tabletMeta.getTableId()); if (table == null) { LOG.debug("table[{}] does not exist", tabletMeta.getTableId()); return false; @@ -254,13 +254,13 @@ public synchronized int tryFinishJob() { return -1; } - Database db = GlobalStateMgr.getCurrentState().getDb(tabletMeta.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tabletMeta.getDbId()); if (db == null) { LOG.warn("db[{}] does not exist", tabletMeta.getDbId()); return -1; } - Table table = db.getTable(tabletMeta.getTableId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tabletMeta.getTableId()); if (table == null) { LOG.warn("table[{}] does not exist", tabletMeta.getTableId()); return -1; diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java index 9aaf27a0ecab1..7f4f0c8067576 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java @@ -284,7 +284,7 @@ protected List chooseTablets() { // skip 'information_schema' database continue; } - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { continue; } @@ -301,7 +301,7 @@ protected List chooseTablets() { long startTime = System.currentTimeMillis(); try { // sort tables - List
tables = db.getTables(); + List
tables = globalStateMgr.getLocalMetastore().getTables(db.getId()); Queue tableQueue = new PriorityQueue<>(Math.max(tables.size(), 1), COMPARATOR); for (Table table : tables) { // Only check the OLAP table who is in NORMAL state. @@ -416,12 +416,13 @@ public void handleFinishedConsistencyCheck(CheckConsistencyTask task, long check } public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateMgr globalStateMgr) { - Database db = globalStateMgr.getDb(info.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(info.getDbId()); if (db == null) { LOG.warn("replay finish consistency check failed, db is null, info: {}", info); return; } - OlapTable table = (OlapTable) db.getTable(info.getTableId()); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), info.getTableId()); if (table == null) { LOG.warn("replay finish consistency check failed, table is null, info: {}", info); return; diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/MetaRecoveryDaemon.java b/fe/fe-core/src/main/java/com/starrocks/consistency/MetaRecoveryDaemon.java index a1010d9a326a1..fef99e8addecd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/MetaRecoveryDaemon.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/MetaRecoveryDaemon.java @@ -71,7 +71,7 @@ public void recover() { List partitionsToRecover = new ArrayList<>(); List dbIds = stateMgr.getLocalMetastore().getDbIds(); for (long dbId : dbIds) { - Database database = stateMgr.getDb(dbId); + Database database = stateMgr.getLocalMetastore().getDb(dbId); if (database == null || database.isSystemDatabase()) { continue; } @@ -79,7 +79,7 @@ public void recover() { Locker locker = new Locker(); locker.lockDatabase(database, LockType.READ); try { - for (Table table : database.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(database.getId())) { if (!table.isOlapTableOrMaterializedView()) { continue; } @@ -186,7 +186,7 @@ public void recover() { public void recoverPartitionVersion(PartitionVersionRecoveryInfo recoveryInfo) { for (PartitionVersion version : recoveryInfo.getPartitionVersions()) { - Database database = GlobalStateMgr.getCurrentState().getDb(version.getDbId()); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(version.getDbId()); if (database == null) { LOG.warn("recover partition version failed, db is null, versionInfo: {}", version); continue; @@ -194,7 +194,8 @@ public void recoverPartitionVersion(PartitionVersionRecoveryInfo recoveryInfo) { Locker locker = new Locker(); locker.lockDatabase(database, LockType.WRITE); try { - Table table = database.getTable(version.getTableId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(database.getId(), version.getTableId()); if (table == null) { LOG.warn("recover partition version failed, table is null, versionInfo: {}", version); continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/CancelStreamLoad.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/CancelStreamLoad.java index 593ecccb70605..8ed36633cf0be 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/CancelStreamLoad.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/CancelStreamLoad.java @@ -76,7 +76,7 @@ public void executeWithoutPassword(BaseRequest request, BaseResponse response) t // FIXME(cmy) // checkWritePriv(authInfo.fullUserName, fullDbName); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("unknown database, database=" + dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/GetDdlStmtAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/GetDdlStmtAction.java index a800f9f09200f..3602631e6fe75 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/GetDdlStmtAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/GetDdlStmtAction.java @@ -90,7 +90,7 @@ public void executeWithoutPassword(BaseRequest request, BaseResponse response) throw new DdlException("Missing params. Need database name and Table name"); } - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database[" + dbName + "] does not exist"); } @@ -102,7 +102,7 @@ public void executeWithoutPassword(BaseRequest request, BaseResponse response) Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new DdlException("Table[" + tableName + "] does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/GetStreamLoadState.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/GetStreamLoadState.java index 9e08246d15776..4d4eb2d36ec74 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/GetStreamLoadState.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/GetStreamLoadState.java @@ -76,7 +76,7 @@ public void executeWithoutPassword(BaseRequest request, BaseResponse response) // FIXME(cmy) // checkReadPriv(authInfo.fullUserName, fullDbName); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("unknown database, database=" + dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java index ede3c0102818b..0b822f6182460 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/MigrationAction.java @@ -96,7 +96,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response throw new DdlException("Missing params. Need database name"); } - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database[" + dbName + "] does not exist"); } @@ -106,7 +106,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response locker.lockDatabase(db, LockType.READ); try { if (!Strings.isNullOrEmpty(tableName)) { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new DdlException("Table[" + tableName + "] does not exist"); } @@ -135,7 +135,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response } } else { // get all olap table - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table.getType() != TableType.OLAP) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/RowCountAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/RowCountAction.java index dc70880fbfd28..50fb2b2d9a236 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/RowCountAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/RowCountAction.java @@ -92,11 +92,11 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response Map indexRowCountMap = Maps.newHashMap(); GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database[" + dbName + "] does not exist"); } - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new DdlException("Table[" + tableName + "] does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowDataAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowDataAction.java index 5938ffacedd29..df67df560deb2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowDataAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowDataAction.java @@ -69,7 +69,7 @@ public long getDataSizeOfDatabase(Database db) { locker.lockDatabase(db, LockType.READ); try { // sort by table name - List
tables = db.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (Table table : tables) { if (!table.isNativeTableOrMaterializedView()) { continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowMetaInfoAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowMetaInfoAction.java index 69fb11dcd89b5..0b276612b5d97 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowMetaInfoAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/ShowMetaInfoAction.java @@ -173,10 +173,10 @@ public Map getDataSize() { for (int i = 0; i < dbNames.size(); i++) { String dbName = dbNames.get(i); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); long totalSize = 0; - List
tables = db.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (int j = 0; j < tables.size(); j++) { Table table = tables.get(j); if (table.getType() != TableType.OLAP) { diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/StorageTypeCheckAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/StorageTypeCheckAction.java index 5af38ba28b190..fd98300045ec8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/StorageTypeCheckAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/StorageTypeCheckAction.java @@ -49,6 +49,7 @@ import com.starrocks.http.IllegalArgException; import com.starrocks.privilege.AccessDeniedException; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.UserIdentity; import com.starrocks.thrift.TStorageType; import io.netty.handler.codec.http.HttpMethod; @@ -77,7 +78,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response throw new DdlException("Parameter db is missing"); } - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exist"); } @@ -86,7 +87,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - List
tbls = db.getTables(); + List
tbls = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (Table tbl : tbls) { if (tbl.getType() != TableType.OLAP) { continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/TableQueryPlanAction.java index 4c8b3a8a492bd..1970aa086125c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/TableQueryPlanAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/TableQueryPlanAction.java @@ -150,7 +150,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response // check privilege for select, otherwise return HTTP 401 Authorizer.checkTableAction(ConnectContext.get().getCurrentUserIdentity(), ConnectContext.get().getCurrentRoleIds(), dbName, tableName, PrivilegeType.SELECT); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new StarRocksHttpException(HttpResponseStatus.NOT_FOUND, "Database [" + dbName + "] " + "does not exists"); @@ -159,7 +159,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new StarRocksHttpException(HttpResponseStatus.NOT_FOUND, "Table [" + tableName + "] " + "does not exists"); diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/TableRowCountAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/TableRowCountAction.java index cdd6279770a3f..d6ac4fd33bf47 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/TableRowCountAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/TableRowCountAction.java @@ -92,7 +92,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response // check privilege for select, otherwise return HTTP 401 Authorizer.checkTableAction(ConnectContext.get().getCurrentUserIdentity(), ConnectContext.get().getCurrentRoleIds(), dbName, tableName, PrivilegeType.SELECT); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new StarRocksHttpException(HttpResponseStatus.NOT_FOUND, "Database [" + dbName + "] " + "does not exists"); @@ -100,7 +100,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new StarRocksHttpException(HttpResponseStatus.NOT_FOUND, "Table [" + tableName + "] " + "does not exists"); diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/TableSchemaAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/TableSchemaAction.java index 23433556c2733..12c221213a8b2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/TableSchemaAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/TableSchemaAction.java @@ -94,7 +94,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response Authorizer.checkTableAction(ConnectContext.get().getCurrentUserIdentity(), ConnectContext.get().getCurrentRoleIds(), dbName, tableName, PrivilegeType.SELECT); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new StarRocksHttpException(HttpResponseStatus.NOT_FOUND, "Database [" + dbName + "] " + "does not exists"); @@ -102,7 +102,7 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new StarRocksHttpException(HttpResponseStatus.NOT_FOUND, "Table [" + tableName + "] " + "does not exists"); diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/TransactionLoadAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/TransactionLoadAction.java index 4280c3467b4a2..af1024be49a33 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/TransactionLoadAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/TransactionLoadAction.java @@ -263,7 +263,7 @@ private TransactionOperationHandler getTxnOperationHandler(TransactionOperationP if (null == sourceType) { String dbName = params.getDbName(); - Database db = Optional.ofNullable(GlobalStateMgr.getCurrentState().getDb(dbName)) + Database db = Optional.ofNullable(GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName)) .orElseThrow(() -> new UserException(String.format("Database[%s] does not exist.", dbName))); TransactionState txnState = GlobalStateMgr.getCurrentState() diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/TriggerAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/TriggerAction.java index 9cf6d32cc5cb2..43b49ea542ff3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/TriggerAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/TriggerAction.java @@ -78,7 +78,7 @@ public void executeWithoutPassword(BaseRequest request, BaseResponse response) { return; } - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { response.appendContent("Database[" + dbName + "] does not exist"); writeResponse(request, response, HttpResponseStatus.BAD_REQUEST); @@ -94,7 +94,7 @@ public void executeWithoutPassword(BaseRequest request, BaseResponse response) { locker.lockDatabase(db, LockType.READ); try { if (!Strings.isNullOrEmpty(tableName)) { - table = db.getTable(tableName); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); } if (table == null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/BypassWriteTransactionHandler.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/BypassWriteTransactionHandler.java index 8ea909e78fc88..101e531cf1bf1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/BypassWriteTransactionHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/BypassWriteTransactionHandler.java @@ -68,14 +68,15 @@ public ResultWrapper handle(BaseRequest request, BaseResponse response) throws U Body requestBody = txnOperationParams.getBody(); LOG.info("Handle bypass write transaction, label: {}", label); - Database db = Optional.ofNullable(GlobalStateMgr.getCurrentState().getDb(dbName)) + Database db = Optional.ofNullable(GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName)) .orElseThrow(() -> new UserException(String.format("Database[%s] does not exist.", dbName))); TransactionResult result; switch (txnOperation) { case TXN_BEGIN: - Table table = Optional.ofNullable(db.getTable(tableName)) - .orElseThrow(() -> new UserException( + Table table = Optional.ofNullable(GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName)) + .orElseThrow(() -> new UserException( String.format("Table[%s.%s] does not exist.", dbName, tableName))); result = handleBeginTransaction(db, table, label, sourceType, timeoutMillis); break; diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithoutChannelHandler.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithoutChannelHandler.java index a52e7bb133098..17e9b13563390 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithoutChannelHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithoutChannelHandler.java @@ -64,7 +64,7 @@ public ResultWrapper handle(BaseRequest request, BaseResponse response) throws U Long timeoutMillis = txnOperationParams.getTimeoutMillis(); LOG.info("Handle transaction without channel info, label: {}", label); - Database db = Optional.ofNullable(GlobalStateMgr.getCurrentState().getDb(dbName)) + Database db = Optional.ofNullable(GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName)) .orElseThrow(() -> new UserException(String.format("Database[%s] does not exist.", dbName))); TransactionResult result = null; diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/TableBaseAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/TableBaseAction.java index 81d725210a47d..33ad04484e392 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/TableBaseAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/v2/TableBaseAction.java @@ -88,7 +88,7 @@ protected T getAndApplyOlapTable(String catalogName, PrivilegeType.SELECT ); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (null == db || !Optional.ofNullable(db.getCatalogName()) .orElse(DEFAULT_INTERNAL_CATALOG_NAME).equalsIgnoreCase(catalogName)) { throw new StarRocksHttpException( @@ -99,7 +99,7 @@ protected T getAndApplyOlapTable(String catalogName, final Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (null == table) { throw new StarRocksHttpException( HttpResponseStatus.NOT_FOUND, ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(tableName) diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java b/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java index 4f6417147d048..59e931d5ca666 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java @@ -262,7 +262,7 @@ public int deleteUnusedWorker() { public void syncTableMetaAndColocationInfo() { List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } @@ -270,7 +270,7 @@ public void syncTableMetaAndColocationInfo() { continue; } - List
tables = db.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (Table table : tables) { if (!table.isCloudNativeTableOrMaterializedView()) { continue; @@ -293,7 +293,7 @@ public boolean syncTableMetaInternal(Database db, OlapTable table, boolean force Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - if (db.getTable(table.getId()) == null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), table.getId()) == null) { return false; // table might be dropped } GlobalStateMgr.getCurrentState().getLocalMetastore() @@ -375,10 +375,10 @@ private void syncTableColocationInfo(Database db, OlapTable table) throws DdlExc locker.lockDatabase(db, LockType.WRITE); try { // check db and table again - if (GlobalStateMgr.getCurrentState().getDb(db.getId()) == null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(db.getId()) == null) { return; } - if (db.getTable(table.getId()) == null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), table.getId()) == null) { return; } GlobalStateMgr.getCurrentState().getColocateTableIndex().updateLakeTableColocationInfo(table, true /* isJoin */, @@ -407,12 +407,12 @@ protected void runAfterCatalogReady() { } public void syncTableMeta(String dbName, String tableName, boolean forceDeleteData) throws DdlException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException(String.format("db %s does not exist.", dbName)); } - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new DdlException(String.format("table %s does not exist.", tableName)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeBackupJob.java b/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeBackupJob.java index 6d2e737afed1f..1004bb9d82490 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeBackupJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeBackupJob.java @@ -82,7 +82,7 @@ public LakeBackupJob(String label, long dbId, String dbName, List tabl protected void checkBackupTables(Database db) { for (TableRef tableRef : tableRefs) { String tblName = tableRef.getName().getTbl(); - Table tbl = db.getTable(tblName); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); if (tbl == null) { status = new Status(Status.ErrCode.NOT_FOUND, "table " + tblName + " does not exist"); return; diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java index 86bc5dc5edda1..fde1e0fb051ca 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/backup/LakeRestoreJob.java @@ -124,7 +124,8 @@ protected void prepareAndSendSnapshotTasks(Database db) { for (IdChain idChain : fileMapping.getMapping().keySet()) { LakeTablet tablet = null; try { - OlapTable tbl = (OlapTable) db.getTable(idChain.getTblId()); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), idChain.getTblId()); Partition part = tbl.getPartition(idChain.getPartId()); MaterializedIndex index = part.getIndex(idChain.getIdxId()); tablet = (LakeTablet) index.getTablet(idChain.getTabletId()); @@ -154,7 +155,8 @@ protected void prepareDownloadTasks(List beSnapshotInfos, Database } request.restoreInfos = Lists.newArrayList(); for (SnapshotInfo info : beSnapshotInfos) { - OlapTable tbl = (OlapTable) db.getTable(info.getTblId()); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), info.getTblId()); if (tbl == null) { status = new Status(Status.ErrCode.NOT_FOUND, "restored table " + info.getTblId() + " does not exist"); @@ -289,7 +291,8 @@ protected void modifyInvertedIndex(OlapTable restoreTbl, Partition restorePart) @Override protected void addRestoredPartitions(Database db, boolean modify) { for (Pair entry : restoredPartitions) { - OlapTable localTbl = (OlapTable) db.getTable(entry.first); + OlapTable localTbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), entry.first); Partition restorePart = entry.second; OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); RangePartitionInfo localPartitionInfo = (RangePartitionInfo) localTbl.getPartitionInfo(); diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java b/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java index 8128e156d3d69..3f318fbe224da 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java @@ -256,7 +256,7 @@ private void cleanPartition() { } private CompactionJob startCompaction(PartitionIdentifier partitionIdentifier) { - Database db = stateMgr.getDb(partitionIdentifier.getDbId()); + Database db = stateMgr.getLocalMetastore().getDb(partitionIdentifier.getDbId()); if (db == null) { compactionManager.removePartition(partitionIdentifier); return null; @@ -273,7 +273,8 @@ private CompactionJob startCompaction(PartitionIdentifier partitionIdentifier) { try { // lake table or lake materialized view - table = (OlapTable) db.getTable(partitionIdentifier.getTableId()); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), partitionIdentifier.getTableId()); // Compact a table of SCHEMA_CHANGE state does not make much sense, because the compacted data // will not be used after the schema change job finished. if (table != null && table.getState() == OlapTable.OlapTableState.SCHEMA_CHANGE) { @@ -401,7 +402,7 @@ private void commitCompaction(PartitionIdentifier partition, CompactionJob job, throws UserException { List commitInfoList = job.buildTabletCommitInfo(); - Database db = stateMgr.getDb(partition.getDbId()); + Database db = stateMgr.getLocalMetastore().getDb(partition.getDbId()); if (db == null) { throw new MetaNotFoundException("database not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java b/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java index 61f3419a6b0ae..04037e61d52c2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java @@ -70,13 +70,13 @@ public AutovacuumDaemon() { protected void runAfterCatalogReady() { List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } List
tables = new ArrayList<>(); - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table.isCloudNativeTableOrMaterializedView()) { tables.add(table); } diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java index a65d7ff1797a3..2d0b6c217bf78 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java @@ -473,12 +473,13 @@ private void finishUpdateSchemaTask(AgentTask task, TFinishTaskRequest request) long tableId = task.getTableId(); long indexId = task.getIndexId(); long backendId = task.getBackendId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (olapTable != null) { MaterializedIndexMeta indexMeta = olapTable.getIndexMetaByIndexId(indexId); if (indexMeta != null) { @@ -504,7 +505,7 @@ private void finishRealtimePush(AgentTask task, TFinishTaskRequest request) { long backendId = pushTask.getBackendId(); long signature = task.getSignature(); long transactionId = ((PushTask) task).getTransactionId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { AgentTaskQueue.removeTask(backendId, TTaskType.REALTIME_PUSH, signature); return; @@ -539,7 +540,7 @@ private void finishRealtimePush(AgentTask task, TFinishTaskRequest request) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (olapTable == null) { throw new MetaNotFoundException("cannot find table[" + tableId + "] when push finished"); } @@ -771,7 +772,7 @@ private void finishStorageMigration(AgentTask task, TFinishTaskRequest request) } long dbId = tabletMeta.getDbId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.warn("db does not exist. db id: {}", dbId); return; @@ -892,7 +893,7 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { } // checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { TStatus status = new TStatus(TStatusCode.NOT_FOUND); status.setError_msgs(Lists.newArrayList("db not exist")); @@ -904,7 +905,7 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { try { locker.lockDatabase(db, LockType.READ); - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { TStatus status = new TStatus(TStatusCode.NOT_FOUND); status.setError_msgs(Lists.newArrayList("table " + tableName + " not exist")); @@ -1214,7 +1215,7 @@ public TBeginRemoteTxnResponse beginRemoteTxn(TBeginRemoteTxnRequest request) th return response; } - Database db = globalStateMgr.getDb(request.getDb_id()); + Database db = globalStateMgr.getLocalMetastore().getDb(request.getDb_id()); if (db == null) { TStatus status = new TStatus(TStatusCode.NOT_FOUND); status.setError_msgs(Lists.newArrayList("db not exist")); @@ -1273,7 +1274,7 @@ public TCommitRemoteTxnResponse commitRemoteTxn(TCommitRemoteTxnRequest request) return response; } - Database db = globalStateMgr.getDb(request.getDb_id()); + Database db = globalStateMgr.getLocalMetastore().getDb(request.getDb_id()); if (db == null) { TStatus status = new TStatus(TStatusCode.NOT_FOUND); status.setError_msgs(Lists.newArrayList("db not exist or already deleted")); @@ -1339,7 +1340,7 @@ public TAbortRemoteTxnResponse abortRemoteTxn(TAbortRemoteTxnRequest request) th return response; } - Database db = globalStateMgr.getDb(request.getDb_id()); + Database db = globalStateMgr.getLocalMetastore().getDb(request.getDb_id()); if (db == null) { TStatus status = new TStatus(TStatusCode.NOT_FOUND); status.setError_msgs(Lists.newArrayList("db not exist or already deleted")); diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java index f2d78bbe64a9e..3c88d91e30e1a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java @@ -1229,11 +1229,12 @@ protected static void handleMigration(ListMultimap tabletM } // 3. There are some limitations for primary table, details in migratableTablet() - Database db = GlobalStateMgr.getCurrentState().getDb(tabletMeta.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tabletMeta.getDbId()); if (db == null) { continue; } - OlapTable table = (OlapTable) db.getTable(tabletMeta.getTableId()); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tabletMeta.getTableId()); if (table == null) { continue; } @@ -1292,7 +1293,7 @@ private static void handleRecoverTablet(ListMultimap tabletRecoveryM BackendTabletsInfo backendTabletsInfo = new BackendTabletsInfo(backendId); backendTabletsInfo.setBad(true); for (Long dbId : tabletRecoveryMap.keySet()) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } @@ -1308,7 +1309,8 @@ private static void handleRecoverTablet(ListMultimap tabletRecoveryM } long tabletId = tabletIds.get(i); long tableId = tabletMeta.getTableId(); - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (olapTable == null) { continue; } @@ -1388,12 +1390,13 @@ private static void handleSetTabletInMemory(long backendId, Map b long partitionId = tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (olapTable == null) { continue; } @@ -1449,12 +1452,13 @@ private static void handleSetTabletEnablePersistentIndex(long backendId, Map b long tableId = tabletMeta.getTableId(); long indexId = tabletMeta.getIndexId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (olapTable == null) { continue; } @@ -1611,12 +1617,12 @@ private static void handleUpdateTableSchema(long backendId, Map b List tablets = cell.getValue(); Long dbId = tableToDb.get(tableId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (olapTable == null) { continue; } @@ -1682,13 +1688,14 @@ private static void handleSetTabletBinlogConfig(long backendId, Map tables = transactionState.getIdToTableCommitInfos().values().stream() .map(x -> x.getTableId()) .distinct() - .map(db::getTable) + .map(tableId -> GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(dbId, tableId)) .filter(Objects::nonNull) .filter(t -> !t.isMaterializedView()) // skip mvs since its stats will be triggered after refresh .collect(Collectors.toList()); diff --git a/fe/fe-core/src/main/java/com/starrocks/load/BrokerFileGroup.java b/fe/fe-core/src/main/java/com/starrocks/load/BrokerFileGroup.java index dd0214f3be56b..c539112f8f333 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/BrokerFileGroup.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/BrokerFileGroup.java @@ -59,6 +59,7 @@ import com.starrocks.common.Pair; import com.starrocks.common.io.Text; import com.starrocks.common.io.Writable; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.DataDescription; import com.starrocks.sql.ast.ImportColumnDesc; import com.starrocks.sql.ast.PartitionNames; @@ -168,7 +169,8 @@ public void parseFormatProperties(DataDescription dataDescription) { // This will parse the input DataDescription to list for BrokerFileInfo public void parse(Database db, DataDescription dataDescription) throws DdlException { // tableId - Table table = db.getTable(dataDescription.getTableName()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), dataDescription.getTableName()); if (table == null) { throw new DdlException("Unknown table " + dataDescription.getTableName() + " in database " + db.getOriginName()); @@ -237,7 +239,7 @@ public void parse(Database db, DataDescription dataDescription) throws DdlExcept if (dataDescription.isLoadFromTable()) { String srcTableName = dataDescription.getSrcTableName(); // src table should be hive table - Table srcTable = db.getTable(srcTableName); + Table srcTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), srcTableName); if (srcTable == null) { throw new DdlException("Unknown table " + srcTableName + " in database " + db.getOriginName()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java index b814aa7418866..de472241445b8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/DeleteMgr.java @@ -176,12 +176,12 @@ public void process(DeleteStmt stmt) throws DdlException, QueryStateException { String dbName = stmt.getTableName().getDb(); String tableName = stmt.getTableName().getTbl(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Db does not exist. name: " + dbName); } - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new DdlException("Table does not exist. name: " + tableName); } @@ -722,7 +722,7 @@ public void updatePredicate(Predicate predicate, Column column, int childNo) thr // show delete stmt public List> getDeleteInfosByDb(long dbId) { LinkedList> infos = new LinkedList>(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { return infos; } @@ -816,11 +816,11 @@ public void replayMultiDelete(MultiDeleteInfo deleteInfo, GlobalStateMgr globalS } public void updateTableDeleteInfo(GlobalStateMgr globalStateMgr, long dbId, long tableId) { - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { return; } - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { return; } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java b/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java index d0e1a76fc22bf..895746176ac94 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/ExportJob.java @@ -240,7 +240,7 @@ public long getWarehouseId() { public void setJob(ExportStmt stmt) throws UserException { String dbName = stmt.getTblName().getDb(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database " + dbName + " does not exist"); } @@ -266,7 +266,8 @@ public void setJob(ExportStmt stmt) throws UserException { this.columnNames = stmt.getColumnNames(); this.dbId = db.getId(); - this.exportTable = db.getTable(stmt.getTblName().getTbl()); + this.exportTable = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), stmt.getTblName().getTbl()); if (exportTable == null) { throw new DdlException("Table " + stmt.getTblName().getTbl() + " does not exist"); } @@ -993,10 +994,10 @@ public void readFields(DataInput in) throws IOException { GlobalStateMgr stateMgr = GlobalStateMgr.getCurrentState(); Database db = null; if (stateMgr.getMetadata() != null) { - db = stateMgr.getDb(dbId); + db = stateMgr.getLocalMetastore().getDb(dbId); } if (db != null) { - exportTable = db.getTable(tableId); + exportTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); } int count = in.readInt(); @@ -1086,10 +1087,10 @@ public void gsonPostProcess() throws IOException { GlobalStateMgr stateMgr = GlobalStateMgr.getCurrentState(); Database db = null; if (stateMgr.getMetadata() != null) { - db = stateMgr.getDb(dbId); + db = stateMgr.getLocalMetastore().getDb(dbId); } if (db != null) { - exportTable = db.getTable(tableId); + exportTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/ExportMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/ExportMgr.java index d2bc0986a7041..b89765c7e4778 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/ExportMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/ExportMgr.java @@ -134,7 +134,7 @@ private ExportJob createJob(long jobId, UUID queryId, ExportStmt stmt) throws Ex } public ExportJob getExportJob(String dbName, UUID queryId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); MetaUtils.checkDbNullAndReport(db, dbName); long dbId = db.getId(); ExportJob matchedJob = null; @@ -234,7 +234,7 @@ public List> getExportJobInfosByIdOrState( TableName tableName = job.getTableName(); if (tableName == null || tableName.getTbl().equals("DUMMY")) { // forward compatibility, no table name is saved before - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java b/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java index ea5b1ceac93f9..f8683c9ab83e2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/InsertOverwriteJobRunner.java @@ -215,7 +215,7 @@ private void prepare() throws Exception { } job.setTmpPartitionIds(tmpPartitionIds); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new DmlException("database id:%s does not exist", dbId); } @@ -292,7 +292,7 @@ private void createPartitionByValue(InsertStmt insertStmt) { } GlobalStateMgr state = GlobalStateMgr.getCurrentState(); String targetDb = insertStmt.getTableName().getDb(); - Database db = state.getDb(targetDb); + Database db = state.getLocalMetastore().getDb(targetDb); List sourcePartitionIds = job.getSourcePartitionIds(); try { AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(olapTable); @@ -344,7 +344,7 @@ private void executeInsert() throws Exception { private void createTempPartitions() throws DdlException { long createPartitionStartTimestamp = System.currentTimeMillis(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new DmlException("database id:%s does not exist", dbId); } @@ -366,7 +366,7 @@ private void createTempPartitions() throws DdlException { private void gc(boolean isReplay) { LOG.info("start to garbage collect"); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new DmlException("database id:%s does not exist", dbId); } @@ -376,7 +376,7 @@ private void gc(boolean isReplay) { } try { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new DmlException("table:%d does not exist in database:%s", tableId, db.getFullName()); } @@ -416,7 +416,7 @@ private void gc(boolean isReplay) { } private void doCommit(boolean isReplay) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new DmlException("database id:%s does not exist", dbId); } @@ -487,7 +487,7 @@ private void prepareInsert() { Preconditions.checkState(job.getJobState() == InsertOverwriteJobState.OVERWRITE_RUNNING); Preconditions.checkState(insertStmt != null); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new DmlException("database id:%s does not exist", dbId); } @@ -522,7 +522,7 @@ private void prepareInsert() { } private OlapTable checkAndGetTable(Database db, long tableId) { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new DmlException("table:% does not exist in database:%s", tableId, db.getFullName()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/Load.java b/fe/fe-core/src/main/java/com/starrocks/load/Load.java index d17b711ee7532..b5c3bd5f874ae 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/Load.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/Load.java @@ -464,7 +464,7 @@ public static void initColumns(Table tbl, List columnExprs, if (GlobalStateMgr.getCurrentState().getLocalMetastore().getIdToDb() != null) { for (Map.Entry entry : GlobalStateMgr.getCurrentState().getLocalMetastore().getIdToDb().entrySet()) { Database db = entry.getValue(); - if (db.getTable(tbl.getId()) != null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tbl.getId()) != null) { dbName = db.getFullName(); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java b/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java index 3d5429a654250..ba270c1963374 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java @@ -298,7 +298,7 @@ public void run(DeleteStmt stmt, Database db, Table table, List parti */ public void checkAndUpdateQuorum() throws MetaNotFoundException { long dbId = deleteInfo.getDbId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("can not find database " + dbId + " when commit delete"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java b/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java index 85cd0456802ae..ecacec80acc97 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/PartitionUtils.java @@ -66,7 +66,7 @@ public static void createAndAddTempPartitionsForTable(Database db, OlapTable tar boolean success = false; try { // should check whether targetTable exists - Table tmpTable = db.getTable(targetTable.getId()); + Table tmpTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), targetTable.getId()); if (tmpTable == null) { throw new DdlException("create partition failed because target table does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BrokerLoadJob.java index b6095eb56a92c..92d4fbb44e0a0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BrokerLoadJob.java @@ -247,7 +247,7 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme FileGroupAggKey aggKey = entry.getKey(); List brokerFileGroups = entry.getValue(); long tableId = aggKey.getTableId(); - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id) .add("database_id", dbId) diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BulkLoadJob.java index 62296bb18e675..ff61e79b2dea2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/BulkLoadJob.java @@ -127,7 +127,7 @@ public BulkLoadJob(long dbId, String label, OriginStatement originStmt) throws M public static BulkLoadJob fromLoadStmt(LoadStmt stmt, ConnectContext context) throws DdlException { // get db id String dbName = stmt.getLabel().getDbName(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new DdlException("Database[" + dbName + "] does not exist"); } @@ -184,7 +184,7 @@ private void checkAndSetDataSourceInfo(Database db, List dataDe } private AuthorizationInfo gatherAuthInfo() throws MetaNotFoundException { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { throw new MetaNotFoundException("Database " + dbId + "has been deleted"); } @@ -194,7 +194,7 @@ private AuthorizationInfo gatherAuthInfo() throws MetaNotFoundException { @Override public Set getTableNamesForShow() { Set result = Sets.newHashSet(); - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { for (long tableId : fileGroupAggInfo.getAllTableIds()) { result.add(String.valueOf(tableId)); @@ -202,7 +202,7 @@ public Set getTableNamesForShow() { return result; } for (long tableId : fileGroupAggInfo.getAllTableIds()) { - Table table = database.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { result.add(String.valueOf(tableId)); } else { @@ -215,7 +215,7 @@ public Set getTableNamesForShow() { @Override public Set getTableNames(boolean noThrow) throws MetaNotFoundException { Set result = Sets.newHashSet(); - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { if (noThrow) { return result; @@ -226,7 +226,7 @@ public Set getTableNames(boolean noThrow) throws MetaNotFoundException { // The database will not be locked in here. // The getTable is a thread-safe method called without read lock of database for (long tableId : fileGroupAggInfo.getAllTableIds()) { - Table table = database.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { if (!noThrow) { throw new MetaNotFoundException("Failed to find table " + tableId + " in db " + dbId); @@ -287,7 +287,7 @@ public void analyze() { for (DataDescription dataDescription : stmt.getDataDescriptions()) { dataDescription.analyzeWithoutCheckPriv(); } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new DdlException("Database[" + dbId + "] does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/InsertLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/InsertLoadJob.java index 10314908e403d..536bbbc3396f9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/InsertLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/InsertLoadJob.java @@ -153,7 +153,7 @@ public void setLoadFinishOrCancel(String failMsg, String trackingUrl) throws Use } public AuthorizationInfo gatherAuthInfo() throws MetaNotFoundException { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { throw new MetaNotFoundException("Database " + dbId + "has been deleted"); } @@ -192,13 +192,13 @@ public void updateProgress(TReportExecStatusParams params) { @Override public Set getTableNamesForShow() { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { return Sets.newHashSet(String.valueOf(tableId)); } // The database will not be locked in here. // The getTable is a thread-safe method called without read lock of database - Table table = database.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { return Sets.newHashSet(String.valueOf(tableId)); } @@ -207,11 +207,11 @@ public Set getTableNamesForShow() { @Override public Set getTableNames(boolean noThrow) throws MetaNotFoundException { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { throw new MetaNotFoundException("Database " + dbId + "has been deleted"); } - Table table = database.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { if (noThrow) { return Sets.newHashSet(); @@ -224,12 +224,12 @@ public Set getTableNames(boolean noThrow) throws MetaNotFoundException { @Override public boolean hasTxn() { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { return true; } - Table table = database.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { return true; } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java index 2c7b56eeb741c..ea285399808b7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java @@ -256,7 +256,7 @@ public void setId(long id) { public Database getDb() throws MetaNotFoundException { // get db - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("Database " + dbId + " already has been deleted"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadLoadingTask.java index 26ce744d6d50f..8e6f80760c02c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadLoadingTask.java @@ -301,7 +301,7 @@ private void checkMeta() throws LoadException { throw new LoadException(String.format("db: %s-%d has been dropped", db.getFullName(), db.getId())); } - if (database.getTable(table.getId()) == null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), table.getId()) == null) { throw new LoadException(String.format("table: %s-%d has been dropped from db: %s-%d", table.getName(), table.getId(), db.getFullName(), db.getId())); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java index 91e59d552d336..e077500673c9e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java @@ -154,7 +154,7 @@ public void createLoadJobFromStmt(LoadStmt stmt, ConnectContext context) throws } public void alterLoadJob(AlterLoadStmt stmt) throws DdlException { - Database db = GlobalStateMgr.getCurrentState().getDb(stmt.getDbName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(stmt.getDbName()); if (db == null) { throw new DdlException("Db does not exist. name: " + stmt.getDbName()); } @@ -237,7 +237,7 @@ public InsertLoadJob registerInsertLoadJob(String label, String dbName, long tab int estimateFileNum, long estimateFileSize, TLoadJobType type, long timeout, Coordinator coordinator) throws UserException { // get db id - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new MetaNotFoundException("Database[" + dbName + "] does not exist"); } @@ -262,7 +262,7 @@ public InsertLoadJob registerInsertLoadJob(String label, String dbName, long tab } public void cancelLoadJob(CancelLoadStmt stmt) throws DdlException { - Database db = GlobalStateMgr.getCurrentState().getDb(stmt.getDbName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(stmt.getDbName()); if (db == null) { throw new DdlException("Db does not exist. name: " + stmt.getDbName()); } @@ -662,7 +662,7 @@ private void analyzeLoadJobs() { private Database checkDb(String dbName) throws DdlException { // get db - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { LOG.warn("Database {} does not exist", dbName); throw new DdlException("Database[" + dbName + "] does not exist"); diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadsHistorySyncer.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadsHistorySyncer.java index 092ecf1cb3d01..3f7cbad563491 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadsHistorySyncer.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadsHistorySyncer.java @@ -88,7 +88,7 @@ public LoadsHistorySyncer() { } public boolean checkDatabaseExists() { - return GlobalStateMgr.getCurrentState().getDb(LOADS_HISTORY_DB_NAME) != null; + return GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(LOADS_HISTORY_DB_NAME) != null; } public static void createTable() throws UserException { @@ -98,9 +98,8 @@ public static void createTable() throws UserException { public static boolean correctTable() { int numBackends = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getTotalBackendNumber(); - int replica = GlobalStateMgr.getCurrentState() - .mayGetDb(LOADS_HISTORY_DB_NAME) - .flatMap(db -> db.mayGetTable(LOADS_HISTORY_TABLE_NAME)) + int replica = GlobalStateMgr.getCurrentState().getLocalMetastore() + .mayGetTable(LOADS_HISTORY_DB_NAME, LOADS_HISTORY_TABLE_NAME) .map(tbl -> ((OlapTable) tbl).getPartitionInfo().getMinReplicationNum()) .orElse((short) 1); if (numBackends < 3) { diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java index 7584be8281817..df06da905f646 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java @@ -492,7 +492,8 @@ private Set submitPushTasks() throws UserException { for (Map.Entry> entry : tableToLoadPartitions.entrySet()) { long tableId = entry.getKey(); - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); if (table == null) { LOG.warn("table does not exist. id: {}", tableId); continue; @@ -821,7 +822,7 @@ private void clearJob() { public void afterVisible(TransactionState txnState, boolean txnOperated) { super.afterVisible(txnState, txnOperated); // collect table-level metrics after spark load job finished - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (null == db) { return; } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadPendingTask.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadPendingTask.java index f8382cb4534d5..64b5078ed6e81 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadPendingTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadPendingTask.java @@ -156,7 +156,7 @@ public void init() throws LoadException { } private void createEtlJobConf() throws LoadException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new LoadException("db does not exist. id: " + dbId); } @@ -173,7 +173,7 @@ private void createEtlJobConf() throws LoadException { FileGroupAggKey aggKey = entry.getKey(); long tableId = aggKey.getTableId(); - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new LoadException("table does not exist. id: " + tableId); } @@ -224,7 +224,7 @@ private void prepareTablePartitionInfos(Database db, Map> tableI continue; } - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new LoadException("table does not exist. id: " + tableId); } @@ -571,7 +571,8 @@ private EtlFileGroup createEtlFileGroup(BrokerFileGroup fileGroup, Set tab Map hiveTableProperties = Maps.newHashMap(); if (fileGroup.isLoadFromTable()) { long srcTableId = fileGroup.getSrcTableId(); - HiveTable srcHiveTable = (HiveTable) db.getTable(srcTableId); + HiveTable srcHiveTable = (HiveTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), srcTableId); if (srcHiveTable == null) { throw new LoadException("table does not exist. id: " + srcTableId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/pipe/Pipe.java b/fe/fe-core/src/main/java/com/starrocks/load/pipe/Pipe.java index aea2bc317f99f..fd9b326a0c288 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/pipe/Pipe.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/pipe/Pipe.java @@ -133,7 +133,7 @@ protected Pipe(PipeId id, String name, TableName targetTable, FilePipeSource sou public static Pipe fromStatement(long id, CreatePipeStmt stmt) { PipeName pipeName = stmt.getPipeName(); - long dbId = GlobalStateMgr.getCurrentState().getDb(pipeName.getDbName()).getId(); + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(pipeName.getDbName()).getId(); PipeId pipeId = new PipeId(dbId, id); Pipe res = new Pipe(pipeId, pipeName.getPipeName(), stmt.getTargetTable(), stmt.getDataSource(), stmt.getInsertSql()); @@ -314,7 +314,7 @@ private void buildNewTasks() { long taskId = GlobalStateMgr.getCurrentState().getNextId(); PipeId pipeId = getPipeId(); String uniqueName = PipeTaskDesc.genUniqueTaskName(getName(), taskId, 0); - String dbName = GlobalStateMgr.getCurrentState().mayGetDb(pipeId.getDbId()) + String dbName = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(pipeId.getDbId()) .map(Database::getOriginName) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_DB_ERROR)); String sqlTask = FilePipeSource.buildInsertSql(this, piece, uniqueName); diff --git a/fe/fe-core/src/main/java/com/starrocks/load/pipe/PipeManager.java b/fe/fe-core/src/main/java/com/starrocks/load/pipe/PipeManager.java index af6e39063c30b..f17b8cc9f25e5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/pipe/PipeManager.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/pipe/PipeManager.java @@ -184,7 +184,7 @@ protected void updatePipe(Pipe pipe) { } private Pair resolvePipeNameUnlock(PipeName name) { - long dbId = GlobalStateMgr.getCurrentState().mayGetDb(name.getDbName()) + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(name.getDbName()) .map(Database::getId) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_NO_DB_ERROR)); return Pair.create(dbId, name.getPipeName()); diff --git a/fe/fe-core/src/main/java/com/starrocks/load/pipe/filelist/RepoCreator.java b/fe/fe-core/src/main/java/com/starrocks/load/pipe/filelist/RepoCreator.java index bfcf8a0cbd746..aea6240a646a2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/pipe/filelist/RepoCreator.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/pipe/filelist/RepoCreator.java @@ -60,7 +60,7 @@ public void run() { } public boolean checkDatabaseExists() { - return GlobalStateMgr.getCurrentState().getDb(FileListTableRepo.FILE_LIST_DB_NAME) != null; + return GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(FileListTableRepo.FILE_LIST_DB_NAME) != null; } public static void createTable() throws UserException { @@ -71,8 +71,7 @@ public static void createTable() throws UserException { public static boolean correctTable() { int numBackends = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getTotalBackendNumber(); int replica = GlobalStateMgr.getCurrentState() - .mayGetDb(FileListTableRepo.FILE_LIST_DB_NAME) - .flatMap(db -> db.mayGetTable(FileListTableRepo.FILE_LIST_TABLE_NAME)) + .getLocalMetastore().mayGetTable(FileListTableRepo.FILE_LIST_DB_NAME, FileListTableRepo.FILE_LIST_TABLE_NAME) .map(tbl -> ((OlapTable) tbl).getPartitionInfo().getMinReplicationNum()) .orElse((short) 1); if (numBackends < 3) { diff --git a/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaRoutineLoadJob.java index e169c80f71b2d..32f7b092d2289 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaRoutineLoadJob.java @@ -466,12 +466,12 @@ private List getAllKafkaPartitions() throws UserException { public static KafkaRoutineLoadJob fromCreateStmt(CreateRoutineLoadStmt stmt) throws UserException { // check db and table - Database db = GlobalStateMgr.getCurrentState().getDb(stmt.getDBName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(stmt.getDBName()); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, stmt.getDBName()); } - Table table = db.getTable(stmt.getTableName()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), stmt.getTableName()); if (table == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, stmt.getTableName()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaTaskInfo.java b/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaTaskInfo.java index 2d6f33c6c7162..da141b9b6e6d3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaTaskInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/routineload/KafkaTaskInfo.java @@ -171,12 +171,12 @@ public TRoutineLoadTask createRoutineLoadTask() throws UserException { tRoutineLoadTask.setId(queryId); tRoutineLoadTask.setJob_id(routineLoadJob.getId()); tRoutineLoadTask.setTxn_id(txnId); - Database database = GlobalStateMgr.getCurrentState().getDb(routineLoadJob.getDbId()); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(routineLoadJob.getDbId()); if (database == null) { throw new MetaNotFoundException("database " + routineLoadJob.getDbId() + " does not exist"); } tRoutineLoadTask.setDb(database.getFullName()); - Table tbl = database.getTable(routineLoadJob.getTableId()); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), routineLoadJob.getTableId()); if (tbl == null) { throw new MetaNotFoundException("table " + routineLoadJob.getTableId() + " does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarRoutineLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarRoutineLoadJob.java index da08aa4367d8d..f4d88d39f5d0e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarRoutineLoadJob.java @@ -412,7 +412,7 @@ private List getAllPulsarPartitions() throws UserException { public static PulsarRoutineLoadJob fromCreateStmt(CreateRoutineLoadStmt stmt) throws UserException { // check db and table - Database db = GlobalStateMgr.getCurrentState().getDb(stmt.getDBName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(stmt.getDBName()); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, stmt.getDBName()); } @@ -422,7 +422,7 @@ public static PulsarRoutineLoadJob fromCreateStmt(CreateRoutineLoadStmt stmt) th locker.lockDatabase(db, LockType.READ); try { unprotectedCheckMeta(db, stmt.getTableName(), stmt.getRoutineLoadDesc()); - Table table = db.getTable(stmt.getTableName()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), stmt.getTableName()); Load.checkMergeCondition(stmt.getMergeConditionStr(), (OlapTable) table, table.getFullSchema(), false); tableId = table.getId(); } finally { diff --git a/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarTaskInfo.java b/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarTaskInfo.java index 33f93053d119d..f4047bf197731 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarTaskInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/routineload/PulsarTaskInfo.java @@ -109,12 +109,12 @@ public TRoutineLoadTask createRoutineLoadTask() throws UserException { tRoutineLoadTask.setId(queryId); tRoutineLoadTask.setJob_id(routineLoadJob.getId()); tRoutineLoadTask.setTxn_id(txnId); - Database database = GlobalStateMgr.getCurrentState().getDb(routineLoadJob.getDbId()); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(routineLoadJob.getDbId()); if (database == null) { throw new MetaNotFoundException("database " + routineLoadJob.getDbId() + " does not exist"); } tRoutineLoadTask.setDb(database.getFullName()); - Table tbl = database.getTable(routineLoadJob.getTableId()); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), routineLoadJob.getTableId()); if (tbl == null) { throw new MetaNotFoundException("table " + routineLoadJob.getTableId() + " does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java index d6929d5e99572..ac5c4c5e590e6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java @@ -512,7 +512,7 @@ public void clearOtherMsg() { } public String getDbFullName() throws MetaNotFoundException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("Database " + dbId + "has been deleted"); } @@ -524,11 +524,11 @@ public long getTableId() { } public String getTableName() throws MetaNotFoundException { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (database == null) { throw new MetaNotFoundException("Database " + dbId + "has been deleted"); } - Table table = database.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { throw new MetaNotFoundException("Failed to find table " + tableId + " in db " + dbId); } @@ -856,12 +856,12 @@ private Coordinator.Factory getCoordinatorFactory() { } public TExecPlanFragmentParams plan(TUniqueId loadId, long txnId, String label) throws UserException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("db " + dbId + " does not exist"); } - Table table = db.getTable(this.tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), this.tableId); if (table == null) { throw new MetaNotFoundException("table " + this.tableId + " does not exist"); } @@ -1287,7 +1287,7 @@ private void executeTaskOnTxnStatusChanged(RoutineLoadTaskInfo routineLoadTaskIn protected static void unprotectedCheckMeta(Database db, String tblName, RoutineLoadDesc routineLoadDesc) throws UserException { - Table table = db.getTable(tblName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); if (table instanceof MaterializedView) { throw new AnalysisException(String.format( @@ -1421,7 +1421,7 @@ private void clearTasks() { public void update() throws UserException { // check if db and table exist - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.warn(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, id) .add("db_id", dbId) @@ -1440,7 +1440,7 @@ public void update() throws UserException { } // check table belong to database - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { LOG.warn(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, id).add("db_id", dbId) .add("table_id", tableId) @@ -1495,13 +1495,13 @@ protected abstract boolean checkCommitInfo(RLTaskTxnCommitAttachment rlTaskTxnCo protected abstract String getStatistic(); public List getShowInfo() { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); Table tbl = null; if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - tbl = db.getTable(tableId); + tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); } finally { locker.unLockDatabase(db, LockType.READ); } @@ -1585,7 +1585,7 @@ public List> getTasksShowInfo() { } public List getShowStatistic() { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); readLock(); try { List row = Lists.newArrayList(); @@ -2000,13 +2000,13 @@ public void gsonPostProcess() throws IOException { } public TRoutineLoadJobInfo toThrift() { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); Table tbl = null; if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - tbl = db.getTable(tableId); + tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); } finally { locker.unLockDatabase(db, LockType.READ); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadMgr.java index 48a3578493741..7b41156b3325d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadMgr.java @@ -508,7 +508,7 @@ public List getJob(String dbFullName, String jobName, boolean in sortRoutineLoadJob(result); } else { long dbId = 0L; - Database database = GlobalStateMgr.getCurrentState().getDb(dbFullName); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbFullName); if (database == null) { throw new MetaNotFoundException("failed to find database by dbFullName " + dbFullName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java index 6ec158077016a..e1b31b4e41eb8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java @@ -175,7 +175,7 @@ public StreamLoadTask createLoadTask(Database db, String tableName, String label locker.lockDatabase(db, LockType.READ); try { unprotectedCheckMeta(db, tableName); - table = db.getTable(tableName); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); } finally { locker.unLockDatabase(db, LockType.READ); } @@ -192,7 +192,8 @@ public StreamLoadTask createLoadTaskWithoutLock(Database db, String tableName, S throws UserException { // init stream load task long id = GlobalStateMgr.getCurrentState().getNextId(); - StreamLoadTask streamLoadTask = new StreamLoadTask(id, db, (OlapTable) db.getTable(tableName), + StreamLoadTask streamLoadTask = new StreamLoadTask(id, db, + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName), label, user, clientIp, timeoutMillis, System.currentTimeMillis(), isRoutineLoad, warehouseId); return streamLoadTask; } @@ -205,7 +206,7 @@ public StreamLoadTask createLoadTask(Database db, String tableName, String label locker.lockDatabase(db, LockType.READ); try { unprotectedCheckMeta(db, tableName); - table = db.getTable(tableName); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); } finally { locker.unLockDatabase(db, LockType.READ); } @@ -223,7 +224,7 @@ public void unprotectedCheckMeta(Database db, String tblName) throw new AnalysisException("Table name must be specified when calling /begin/transaction/ first time"); } - Table table = db.getTable(tblName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); if (table == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tblName); } @@ -248,7 +249,7 @@ public void replayCreateLoadTask(StreamLoadTask loadJob) { } public Database checkDbName(String dbName) throws UserException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { LOG.warn("Database {} does not exist", dbName); throw new UserException("Database[" + dbName + "] does not exist"); @@ -502,7 +503,7 @@ public List getTask(String dbFullName, String label, boolean inc } long dbId = 0L; - Database database = GlobalStateMgr.getCurrentState().getDb(dbFullName); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbFullName); if (database == null) { throw new MetaNotFoundException("failed to find database by dbFullName " + dbFullName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadTask.java b/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadTask.java index cbed0ff10ae77..378fbcfe3b515 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadTask.java @@ -1254,14 +1254,14 @@ private void replayTxnAttachment(TransactionState txnState) { } public OlapTable getTable() throws MetaNotFoundException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("Database " + dbId + "has been deleted"); } Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new MetaNotFoundException("Failed to find table " + tableId + " in db " + dbId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/memory/InternalCatalogMemoryTracker.java b/fe/fe-core/src/main/java/com/starrocks/memory/InternalCatalogMemoryTracker.java index 934267e392998..1d3a51e409532 100644 --- a/fe/fe-core/src/main/java/com/starrocks/memory/InternalCatalogMemoryTracker.java +++ b/fe/fe-core/src/main/java/com/starrocks/memory/InternalCatalogMemoryTracker.java @@ -34,7 +34,7 @@ public long estimateSize() { long estimateSize = 0L; List databases = new ArrayList<>(GlobalStateMgr.getCurrentState().getLocalMetastore().getIdToDb().values()); for (Database database : databases) { - List
tables = database.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(database.getId()); for (Table table : tables) { Collection partitions = table.getPartitions(); Iterator iterator = partitions.iterator(); @@ -51,7 +51,7 @@ public Map estimateCount() { long estimateCount = 0; List databases = new ArrayList<>(GlobalStateMgr.getCurrentState().getLocalMetastore().getIdToDb().values()); for (Database database : databases) { - List
tables = database.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(database.getId()); for (Table table : tables) { Collection partitions = table.getPartitions(); estimateCount += partitions.size(); diff --git a/fe/fe-core/src/main/java/com/starrocks/metric/MaterializedViewMetricsEntity.java b/fe/fe-core/src/main/java/com/starrocks/metric/MaterializedViewMetricsEntity.java index ba1437bb5e5ed..a1836c65bbc8f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/metric/MaterializedViewMetricsEntity.java +++ b/fe/fe-core/src/main/java/com/starrocks/metric/MaterializedViewMetricsEntity.java @@ -104,10 +104,10 @@ public boolean initDbAndTableName() { } GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb(mvId.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(mvId.getDbId()); if (db != null) { dbNameOpt = Optional.of(db.getFullName()); - Table table = db.getTable(mvId.getId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId.getId()); if (table != null) { mvNameOpt = Optional.of(table.getName()); return true; @@ -159,8 +159,9 @@ protected void initMaterializedViewMetrics() { // histogram metrics try { - Database db = GlobalStateMgr.getCurrentState().getDb(mvId.getDbId()); - MaterializedView mv = (MaterializedView) db.getTable(mvId.getId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvId.getDbId()); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), mvId.getId()); histRefreshJobDuration = metricRegistry.histogram(MetricRegistry.name("mv_refresh_duration", db.getFullName(), mv.getName())); } catch (Exception e) { @@ -214,11 +215,11 @@ public Long getValue() { "current materialized view's row count") { @Override public Long getValue() { - Database db = GlobalStateMgr.getCurrentState().getDb(mvId.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvId.getDbId()); if (db == null) { return 0L; } - Table table = db.getTable(mvId.getId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId.getId()); if (table == null || !table.isMaterializedView()) { return 0L; } @@ -238,11 +239,11 @@ public Long getValue() { "current materialized view's storage size") { @Override public Long getValue() { - Database db = GlobalStateMgr.getCurrentState().getDb(mvId.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvId.getDbId()); if (db == null) { return 0L; } - Table table = db.getTable(mvId.getId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId.getId()); if (table == null || !table.isMaterializedView()) { return 0L; } @@ -262,11 +263,11 @@ public Long getValue() { "current materialized view's inactive or not, 0: active, 1: inactive") { @Override public Integer getValue() { - Database db = GlobalStateMgr.getCurrentState().getDb(mvId.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvId.getDbId()); if (db == null) { return 0; } - Table table = db.getTable(mvId.getId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId.getId()); if (table == null || !table.isMaterializedView()) { return 0; } @@ -284,11 +285,11 @@ public Integer getValue() { "current materialized view's partition count, 0 if the materialized view is not partitioned") { @Override public Integer getValue() { - Database db = GlobalStateMgr.getCurrentState().getDb(mvId.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvId.getDbId()); if (db == null) { return 0; } - Table table = db.getTable(mvId.getId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId.getId()); if (table == null || !table.isMaterializedView()) { return 0; } diff --git a/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java b/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java index 1b780dde35403..98873ff11b31c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java @@ -827,13 +827,13 @@ private static void collectTableMetrics(MetricVisitor visitor, boolean minifyTab GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); List dbNames = globalStateMgr.getLocalMetastore().listDbNames(); for (String dbName : dbNames) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (null == db) { continue; } // NOTE: avoid holding database lock here, since we only read all tables, and immutable fields of table - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { long tableId = table.getId(); String tableName = table.getName(); @@ -875,7 +875,7 @@ private static void collectDatabaseMetrics(MetricVisitor visitor) { "database_num", MetricUnit.OPERATIONS, "count of database"); int dbNum = 0; for (String dbName : dbNames) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (null == db) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/mv/MVMetaVersionRepairer.java b/fe/fe-core/src/main/java/com/starrocks/mv/MVMetaVersionRepairer.java index 8b8ebf8cd1111..0555931da923a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/mv/MVMetaVersionRepairer.java +++ b/fe/fe-core/src/main/java/com/starrocks/mv/MVMetaVersionRepairer.java @@ -62,7 +62,8 @@ public static void repairBaseTableVersionChanges(Database db, Table table, LOG.warn("mv db {} not found", mvId.getDbId()); continue; } - MaterializedView mv = (MaterializedView) mvDb.getTable(mvId.getId()); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(mvDb.getId(), mvId.getId()); if (mv == null) { LOG.warn("mv {} not found", mvId.getId()); continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/mv/MVRepairHandler.java b/fe/fe-core/src/main/java/com/starrocks/mv/MVRepairHandler.java index a10450c6bb5a8..e64b7c6ae41fa 100644 --- a/fe/fe-core/src/main/java/com/starrocks/mv/MVRepairHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/mv/MVRepairHandler.java @@ -88,7 +88,7 @@ default void handleMVRepair(TransactionState transactionState) { } for (TableCommitInfo tableCommitInfo : transactionState.getIdToTableCommitInfos().values()) { - Table table = db.getTable(tableCommitInfo.getTableId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableCommitInfo.getTableId()); if (table == null || !(table instanceof OlapTable) || table.getRelatedMaterializedViews().isEmpty()) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java index fa34ec2972df0..68a2a8bb8eedc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java @@ -228,7 +228,7 @@ public void loadJournal(GlobalStateMgr globalStateMgr, JournalEntity journal) case OperationType.OP_DROP_TABLE: case OperationType.OP_DROP_TABLE_V2: { DropInfo info = (DropInfo) journal.getData(); - Database db = globalStateMgr.getDb(info.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(info.getDbId()); if (db == null) { LOG.warn("failed to get db[{}]", info.getDbId()); break; diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/BinlogScanNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/BinlogScanNode.java index 892258a9a96c0..37914df674e09 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/BinlogScanNode.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/BinlogScanNode.java @@ -141,7 +141,7 @@ public void computeScanRanges() throws UserException { if (dbId == -1) { TabletMeta meta = invertedIndex.getTabletMeta(tablet.getId()); dbId = meta.getDbId(); - dbName = GlobalStateMgr.getCurrentState().getDb(dbId).getFullName(); + dbName = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId).getFullName(); } long tabletId = tablet.getId(); diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java b/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java index 83c2d195175ec..8ecfee5c5db47 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java @@ -47,6 +47,7 @@ import com.starrocks.analysis.TableName; import com.starrocks.analysis.TupleDescriptor; import com.starrocks.catalog.Column; +import com.starrocks.catalog.ColumnId; import com.starrocks.catalog.Database; import com.starrocks.catalog.DistributionInfo; import com.starrocks.catalog.ExpressionRangePartitionInfo; @@ -63,7 +64,6 @@ import com.starrocks.catalog.Partition; import com.starrocks.catalog.PartitionKey; import com.starrocks.catalog.PartitionType; -import com.starrocks.catalog.ColumnId; import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.RangePartitionInfo; import com.starrocks.catalog.Replica; @@ -86,7 +86,7 @@ import com.starrocks.sql.analyzer.RelationId; import com.starrocks.sql.analyzer.Scope; import com.starrocks.sql.analyzer.SelectAnalyzer; -import com.starrocks.sql.common.MetaUtils; +import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.system.SystemInfoService; import com.starrocks.thrift.TColumn; import com.starrocks.thrift.TDataSink; @@ -116,9 +116,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.Set; import java.util.stream.Collectors; public class OlapTableSink extends DataSink { @@ -201,7 +201,7 @@ public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeou tSink.setEnable_replicated_storage(enableReplicatedStorage); tSink.setAutomatic_bucket_size(automaticBucketSize); tSink.setEncryption_meta(GlobalStateMgr.getCurrentState().getKeyMgr().getCurrentKEKAsEncryptionMeta()); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { tSink.setDb_name(db.getFullName()); } @@ -385,7 +385,11 @@ public static TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Tup indexSchema.setSchema_id(indexMeta.getSchemaId()); schemaParam.addToIndexes(indexSchema); if (indexMeta.getWhereClause() != null) { - String dbName = MetaUtils.getDatabase(dbId).getFullName(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + if (db == null) { + throw new SemanticException("Database %s is not found", dbId); + } + String dbName = db.getFullName(); Map descMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (SlotDescriptor slot : tupleDescriptor.getSlots()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/DbPEntryObject.java b/fe/fe-core/src/main/java/com/starrocks/privilege/DbPEntryObject.java index d7bc98f5df3c9..1e89a795f8504 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/DbPEntryObject.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/DbPEntryObject.java @@ -199,7 +199,7 @@ public String toString() { return "ALL DATABASES"; } else { if (CatalogMgr.isInternalCatalog(catalogId)) { - Database database = GlobalStateMgr.getCurrentState().getDb(Long.parseLong(uuid)); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(Long.parseLong(uuid)); if (database == null) { throw new MetaNotFoundException("Can't find database : " + uuid); } diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/FunctionPEntryObject.java b/fe/fe-core/src/main/java/com/starrocks/privilege/FunctionPEntryObject.java index ab7f69483ed8f..677911af8ef28 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/FunctionPEntryObject.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/FunctionPEntryObject.java @@ -158,7 +158,7 @@ public String toString() { if (databaseId == PrivilegeBuiltinConstants.ALL_DATABASE_ID) { return "ALL FUNCTIONS IN ALL DATABASES"; } else { - Database database = GlobalStateMgr.getCurrentState().getDb(databaseId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(databaseId); if (database == null) { throw new MetaNotFoundException("Can't find database : " + databaseId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/NativeAccessController.java b/fe/fe-core/src/main/java/com/starrocks/privilege/NativeAccessController.java index f4059c5c01f57..2ead8329fd408 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/NativeAccessController.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/NativeAccessController.java @@ -20,10 +20,11 @@ import com.starrocks.catalog.Database; import com.starrocks.catalog.Function; import com.starrocks.catalog.InternalCatalog; +import com.starrocks.common.ErrorCode; +import com.starrocks.common.ErrorReport; import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.UserIdentity; import com.starrocks.sql.ast.pipe.PipeName; -import com.starrocks.sql.common.MetaUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -364,7 +365,11 @@ private static void checkAnyActionOnFunctionObject(UserIdentity currentUser, Set if (dbName == null) { databaseId = PrivilegeBuiltinConstants.GLOBAL_FUNCTION_DEFAULT_DATABASE_ID; } else { - Database database = MetaUtils.getDatabase(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, dbName); + Database database = GlobalStateMgr.getCurrentState().getMetadataMgr() + .getDb(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, dbName); + if (database == null) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } databaseId = database.getId(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/PipePEntryObject.java b/fe/fe-core/src/main/java/com/starrocks/privilege/PipePEntryObject.java index cd5e9616ad5c9..dca02bca3c6f4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/PipePEntryObject.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/PipePEntryObject.java @@ -56,7 +56,7 @@ public static PEntryObject generate(GlobalStateMgr mgr, List tokens) thr pipeId = PrivilegeBuiltinConstants.ALL_PIPES_ID; } else { String dbName = tokens.get(0); - Database database = mgr.getDb(dbName); + Database database = mgr.getLocalMetastore().getDb(dbName); if (database == null) { throw new PrivObjNotFoundException("cannot find db: " + dbName); } @@ -129,7 +129,7 @@ public List> expandObjectNames() { .map(Pipe::getDbAndName) .collect(Collectors.toList()); for (Pair dbAndName : ListUtils.emptyIfNull(dbAndNames)) { - Optional db = GlobalStateMgr.getCurrentState().mayGetDb(dbAndName.first); + Optional db = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(dbAndName.first); db.ifPresent(database -> objects.add( Lists.newArrayList(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, database.getFullName(), dbAndName.second))); @@ -206,7 +206,7 @@ public Optional getDatabase() { return Optional.empty(); } long dbId = Long.parseLong(getDbUUID()); - return GlobalStateMgr.getCurrentState().mayGetDb(dbId); + return GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(dbId); } catch (NumberFormatException e) { return Optional.empty(); } @@ -227,7 +227,7 @@ public String toString() { sb.append("ALL ").append("DATABASES"); } else { String dbName; - Database database = GlobalStateMgr.getCurrentState().getDb(Long.parseLong(getDbUUID())); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(Long.parseLong(getDbUUID())); if (database == null) { throw new MetaNotFoundException("Cannot find database : " + getDbUUID()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/TablePEntryObject.java b/fe/fe-core/src/main/java/com/starrocks/privilege/TablePEntryObject.java index 48ba74946c904..0546d82b5ceab 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/TablePEntryObject.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/TablePEntryObject.java @@ -275,8 +275,8 @@ protected String toStringImpl(String plural) { } else { String tblName = null; if (CatalogMgr.isInternalCatalog(catalogId)) { - Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable( - Long.parseLong(getDatabaseUUID()), Long.parseLong(getTableUUID())); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(Long.parseLong(getDatabaseUUID()), Long.parseLong(getTableUUID())); if (table == null) { throw new MetaNotFoundException("Cannot find table : " + tableUUID); } diff --git a/fe/fe-core/src/main/java/com/starrocks/privilege/ranger/starrocks/RangerStarRocksAccessController.java b/fe/fe-core/src/main/java/com/starrocks/privilege/ranger/starrocks/RangerStarRocksAccessController.java index 6bd4367db6691..608395af130cd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/privilege/ranger/starrocks/RangerStarRocksAccessController.java +++ b/fe/fe-core/src/main/java/com/starrocks/privilege/ranger/starrocks/RangerStarRocksAccessController.java @@ -129,7 +129,7 @@ public void checkAnyActionOnTable(UserIdentity currentUser, Set roleIds, T public void checkAnyActionOnAnyTable(UserIdentity currentUser, Set roleIds, String catalog, String db) throws AccessDeniedException { Database database = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalog, db); - for (Table table : database.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(database.getId())) { try { hasPermission( RangerStarRocksResource.builder() @@ -187,7 +187,7 @@ public void checkAnyActionOnView(UserIdentity currentUser, Set roleIds, Ta @Override public void checkAnyActionOnAnyView(UserIdentity currentUser, Set roleIds, String db) throws AccessDeniedException { - Database database = GlobalStateMgr.getServingState().getDb(db); + Database database = GlobalStateMgr.getServingState().getLocalMetastore().getDb(db); for (Table table : database.getViews()) { try { hasPermission( @@ -235,7 +235,7 @@ public void checkAnyActionOnMaterializedView(UserIdentity currentUser, Set @Override public void checkAnyActionOnAnyMaterializedView(UserIdentity currentUser, Set roleIds, String db) throws AccessDeniedException { - Database database = GlobalStateMgr.getServingState().getDb(db); + Database database = GlobalStateMgr.getServingState().getLocalMetastore().getDb(db); for (Table table : database.getMaterializedViews()) { try { hasPermission( @@ -282,7 +282,7 @@ public void checkAnyActionOnFunction(UserIdentity currentUser, Set roleIds @Override public void checkAnyActionOnAnyFunction(UserIdentity currentUser, Set roleIds, String db) throws AccessDeniedException { - Database database = GlobalStateMgr.getServingState().getDb(db); + Database database = GlobalStateMgr.getServingState().getLocalMetastore().getDb(db); for (Function function : database.getFunctions()) { try { hasPermission( @@ -329,8 +329,8 @@ public void checkAnyActionOnGlobalFunction(UserIdentity currentUser, Set r @Override public void checkActionInDb(UserIdentity userIdentity, Set roleIds, String db, PrivilegeType privilegeType) throws AccessDeniedException { - Database database = GlobalStateMgr.getCurrentState().getDb(db); - for (Table table : database.getTables()) { + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(db); + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(database.getId())) { if (table.isOlapView()) { checkViewAction(userIdentity, roleIds, new TableName(database.getFullName(), table.getName()), privilegeType); } else if (table.isMaterializedView()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java index 496ad7a34fc68..22be7914ee5b3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java @@ -243,7 +243,7 @@ public ShowResultSet visitCreateFunctionStatement(CreateFunctionStmt stmt, Conne .getGlobalFunctionMgr() .userAddFunction(stmt.getFunction(), stmt.shouldReplaceIfExists()); } else { - Database db = context.getGlobalStateMgr().getDb(name.getDb()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(name.getDb()); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, name.getDb()); } @@ -260,7 +260,7 @@ public ShowResultSet visitDropFunctionStatement(DropFunctionStmt stmt, ConnectCo if (name.isGlobalFunction()) { context.getGlobalStateMgr().getGlobalFunctionMgr().userDropFunction(stmt.getFunctionSearchDesc()); } else { - Database db = context.getGlobalStateMgr().getDb(name.getDb()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(name.getDb()); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, name.getDb()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java index 9c35e77748b25..1a088ad6b4ffd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java @@ -301,7 +301,7 @@ public ShowResultSet visitShowStatement(ShowStmt statement, ConnectContext conte @Override public ShowResultSet visitShowMaterializedViewStatement(ShowMaterializedViewsStmt statement, ConnectContext context) { String dbName = statement.getDb(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); MetaUtils.checkDbNullAndReport(db, dbName); List materializedViews = Lists.newArrayList(); @@ -315,7 +315,7 @@ public ShowResultSet visitShowMaterializedViewStatement(ShowMaterializedViewsStm CaseSensibility.TABLE.getCaseSensibility()); } - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table.isMaterializedView()) { MaterializedView mvTable = (MaterializedView) table; if (matcher != null && !matcher.match(mvTable.getName())) { @@ -570,7 +570,7 @@ public ShowResultSet visitShowTemporaryTablesStatement(ShowTemporaryTableStmt st @Override public ShowResultSet visitShowTableStatusStatement(ShowTableStatusStmt statement, ConnectContext context) { List> rows = Lists.newArrayList(); - Database db = context.getGlobalStateMgr().getDb(statement.getDb()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDb()); ZoneId currentTimeZoneId = TimeUtils.getTimeZone().toZoneId(); if (db != null) { Locker locker = new Locker(); @@ -581,7 +581,7 @@ public ShowResultSet visitShowTableStatusStatement(ShowTableStatusStmt statement matcher = PatternMatcher.createMysqlPattern(statement.getPattern(), CaseSensibility.TABLE.getCaseSensibility()); } - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (matcher != null && !matcher.match(table.getName())) { continue; } @@ -664,7 +664,7 @@ public ShowResultSet visitShowCreateDbStatement(ShowCreateDbStmt statement, Conn Database db; if (Strings.isNullOrEmpty(catalogName) || CatalogMgr.isInternalCatalog(catalogName)) { - db = context.getGlobalStateMgr().getDb(dbName); + db = context.getGlobalStateMgr().getLocalMetastore().getDb(dbName); } else { db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); } @@ -697,7 +697,7 @@ public ShowResultSet visitShowCreateTableStatement(ShowCreateTableStmt statement } private ShowResultSet showCreateInternalCatalogTable(ShowCreateTableStmt showStmt, ConnectContext connectContext) { - Database db = GlobalStateMgr.getCurrentState().getDb(showStmt.getDb()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(showStmt.getDb()); MetaUtils.checkDbNullAndReport(db, showStmt.getDb()); List> rows = Lists.newArrayList(); Locker locker = new Locker(); @@ -710,7 +710,7 @@ private ShowResultSet showCreateInternalCatalogTable(ShowCreateTableStmt showStm } else { // For Sync Materialized View, it is a mv index inside OLAP table, // so we can not get it from database. - for (Table tbl : db.getTables()) { + for (Table tbl : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (tbl.getType() == Table.TableType.OLAP) { OlapTable olapTable = (OlapTable) tbl; List visibleMaterializedViews = @@ -899,7 +899,7 @@ public ShowResultSet visitShowFunctionsStatement(ShowFunctionsStmt statement, Co } else if (statement.getIsGlobal()) { functions = context.getGlobalStateMgr().getGlobalFunctionMgr().getFunctions(); } else { - Database db = context.getGlobalStateMgr().getDb(statement.getDbName()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDbName()); MetaUtils.checkDbNullAndReport(db, statement.getDbName()); functions = db.getFunctions(); } @@ -917,7 +917,7 @@ public ShowResultSet visitShowFunctionsStatement(ShowFunctionsStmt statement, Co continue; } } else if (!statement.getIsBuiltin()) { - Database db = context.getGlobalStateMgr().getDb(statement.getDbName()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDbName()); try { Authorizer.checkAnyActionOnFunction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), db.getFullName(), function); @@ -1046,7 +1046,7 @@ public ShowResultSet visitShowLoadStatement(ShowLoadStmt statement, ConnectConte if (statement.isAll()) { dbId = -1; } else { - Database db = globalStateMgr.getDb(statement.getDbName()); + Database db = globalStateMgr.getLocalMetastore().getDb(statement.getDbName()); MetaUtils.checkDbNullAndReport(db, statement.getDbName()); dbId = db.getId(); } @@ -1300,7 +1300,7 @@ public ShowResultSet visitShowStreamLoadStatement(ShowStreamLoadStmt statement, @Override public ShowResultSet visitShowDeleteStatement(ShowDeleteStmt statement, ConnectContext context) { GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb(statement.getDbName()); + Database db = globalStateMgr.getLocalMetastore().getDb(statement.getDbName()); MetaUtils.checkDbNullAndReport(db, statement.getDbName()); long dbId = db.getId(); @@ -1363,7 +1363,7 @@ public ShowResultSet visitShowDataStatement(ShowDataStmt statement, ConnectConte long totalReplicaCount = 0; // sort by table name - List
tables = db.getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); SortedSet
sortedTables = new TreeSet<>(Comparator.comparing(Table::getName)); for (Table table : tables) { @@ -1566,7 +1566,7 @@ public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectC // check real meta do { - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { isSync = false; break; @@ -1576,7 +1576,7 @@ public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectC Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (!(table instanceof OlapTable)) { isSync = false; break; @@ -1642,7 +1642,7 @@ public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectC partitionId.toString(), indexId.toString(), isSync.toString(), detailCmd)); } else { - Database db = globalStateMgr.getDb(statement.getDbName()); + Database db = globalStateMgr.getLocalMetastore().getDb(statement.getDbName()); MetaUtils.checkDbNullAndReport(db, statement.getDbName()); Locker locker = new Locker(); @@ -1760,7 +1760,7 @@ public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectC @Override public ShowResultSet visitShowBackupStatement(ShowBackupStmt statement, ConnectContext context) { - Database filterDb = GlobalStateMgr.getCurrentState().getDb(statement.getDbName()); + Database filterDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(statement.getDbName()); List> infos = Lists.newArrayList(); List dbs = Lists.newArrayList(); @@ -1806,7 +1806,7 @@ public ShowResultSet visitShowBackupStatement(ShowBackupStmt statement, ConnectC @Override public ShowResultSet visitShowRestoreStatement(ShowRestoreStmt statement, ConnectContext context) { - Database filterDb = GlobalStateMgr.getCurrentState().getDb(statement.getDbName()); + Database filterDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(statement.getDbName()); List> infos = Lists.newArrayList(); List dbs = Lists.newArrayList(); @@ -1852,7 +1852,7 @@ public ShowResultSet visitShowResourceStatement(ShowResourcesStmt statement, Con @Override public ShowResultSet visitShowExportStatement(ShowExportStmt statement, ConnectContext context) { GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb(statement.getDbName()); + Database db = globalStateMgr.getLocalMetastore().getDb(statement.getDbName()); MetaUtils.checkDbNullAndReport(db, statement.getDbName()); long dbId = db.getId(); @@ -2082,12 +2082,12 @@ public ShowResultSet visitShowSmallFilesStatement(ShowSmallFilesStmt statement, @Override public ShowResultSet visitShowDynamicPartitionStatement(ShowDynamicPartitionStmt statement, ConnectContext context) { List> rows = Lists.newArrayList(); - Database db = context.getGlobalStateMgr().getDb(statement.getDb()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDb()); if (db != null) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - for (Table tbl : db.getTables()) { + for (Table tbl : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (!(tbl instanceof OlapTable)) { continue; } @@ -2148,7 +2148,7 @@ public ShowResultSet visitShowDynamicPartitionStatement(ShowDynamicPartitionStmt @Override public ShowResultSet visitShowIndexStatement(ShowIndexStmt statement, ConnectContext context) { List> rows = Lists.newArrayList(); - Database db = context.getGlobalStateMgr().getDb(statement.getDbName()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDbName()); MetaUtils.checkDbNullAndReport(db, statement.getDbName()); Table table = MetaUtils.getSessionAwareTable(context, db, statement.getTableName()); if (table == null) { @@ -2180,7 +2180,7 @@ public ShowResultSet visitShowIndexStatement(ShowIndexStmt statement, ConnectCon @Override public ShowResultSet visitShowTransactionStatement(ShowTransactionStmt statement, ConnectContext context) { - Database db = context.getGlobalStateMgr().getDb(statement.getDbName()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDbName()); MetaUtils.checkDbNullAndReport(db, statement.getDbName()); long txnId = statement.getTxnId(); @@ -2502,7 +2502,7 @@ public ShowResultSet visitDescStorageVolumeStatement(DescStorageVolumeStmt state public ShowResultSet visitShowPipeStatement(ShowPipeStmt statement, ConnectContext context) { List> rows = Lists.newArrayList(); String dbName = statement.getDbName(); - long dbId = GlobalStateMgr.getCurrentState().mayGetDb(dbName) + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(dbName) .map(Database::getId) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName)); PipeManager pipeManager = GlobalStateMgr.getCurrentState().getPipeManager(); diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java index 8a47824dd4704..b2b5834a3b71a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java @@ -1253,7 +1253,11 @@ private void handleQueryStmt(ExecPlan execPlan) throws Exception { private void handleAnalyzeStmt() throws IOException { AnalyzeStmt analyzeStmt = (AnalyzeStmt) parsedStmt; - Database db = MetaUtils.getDatabase(context, analyzeStmt.getTableName()); + TableName tableName = analyzeStmt.getTableName(); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(tableName.getCatalog(), tableName.getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", tableName.getCatalogAndDb()); + } Table table = MetaUtils.getSessionAwareTable(context, db, analyzeStmt.getTableName()); if (StatisticUtils.isEmptyTable(table)) { return; @@ -1411,12 +1415,17 @@ private void executeAnalyze(ConnectContext statsConnectCtx, AnalyzeStmt analyzeS private void handleDropStatsStmt() { DropStatsStmt dropStatsStmt = (DropStatsStmt) parsedStmt; - Table table = MetaUtils.getTable(context, dropStatsStmt.getTableName()); + TableName tableName = dropStatsStmt.getTableName(); + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(tableName.getCatalog(), + tableName.getDb(), tableName.getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName.toString()); + } + if (dropStatsStmt.isExternal()) { GlobalStateMgr.getCurrentState().getAnalyzeMgr().dropExternalAnalyzeStatus(table.getUUID()); GlobalStateMgr.getCurrentState().getAnalyzeMgr().dropExternalBasicStatsData(table.getUUID()); - TableName tableName = dropStatsStmt.getTableName(); GlobalStateMgr.getCurrentState().getAnalyzeMgr().removeExternalBasicStatsMeta(tableName.getCatalog(), tableName.getDb(), tableName.getTbl()); List columns = table.getBaseSchema().stream().map(Column::getName).collect(Collectors.toList()); @@ -1434,7 +1443,13 @@ private void handleDropStatsStmt() { private void handleDropHistogramStmt() { DropHistogramStmt dropHistogramStmt = (DropHistogramStmt) parsedStmt; - Table table = MetaUtils.getTable(context, dropHistogramStmt.getTableName()); + TableName tableName = dropHistogramStmt.getTableName(); + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(tableName.getCatalog(), + tableName.getDb(), tableName.getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName.toString()); + } + if (dropHistogramStmt.isExternal()) { List columns = dropHistogramStmt.getColumnNames(); @@ -1465,8 +1480,15 @@ private void handleKillAnalyzeStmt() { private void checkTblPrivilegeForKillAnalyzeStmt(ConnectContext context, String catalogName, String dbName, String tableName) { - MetaUtils.getDatabase(catalogName, dbName); - MetaUtils.getTable(catalogName, dbName, tableName); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); + if (db == null) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } + + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(catalogName, dbName, tableName); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName); + } Authorizer.checkActionForAnalyzeStatement(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), new TableName(catalogName, dbName, tableName)); } @@ -1944,7 +1966,12 @@ public PQueryStatistics getQueryStatisticsForAuditLog() { } public void handleInsertOverwrite(InsertStmt insertStmt) throws Exception { - Database db = MetaUtils.getDatabase(context, insertStmt.getTableName()); + TableName tableName = insertStmt.getTableName(); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(tableName.getCatalog(), tableName.getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", tableName.getCatalogAndDb()); + } + Locker locker = new Locker(); Table table = insertStmt.getTargetTable(); if (!(table instanceof OlapTable)) { @@ -2035,7 +2062,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { return; } - MetaUtils.normalizationTableName(context, stmt.getTableName()); + stmt.getTableName().normalization(context); String catalogName = stmt.getTableName().getCatalog(); String dbName = stmt.getTableName().getDb(); String tableName = stmt.getTableName().getTbl(); diff --git a/fe/fe-core/src/main/java/com/starrocks/replication/ReplicationJob.java b/fe/fe-core/src/main/java/com/starrocks/replication/ReplicationJob.java index 55eb39cec3d00..2473879ef8db4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/replication/ReplicationJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/replication/ReplicationJob.java @@ -572,7 +572,7 @@ private static TableInfo initTableInfo(TTableReplicationRequest request) throws long tableDataSize; Map partitionInfos = Maps.newHashMap(); - Database db = GlobalStateMgr.getCurrentState().getDb(request.database_id); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(request.database_id); if (db == null) { throw new MetaNotFoundException("Database " + request.database_id + " not found"); } @@ -580,7 +580,7 @@ private static TableInfo initTableInfo(TTableReplicationRequest request) throws Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(request.table_id); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), request.table_id); if (table == null) { throw new MetaNotFoundException( "Table " + request.table_id + " in database " + db.getFullName() + " not found"); @@ -830,8 +830,8 @@ private boolean isTransactionAborted() { } if (txnState.getTransactionStatus() == TransactionStatus.PREPARE) { - Database db = GlobalStateMgr.getServingState().getDb(databaseId); - if (db == null || db.getTable(tableId) == null) { + Database db = GlobalStateMgr.getServingState().getLocalMetastore().getDb(databaseId); + if (db == null || GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId) == null) { abortTransaction("Table is deleted"); return true; } diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/MVActiveChecker.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/MVActiveChecker.java index 5894ff0a77705..1bcba036ca4c3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/MVActiveChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/MVActiveChecker.java @@ -93,7 +93,8 @@ private void clearGracePeriod() { private void process() { Collection dbs = GlobalStateMgr.getCurrentState().getLocalMetastore().getIdToDb().values(); for (Database db : CollectionUtils.emptyIfNull(dbs)) { - for (Table table : CollectionUtils.emptyIfNull(db.getTables())) { + for (Table table : CollectionUtils.emptyIfNull( + GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()))) { if (table.isMaterializedView()) { MaterializedView mv = (MaterializedView) table; if (!mv.isActive()) { @@ -124,7 +125,7 @@ public static void tryToActivate(MaterializedView mv, boolean checkGracePeriod) } long dbId = mv.getDbId(); - Optional dbName = GlobalStateMgr.getCurrentState().mayGetDb(dbId).map(Database::getFullName); + Optional dbName = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(dbId).map(Database::getFullName); if (!dbName.isPresent()) { LOG.warn("[MVActiveChecker] cannot activate MV {} since database {} not found", mv.getName(), dbId); return; diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java index 33811e51fdce0..bdb6f46550b97 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java @@ -792,7 +792,7 @@ private void updateMeta(Set mvRefreshedPartitions, LOG.info("start to update meta for mv:{}", materializedView.getName()); // check - Table mv = db.getTable(materializedView.getId()); + Table mv = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), materializedView.getId()); if (mv == null) { throw new DmlException( "update meta failed. materialized view:" + materializedView.getName() + " not exist"); @@ -849,12 +849,12 @@ public void prepare(TaskRunContext context) { Map properties = context.getProperties(); // NOTE: mvId is set in Task's properties when creating long mvId = Long.parseLong(properties.get(MV_ID)); - db = GlobalStateMgr.getCurrentState().getDb(context.ctx.getDatabase()); + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(context.ctx.getDatabase()); if (db == null) { LOG.warn("database {} do not exist when refreshing materialized view:{}", context.ctx.getDatabase(), mvId); throw new DmlException("database " + context.ctx.getDatabase() + " do not exist."); } - Table table = db.getTable(mvId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId); if (table == null) { LOG.warn("materialized view:{} in database:{} do not exist when refreshing", mvId, context.ctx.getDatabase()); diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java index a4e759007dd11..6b6db7aaeb19b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java @@ -171,13 +171,13 @@ public Map refreshTaskProperties(ConnectContext ctx) { try { // NOTE: mvId is set in Task's properties when creating long mvId = Long.parseLong(properties.get(MV_ID)); - Database database = GlobalStateMgr.getCurrentState().getDb(ctx.getDatabase()); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(ctx.getDatabase()); if (database == null) { LOG.warn("database {} do not exist when refreshing materialized view:{}", ctx.getDatabase(), mvId); return newProperties; } - Table table = database.getTable(mvId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), mvId); if (table == null) { LOG.warn("materialized view:{} in database:{} do not exist when refreshing", mvId, ctx.getDatabase()); diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/history/TableKeeper.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/history/TableKeeper.java index 2037bc32cb053..9bd41bf2f0ef6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/history/TableKeeper.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/history/TableKeeper.java @@ -92,7 +92,7 @@ public synchronized boolean isReady() { } public boolean checkDatabaseExists() { - return GlobalStateMgr.getCurrentState().getDb(databaseName) != null; + return GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(databaseName) != null; } public void createTable() throws UserException { @@ -102,8 +102,7 @@ public void createTable() throws UserException { public void correctTable() { int numBackends = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getTotalBackendNumber(); int replica = GlobalStateMgr.getCurrentState() - .mayGetDb(databaseName) - .flatMap(db -> db.mayGetTable(tableName)) + .getLocalMetastore().mayGetTable(databaseName, tableName) .map(tbl -> ((OlapTable) tbl).getPartitionInfo().getMinReplicationNum()) .orElse((short) 1); if (numBackends < tableReplicas) { @@ -143,8 +142,7 @@ public void changeTTL() { private Optional mayGetTable() { return GlobalStateMgr.getCurrentState() - .mayGetDb(databaseName) - .flatMap(db -> db.mayGetTable(tableName)) + .getLocalMetastore().mayGetTable(databaseName, tableName) .flatMap(x -> Optional.of((OlapTable) x)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceJob.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceJob.java index 7f32f0e0d9aac..edf32f3da781c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceJob.java @@ -125,7 +125,7 @@ public static MVMaintenanceJob read(DataInput input) throws IOException { // TODO recover the entire job state, include execution plan public void restore() { - Table table = GlobalStateMgr.getCurrentState().getDb(dbId).getTable(viewId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(dbId, viewId); Preconditions.checkState(table != null && table.isMaterializedView()); this.view = (MaterializedView) table; this.serializedState = JobState.INIT; @@ -232,7 +232,7 @@ void buildContext() { // Build connection context this.connectContext = StatisticUtils.buildConnectContext(); this.connectContext.setNeedQueued(false); - Database db = GlobalStateMgr.getCurrentState().getDb(view.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(view.getDbId()); this.connectContext.getSessionVariable().setQueryTimeoutS(MV_QUERY_TIMEOUT); if (db != null) { this.connectContext.setDatabase(db.getFullName()); @@ -365,7 +365,7 @@ private void deployTasks() throws Exception { private void setMVMaintenanceTasksInfo(TMVMaintenanceTasks request, MVMaintenanceTask task) { // Request information - String dbName = GlobalStateMgr.getCurrentState().getDb(view.getDbId()).getFullName(); + String dbName = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(view.getDbId()).getFullName(); request.setDb_name(dbName); request.setMv_name(view.getName()); diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceTask.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceTask.java index c3f016fed625b..376d8825ef1ec 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVMaintenanceTask.java @@ -59,7 +59,7 @@ public class MVMaintenanceTask { public static MVMaintenanceTask build(MVMaintenanceJob job, long taskId, TNetworkAddress beRpcAddr, List fragmentInstances) { MVMaintenanceTask task = new MVMaintenanceTask(); - task.dbName = GlobalStateMgr.getCurrentState().getDb(job.getView().getDbId()).getFullName(); + task.dbName = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(job.getView().getDbId()).getFullName(); task.job = job; task.beRpcAddr = beRpcAddr; task.taskId = taskId; diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshPartitioner.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshPartitioner.java index 7d9f8a5c3c1f1..7eddfe8b1858b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshPartitioner.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshPartitioner.java @@ -259,7 +259,7 @@ protected void dropPartition(Database db, MaterializedView materializedView, Str } try { // check - Table mv = db.getTable(materializedView.getId()); + Table mv = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), materializedView.getId()); if (mv == null) { throw new DmlException("drop partition failed. mv:" + materializedView.getName() + " not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/TxnBasedEpochCoordinator.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/TxnBasedEpochCoordinator.java index 9d2e49eccc96c..d8799475f9c27 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/TxnBasedEpochCoordinator.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/TxnBasedEpochCoordinator.java @@ -129,7 +129,7 @@ private boolean waitEpochFinish(MVEpoch epoch) throws Exception { private void commitEpoch(MVEpoch epoch) { LOG.info("commitEpoch: {}", epoch); long dbId = mvMaintenanceJob.getView().getDbId(); - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); try { epoch.onCommitting(); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index 6e88dc19ffbe1..4ee05085788fc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -248,7 +248,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -2044,26 +2043,6 @@ protected void runAfterCatalogReady() { }; } - public Database getDb(String name) { - return localMetastore.getDb(name); - } - - public Optional
mayGetTable(long dbId, long tableId) { - return mayGetDb(dbId).flatMap(db -> db.tryGetTable(tableId)); - } - - public Optional mayGetDb(String name) { - return Optional.ofNullable(localMetastore.getDb(name)); - } - - public Optional mayGetDb(long dbId) { - return Optional.ofNullable(localMetastore.getDb(dbId)); - } - - public Database getDb(long dbId) { - return localMetastore.getDb(dbId); - } - public EditLog getEditLog() { return editLog; } @@ -2086,11 +2065,11 @@ public AlterJobMgr getAlterJobMgr() { } public SchemaChangeHandler getSchemaChangeHandler() { - return (SchemaChangeHandler) this.alterJobMgr.getSchemaChangeHandler(); + return this.alterJobMgr.getSchemaChangeHandler(); } public MaterializedViewHandler getRollupHandler() { - return (MaterializedViewHandler) this.alterJobMgr.getMaterializedViewHandler(); + return this.alterJobMgr.getMaterializedViewHandler(); } public BackupHandler getBackupHandler() { @@ -2452,7 +2431,7 @@ public String dumpImage() { try { // sort all dbs for (long dbId : localMetastore.getDbIds()) { - Database db = getDb(dbId); + Database db = localMetastore.getDb(dbId); Preconditions.checkNotNull(db); lockedDbMap.put(dbId, db); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 0d05d48a643b5..f8ab87e29a6f4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -623,7 +623,7 @@ public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - Table table = db.getTable(tableName); + Table table = getTable(db.getFullName(), tableName); if (table != null) { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); } @@ -632,7 +632,7 @@ public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); } - Table recoverTable = db.getTable(tableName); + Table recoverTable = getTable(db.getFullName(), tableName); if (recoverTable instanceof OlapTable) { DynamicPartitionUtil.registerOrRemovePartitionScheduleInfo(db.getId(), (OlapTable) recoverTable); } @@ -654,7 +654,7 @@ public void recoverPartition(RecoverPartitionStmt recoverStmt) throws DdlExcepti Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - Table table = db.getTable(tableName); + Table table = getTable(db.getFullName(), tableName); if (table == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); } @@ -817,7 +817,7 @@ public boolean createTable(CreateTableStmt stmt) throws DdlException { locker.lockDatabase(db, LockType.READ); try { String tableName = stmt.getTableName(); - if (!isTemporaryTable && db.getTable(tableName) != null) { + if (!isTemporaryTable && getTable(db.getFullName(), tableName) != null) { if (!stmt.isSetIfNotExists()) { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); } @@ -867,7 +867,7 @@ public void addPartitions(ConnectContext ctx, Database db, String tableName, Add Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableName); + Table table = getTable(db.getFullName(), tableName); CatalogUtils.checkTableExist(db, tableName); CatalogUtils.checkNativeTable(db, table); } finally { @@ -881,7 +881,7 @@ public void addPartitions(ConnectContext ctx, Database db, String tableName, Add private OlapTable checkTable(Database db, String tableName) throws DdlException { CatalogUtils.checkTableExist(db, tableName); - Table table = db.getTable(tableName); + Table table = getTable(db.getFullName(), tableName); CatalogUtils.checkNativeTable(db, table); OlapTable olapTable = (OlapTable) table; CatalogUtils.checkTableState(olapTable, tableName); @@ -889,7 +889,7 @@ private OlapTable checkTable(Database db, String tableName) throws DdlException } private OlapTable checkTable(Database db, Long tableId) throws DdlException { - Table table = db.getTable(tableId); + Table table = getTable(db.getId(), tableId); if (table == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableId); } @@ -1321,7 +1321,7 @@ public void replayAddPartition(PartitionPersistInfoV2 info) throws DdlException Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(info.getTableId()); + OlapTable olapTable = (OlapTable) getTable(db.getId(), info.getTableId()); Partition partition = info.getPartition(); PartitionInfo partitionInfo = olapTable.getPartitionInfo(); @@ -1380,7 +1380,7 @@ public void replayAddPartition(PartitionPersistInfo info) throws DdlException { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(info.getTableId()); + OlapTable olapTable = (OlapTable) getTable(db.getId(), info.getTableId()); Partition partition = info.getPartition(); PartitionInfo partitionInfo = olapTable.getPartitionInfo(); @@ -1491,7 +1491,7 @@ public void dropPartition(Database db, Table table, DropPartitionClause clause) if (!isTempPartition) { try { for (MvId mvId : olapTable.getRelatedMaterializedViews()) { - MaterializedView materializedView = (MaterializedView) db.getTable(mvId.getId()); + MaterializedView materializedView = (MaterializedView) getTable(db.getId(), mvId.getId()); if (materializedView != null && materializedView.isLoadTriggeredRefresh()) { GlobalStateMgr.getCurrentState().getLocalMetastore().refreshMaterializedView( db.getFullName(), materializedView.getName(), false, null, @@ -1527,7 +1527,7 @@ public void replayDropPartition(DropPartitionInfo info) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(info.getTableId()); + OlapTable olapTable = (OlapTable) getTable(db.getId(), info.getTableId()); if (info.isTempPartition()) { olapTable.dropTempPartition(info.getPartitionName(), true); } else { @@ -1547,7 +1547,7 @@ public void replayDropPartitions(DropPartitionsInfo info) { + " table = " + info.getTableId() + " partitionNames = " + info.getPartitionNames()); List partitionNames = info.getPartitionNames(); - OlapTable olapTable = (OlapTable) db.getTable(info.getTableId()); + OlapTable olapTable = (OlapTable) getTable(db.getId(), info.getTableId()); boolean isTempPartition = info.isTempPartition(); long dbId = info.getDbId(); boolean isForceDrop = info.isForceDrop(); @@ -1573,7 +1573,7 @@ public void replayRecoverPartition(RecoverInfo info) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - Table table = db.getTable(info.getTableId()); + Table table = getTable(db.getId(), info.getTableId()); recycleBin.replayRecoverPartition((OlapTable) table, info.getPartitionId()); } finally { locker.unLockDatabase(db, LockType.WRITE); @@ -1705,7 +1705,7 @@ public void replayAddSubPartition(PhysicalPartitionPersistInfoV2 info) throws Dd Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(info.getTableId()); + OlapTable olapTable = (OlapTable) getTable(db.getId(), info.getTableId()); Partition partition = olapTable.getPartition(info.getPartitionId()); PhysicalPartition physicalPartition = info.getPhysicalPartition(); partition.addSubPartition(physicalPartition); @@ -2596,7 +2596,7 @@ public void replayDropTable(Database db, long tableId, boolean isForceDrop) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - table = db.getTable(tableId); + table = getTable(db.getId(), tableId); if (table.isTemporaryTable()) { table = db.unprotectDropTemporaryTable(tableId, isForceDrop, false); UUID sessionId = ((OlapTable) table).getSessionId(); @@ -2776,24 +2776,6 @@ public void replayBatchDeleteReplica(BatchDeleteReplicaInfo info) { } } - @Override - public Table getTable(String dbName, String tblName) { - Database database = getDb(dbName); - if (database == null) { - return null; - } - return database.getTable(tblName); - } - - @Override - public Pair getMaterializedViewIndex(String dbName, String indexName) { - Database database = getDb(dbName); - if (database == null) { - return null; - } - return database.getMaterializedViewIndex(indexName); - } - @Override public Database getDb(String name) { if (name == null) { @@ -2820,6 +2802,22 @@ public Database getDb(long dbId) { return idToDb.get(dbId); } + public Optional mayGetDb(String name) { + return Optional.ofNullable(getDb(name)); + } + + public Optional mayGetDb(long dbId) { + return Optional.ofNullable(getDb(dbId)); + } + + public Optional
mayGetTable(long dbId, long tableId) { + return mayGetDb(dbId).flatMap(db -> Optional.ofNullable(db.getTable(tableId))); + } + + public Optional
mayGetTable(String dbName, String tableName) { + return mayGetDb(dbName).flatMap(db -> Optional.ofNullable(db.getTable(tableName))); + } + public ConcurrentHashMap getFullNameToDb() { return fullNameToDb; } @@ -2832,8 +2830,43 @@ public Database getDbIncludeRecycleBin(long dbId) { return db; } + @Override + public Table getTable(String dbName, String tblName) { + Database database = getDb(dbName); + if (database == null) { + return null; + } + return database.getTable(tblName); + } + + public Table getTable(Long dbId, Long tableId) { + Database database = getDb(dbId); + if (database == null) { + return null; + } + return database.getTable(tableId); + } + + public List
getTables(Long dbId) { + Database database = getDb(dbId); + if (database == null) { + return Collections.emptyList(); + } else { + return database.getTables(); + } + } + + @Override + public Pair getMaterializedViewIndex(String dbName, String indexName) { + Database database = getDb(dbName); + if (database == null) { + return null; + } + return database.getMaterializedViewIndex(indexName); + } + public Table getTableIncludeRecycleBin(Database db, long tableId) { - Table table = db.getTable(tableId); + Table table = getTable(db.getId(), tableId); if (table == null) { table = recycleBin.getTable(db.getId(), tableId); } @@ -2994,7 +3027,7 @@ public HashMap getPartitionIdToStorageMediumMap() { Preconditions.checkState(locker.isDbWriteLockHeldByCurrentThread(db)); try { for (Long tableId : tableIdToPartitionIds.keySet()) { - Table table = db.getTable(tableId); + Table table = getTable(db.getId(), tableId); if (table == null) { continue; } @@ -3061,7 +3094,7 @@ public void createMaterializedView(CreateMaterializedViewStmt stmt) String tableName = stmt.getBaseIndexName(); // check db String dbName = stmt.getDBName(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } @@ -3075,7 +3108,7 @@ public void createMaterializedView(CreateMaterializedViewStmt stmt) throw new DdlException("create materialized failed. database:" + db.getFullName() + " not exist"); } try { - Table table = db.getTable(tableName); + Table table = getTable(db.getFullName(), tableName); if (table == null) { throw new DdlException("create materialized failed. table:" + tableName + " not exist"); } @@ -3124,7 +3157,7 @@ public void createMaterializedView(CreateMaterializedViewStatement stmt) Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - if (db.getTable(mvName) != null) { + if (getTable(db.getFullName(), mvName) != null) { if (stmt.isIfNotExists()) { LOG.info("Create materialized view [{}] which already exists", mvName); return; @@ -3437,7 +3470,7 @@ public void dropMaterializedView(DropMaterializedViewStmt stmt) throws DdlExcept Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - table = db.getTable(stmt.getMvName()); + table = getTable(db.getFullName(), stmt.getMvName()); } finally { locker.unLockDatabase(db, LockType.READ); } @@ -3492,7 +3525,7 @@ private MaterializedView getMaterializedViewToRefresh(String dbName, String mvNa if (db == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } - final Table table = db.getTable(mvName); + final Table table = getTable(db.getFullName(), mvName); MaterializedView materializedView = null; if (table instanceof MaterializedView) { materializedView = (MaterializedView) table; @@ -3586,7 +3619,7 @@ public void renameTable(Database db, Table table, TableRenameClause tableRenameC } // check if name is already used - if (db.getTable(newTableName) != null) { + if (getTable(db.getFullName(), newTableName) != null) { throw new DdlException("Table name[" + newTableName + "] is already used"); } @@ -3613,7 +3646,8 @@ public void alterTableComment(Database db, Table table, AlterTableCommentClause public static void inactiveRelatedMaterializedView(Database db, Table olapTable, String reason) { for (MvId mvId : olapTable.getRelatedMaterializedViews()) { - MaterializedView mv = (MaterializedView) db.getTable(mvId.getId()); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), mvId.getId()); if (mv != null) { LOG.warn("Inactive MV {}/{} because {}", mv.getName(), mv.getId(), reason); mv.setInactiveAndReason(reason); @@ -3636,7 +3670,7 @@ public void replayRenameTable(TableInfo tableInfo) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) getTable(db.getId(), tableId); String tableName = table.getName(); db.dropTable(tableName); table.setName(newTableName); @@ -3697,7 +3731,7 @@ public void replayRenamePartition(TableInfo tableInfo) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) getTable(db.getId(), tableId); Partition partition = table.getPartition(partitionId); table.renamePartition(partition.getName(), newPartitionName); LOG.info("replay rename partition[{}] to {}", partition.getName(), newPartitionName); @@ -3751,7 +3785,7 @@ public void replayRenameRollup(TableInfo tableInfo) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) getTable(db.getId(), tableId); String rollupName = table.getIndexNameById(indexId); Map indexNameToIdMap = table.getIndexNameToId(); indexNameToIdMap.remove(rollupName); @@ -3802,7 +3836,7 @@ public void replayRenameColumn(ColumnRenameInfo columnRenameInfo) throws DdlExce Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(tableId); + OlapTable olapTable = (OlapTable) getTable(db.getId(), tableId); olapTable.renameColumn(colName, newColName); LOG.info("replay rename column[{}] to {}", colName, newColName); } finally { @@ -4107,7 +4141,7 @@ public void modifyTableConstraint(Database db, String tableName, Map relatedMvs = olapTable.getRelatedMaterializedViews(); for (MvId mvId : relatedMvs) { - MaterializedView materializedView = (MaterializedView) db.getTable(mvId.getId()); + MaterializedView materializedView = (MaterializedView) getTable(db.getId(), mvId.getId()); if (materializedView == null) { LOG.warn("Table related materialized view {} can not be found", mvId.getId()); continue; } if (materializedView.isLoadTriggeredRefresh()) { - refreshMaterializedView(db.getFullName(), db.getTable(mvId.getId()).getName(), false, null, + refreshMaterializedView(db.getFullName(), getTable(db.getId(), mvId.getId()).getName(), false, null, Constants.TaskRunPriority.NORMAL.value(), true, false); } } @@ -4660,7 +4694,7 @@ public void replayTruncateTable(TruncateTableInfo info) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(info.getTblId()); + OlapTable olapTable = (OlapTable) getTable(db.getId(), info.getTblId()); truncateTableInternal(olapTable, info.getPartitions(), info.isEntireTable(), true); if (!GlobalStateMgr.isCheckpointThread()) { @@ -4724,7 +4758,7 @@ public void replayBackendTabletsInfo(BackendTabletsInfo backendTabletsInfo) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable tbl = (OlapTable) db.getTable(info.getTableId()); + OlapTable tbl = (OlapTable) getTable(db.getId(), info.getTableId()); if (tbl == null) { continue; } @@ -4765,7 +4799,7 @@ public void replayConvertDistributionType(TableInfo tableInfo) { Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable tbl = (OlapTable) db.getTable(tableInfo.getTableId()); + OlapTable tbl = (OlapTable) getTable(db.getId(), tableInfo.getTableId()); LOG.info("replay modify distribution type of table: " + tbl.getName()); } finally { locker.unLockDatabase(db, LockType.WRITE); @@ -4786,7 +4820,7 @@ public void replaceTempPartition(Database db, String tableName, ReplacePartition Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - Table table = db.getTable(tableName); + Table table = getTable(db.getFullName(), tableName); if (table == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); } @@ -4831,7 +4865,7 @@ public void replayReplaceTempPartition(ReplacePartitionOperationLog replaceTempP Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); try { - OlapTable olapTable = (OlapTable) db.getTable(replaceTempPartitionLog.getTblId()); + OlapTable olapTable = (OlapTable) getTable(db.getId(), replaceTempPartitionLog.getTblId()); if (olapTable == null) { return; } @@ -4919,7 +4953,7 @@ public void setPartitionVersion(AdminSetPartitionVersionStmt stmt) { Locker locker = new Locker(); locker.lockDatabase(database, LockType.WRITE); try { - Table table = database.getTable(stmt.getTableName().getTbl()); + Table table = getTable(database.getFullName(), stmt.getTableName().getTbl()); if (table == null) { throw ErrorReportException.report(ErrorCode.ERR_BAD_TABLE_ERROR, stmt.getTableName().getTbl()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java index 4fec339b5ab59..e10ada5c431d3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java @@ -521,14 +521,6 @@ public Table getTable(String catalogName, String dbName, String tblName) { return connectorTable; } - public Table getTable(Long databaseId, Long tableId) { - Database database = localMetastore.getDb(databaseId); - if (database == null) { - return null; - } - return database.getTable(tableId); - } - public TableVersionRange getTableVersionRange(String dbName, Table table, Optional startVersion, Optional endVersion) { @@ -547,7 +539,7 @@ public Optional getDatabase(BaseTableInfo baseTableInfo) { public Optional
getTable(BaseTableInfo baseTableInfo) { if (baseTableInfo.isInternalCatalog()) { - return Optional.ofNullable(getTable(baseTableInfo.getDbId(), baseTableInfo.getTableId())); + return Optional.ofNullable(localMetastore.getTable(baseTableInfo.getDbId(), baseTableInfo.getTableId())); } else { return Optional.ofNullable( getTable(baseTableInfo.getCatalogName(), baseTableInfo.getDbName(), baseTableInfo.getTableName())); @@ -588,7 +580,7 @@ public Table getTemporaryTable(UUID sessionId, String catalogName, Long database if (database == null) { return null; } - return database.getTable(tableId); + return localMetastore.getTable(database.getId(), tableId); } public boolean tableExists(String catalogName, String dbName, String tblName) { diff --git a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java index 832e407bdc651..c981b44577b03 100644 --- a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java @@ -489,7 +489,7 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE } } - Database db = GlobalStateMgr.getCurrentState().getDb(params.db); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(params.db); long limit = params.isSetLimit() ? params.getLimit() : -1; UserIdentity currentUser; if (params.isSetCurrent_user_ident()) { @@ -502,7 +502,8 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE locker.lockDatabase(db, LockType.READ); try { boolean listingViews = params.isSetType() && TTableType.VIEW.equals(params.getType()); - List
tables = listingViews ? db.getViews() : db.getTables(); + List
tables = listingViews ? db.getViews() : + GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); OUTER: for (Table table : tables) { try { @@ -535,7 +536,8 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE try { List allTables = view.getTableRefs(); for (TableName tableName : allTables) { - Table tbl = db.getTable(tableName.getTbl()); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName.getTbl()); if (tbl != null) { try { Authorizer.checkAnyActionOnTableLikeObject(currentUser, @@ -598,7 +600,7 @@ public TListPipesResult listPipes(TListPipesParams params) throws TException { Map pipes = pm.getPipesUnlock(); TListPipesResult result = new TListPipesResult(); for (Pipe pipe : pipes.values()) { - String databaseName = GlobalStateMgr.getCurrentState().mayGetDb(pipe.getPipeId().getDbId()) + String databaseName = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(pipe.getPipeId().getDbId()) .map(Database::getOriginName) .orElse(null); @@ -646,7 +648,7 @@ public TListPipeFilesResult listPipeFiles(TListPipeFilesParams params) throws TE file.setPipe_id(record.pipeId); file.setDatabase_name( mayPipe.flatMap(p -> - GlobalStateMgr.getCurrentState().mayGetDb(p.getDbAndName().first) + GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(p.getDbAndName().first) .map(Database::getOriginName)) .orElse("")); file.setPipe_name(mayPipe.map(Pipe::getName).orElse("")); @@ -694,7 +696,7 @@ private TListMaterializedViewStatusResult listMaterializedViewStatus(long limit, List tablesResult = Lists.newArrayList(); result.setMaterialized_views(tablesResult); String dbName = params.getDb(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { LOG.warn("database not exists: {}", dbName); return result; @@ -765,13 +767,13 @@ private void filterSynchronousMaterializedView(OlapTable olapTable, PatternMatch private List listMaterializedViews(long limit, PatternMatcher matcher, UserIdentity currentUser, TGetTablesParams params) { String dbName = params.getDb(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); List materializedViews = Lists.newArrayList(); List> singleTableMVs = Lists.newArrayList(); Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table.isMaterializedView()) { filterAsynchronousMaterializedView(matcher, currentUser, dbName, (MaterializedView) table, params, materializedViews); @@ -944,13 +946,13 @@ private void describeWithoutDbAndTable(UserIdentity currentUser, List allTables = db.getTables(); + List
allTables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (Table table : allTables) { try { Authorizer.checkAnyActionOnTableLikeObject(result.currentUser, @@ -257,11 +257,11 @@ public static TGetPartitionsMetaResponse generatePartitionsMetaResponse(TGetPart AuthDbRequestResult result = getAuthDbRequestResult(request.getAuth_info()); for (String dbName : result.authorizedDbs) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { continue; } - List
allTables = db.getTables(); + List
allTables = GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()); for (Table table : allTables) { try { Authorizer.checkAnyActionOnTableLikeObject(result.currentUser, @@ -529,7 +529,7 @@ public static TGetTemporaryTablesInfoResponse generateTemporaryTablesInfoRespons for (Map.Entry entry : tableMap.entrySet()) { UUID sessionId = entry.getValue(); Long tableId = entry.getKey(); - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table != null) { TTableInfo info = new TTableInfo(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java index 1b83b6a882131..2f35607191b03 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java @@ -27,6 +27,8 @@ import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; import com.starrocks.common.DuplicatedRequestException; +import com.starrocks.common.ErrorCode; +import com.starrocks.common.ErrorReport; import com.starrocks.common.LabelAlreadyUsedException; import com.starrocks.common.profile.Timer; import com.starrocks.common.profile.Tracers; @@ -430,12 +432,15 @@ private static void beginTransaction(DmlStmt stmt, ConnectContext session) } } - MetaUtils.normalizationTableName(session, stmt.getTableName()); + stmt.getTableName().normalization(session); String catalogName = stmt.getTableName().getCatalog(); String dbName = stmt.getTableName().getDb(); String tableName = stmt.getTableName().getTbl(); - Database db = MetaUtils.getDatabase(catalogName, dbName); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); + if (db == null) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } Table targetTable = MetaUtils.getSessionAwareTable(session, db, stmt.getTableName()); if (targetTable == null) { throw new SemanticException("Table %s is not found", tableName); @@ -521,10 +526,13 @@ private static void abortTransaction(DmlStmt stmt, ConnectContext session, Strin return; } - MetaUtils.normalizationTableName(session, stmt.getTableName()); + stmt.getTableName().normalization(session); String catalogName = stmt.getTableName().getCatalog(); String dbName = stmt.getTableName().getDb(); - Database db = MetaUtils.getDatabase(catalogName, dbName); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); + if (db == null) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } Table targetTable = MetaUtils.getSessionAwareTable(session, db, stmt.getTableName()); try { if (targetTable instanceof ExternalOlapTable) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableStatementAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableStatementAnalyzer.java index ec212d8fff266..9015f51a80950 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableStatementAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableStatementAnalyzer.java @@ -40,7 +40,7 @@ public class AlterTableStatementAnalyzer { public static void analyze(AlterTableStmt statement, ConnectContext context) { TableName tbl = statement.getTbl(); - MetaUtils.normalizationTableName(context, tbl); + tbl.normalization(context); MetaUtils.checkNotSupportCatalog(tbl.getCatalog(), "ALTER"); List alterClauseList = statement.getAlterClauseList(); @@ -50,7 +50,11 @@ public static void analyze(AlterTableStmt statement, ConnectContext context) { checkAlterOpConflict(alterClauseList); - Database db = MetaUtils.getDatabase(context, tbl); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(tbl.getCatalog(), tbl.getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", tbl.getCatalogAndDb()); + } + if (alterClauseList.stream().map(AlterClause::getOpType).anyMatch(AlterOpType::needCheckCapacity)) { try { GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().checkClusterCapacity(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java index d8a74e303dbb5..f7d6755b76a40 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java @@ -100,7 +100,7 @@ public void analyze(StatementBase statement, ConnectContext session) { @Override public Void visitAnalyzeStatement(AnalyzeStmt statement, ConnectContext session) { - MetaUtils.normalizationTableName(session, statement.getTableName()); + statement.getTableName().normalization(session); Table analyzeTable = MetaUtils.getSessionAwareTable(session, null, statement.getTableName()); if (StatisticUtils.statisticDatabaseBlackListCheck(statement.getTableName().getDb())) { @@ -178,7 +178,10 @@ public Void visitCreateAnalyzeJobStatement(CreateAnalyzeJobStmt statement, Conne } if (null != tbl.getDb() && null == tbl.getTbl()) { - Database db = MetaUtils.getDatabase(session, tbl); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(tbl.getCatalog(), tbl.getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", tbl.getCatalogAndDb()); + } if (statement.isNative() && StatisticUtils.statisticDatabaseBlackListCheck(statement.getTableName().getDb())) { @@ -187,8 +190,12 @@ public Void visitCreateAnalyzeJobStatement(CreateAnalyzeJobStmt statement, Conne statement.setDbId(db.getId()); } else if (null != statement.getTableName().getTbl()) { - MetaUtils.normalizationTableName(session, statement.getTableName()); - Database db = MetaUtils.getDatabase(session, statement.getTableName()); + statement.getTableName().normalization(session); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr() + .getDb(statement.getTableName().getCatalog(), statement.getTableName().getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", statement.getTableName().getCatalogAndDb()); + } Table analyzeTable = MetaUtils.getSessionAwareTable(session, db, statement.getTableName()); if (analyzeTable.isTemporaryTable()) { @@ -339,7 +346,7 @@ private void analyzeAnalyzeTypeDesc(ConnectContext session, AnalyzeStmt statemen @Override public Void visitDropStatsStatement(DropStatsStmt statement, ConnectContext session) { - MetaUtils.normalizationTableName(session, statement.getTableName()); + statement.getTableName().normalization(session); if (CatalogMgr.isExternalCatalog(statement.getTableName().getCatalog())) { statement.setExternal(true); } @@ -348,7 +355,7 @@ public Void visitDropStatsStatement(DropStatsStmt statement, ConnectContext sess @Override public Void visitDropHistogramStatement(DropHistogramStmt statement, ConnectContext session) { - MetaUtils.normalizationTableName(session, statement.getTableName()); + statement.getTableName().normalization(session); if (CatalogMgr.isExternalCatalog(statement.getTableName().getCatalog())) { statement.setExternal(true); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzerUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzerUtils.java index a4b9af4f60aed..4e15fbe63e208 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzerUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzerUtils.java @@ -207,7 +207,7 @@ public static Function getDBUdfFunction(ConnectContext context, FunctionName fnN dbName = context.getDatabase(); } - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { return null; } @@ -384,7 +384,7 @@ private void getDB(TableName tableName) { return; } - Database db = session.getGlobalStateMgr().getDb(dbName); + Database db = session.getGlobalStateMgr().getLocalMetastore().getDb(dbName); if (db == null) { return; } @@ -1113,9 +1113,9 @@ public Void visitFileTableFunction(FileTableFunctionRelation node, Void context) public static Set getAllTableNamesForAnalyzeJobStmt(long dbId, long tableId) { Set tableNames = Sets.newHashSet(); if (StatsConstants.DEFAULT_ALL_ID != tableId && StatsConstants.DEFAULT_ALL_ID != dbId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null && !db.isSystemDatabase()) { - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table != null && table.isOlapOrCloudNativeTable()) { tableNames.add(new TableName(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, db.getFullName(), table.getName())); @@ -1134,9 +1134,9 @@ public static Set getAllTableNamesForAnalyzeJobStmt(long dbId, long t } private static void getTableNamesInDb(Set tableNames, Long id) { - Database db = GlobalStateMgr.getCurrentState().getDb(id); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(id); if (db != null && !db.isSystemDatabase()) { - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table == null || !table.isOlapOrCloudNativeTable()) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizationAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizationAnalyzer.java index a3ae0f8124d70..1fce803df76a9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizationAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizationAnalyzer.java @@ -45,7 +45,6 @@ import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.ast.UserIdentity; import com.starrocks.sql.ast.pipe.PipeName; -import com.starrocks.sql.common.MetaUtils; import java.util.ArrayList; import java.util.Arrays; @@ -229,7 +228,7 @@ public List> analyzeFuncPrivToken(BaseGrantRevokePrivilegeStmt funcPrivTokenList.add(new Pair<>(PrivilegeBuiltinConstants.ALL_DATABASE_ID, PrivilegeBuiltinConstants.ALL_FUNCTIONS_ID)); } else { - Database database = GlobalStateMgr.getServingState().getDb(tokens.get(0)); + Database database = GlobalStateMgr.getServingState().getLocalMetastore().getDb(tokens.get(0)); if (database == null) { throw new SemanticException("Database %s is not found", tokens.get(0)); } @@ -262,7 +261,7 @@ public List> analyzeFuncPrivToken(BaseGrantRevokePrivilegeStmt FunctionSearchDesc searchDesc = new FunctionSearchDesc(functionName, argsDef.getArgTypes(), argsDef.isVariadic()); - Database db = GlobalStateMgr.getCurrentState().getDb(functionName.getDb()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(functionName.getDb()); long databaseID = db.getId(); Function function = db.getFunction(searchDesc); @@ -346,10 +345,10 @@ public List> analyzeTokens(BaseGrantRevokePrivilegeStmt stmt, Objec tableName = new TableName(tokens.get(0), tokens.get(1), tokens.get(2)); } else if (tokens.size() == 2) { tableName = new TableName(tokens.get(0), tokens.get(1)); - MetaUtils.normalizationTableName(session, tableName); + tableName.normalization(session); } else if (tokens.size() == 1) { tableName = new TableName("", tokens.get(0)); - MetaUtils.normalizationTableName(session, tableName); + tableName.normalization(session); } else { throw new SemanticException( "Invalid grant statement with error privilege object " + tokens); @@ -366,7 +365,7 @@ public List> analyzeTokens(BaseGrantRevokePrivilegeStmt stmt, Objec tableName = new TableName(tokens.get(0), tokens.get(1)); } else if (tokens.size() == 1) { tableName = new TableName("", tokens.get(0)); - MetaUtils.normalizationTableName(session, tableName); + tableName.normalization(session); } else { throw new SemanticException( "Invalid grant statement with error privilege object " + tokens); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java index 85de25910ea89..c95327e523ece 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java @@ -1569,9 +1569,10 @@ public Void visitAlterTableStatement(AlterTableStmt statement, ConnectContext co @Override public Void visitCancelAlterTableStatement(CancelAlterTableStmt statement, ConnectContext context) { if (statement.getAlterType() == ShowAlterStmt.AlterType.MATERIALIZED_VIEW) { - Database db = GlobalStateMgr.getCurrentState().getDb(statement.getDbName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(statement.getDbName()); if (db != null) { - Table table = db.getTable(statement.getTableName()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), statement.getTableName()); if (table == null || !table.isMaterializedView()) { // ignore privilege check for old mv return null; @@ -2151,7 +2152,7 @@ public Void visitRestoreStatement(RestoreStmt statement, ConnectContext context) } } else { // going to restore some tables in database or some partitions in table - Database db = globalStateMgr.getDb(statement.getDbName()); + Database db = globalStateMgr.getLocalMetastore().getDb(statement.getDbName()); Locker locker = new Locker(); if (db != null) { try { @@ -2168,7 +2169,8 @@ public Void visitRestoreStatement(RestoreStmt statement, ConnectContext context) } // check insert on specified table for (TableRef tableRef : tableRefs) { - Table table = db.getTable(tableRef.getName().getTbl()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableRef.getName().getTbl()); if (table != null) { try { Authorizer.checkTableAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), @@ -2335,7 +2337,7 @@ public Void visitDropFunctionStatement(DropFunctionStmt statement, ConnectContex } // db function. - Database db = GlobalStateMgr.getCurrentState().getDb(functionName.getDb()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(functionName.getDb()); if (db != null) { Locker locker = new Locker(); try { @@ -2534,7 +2536,7 @@ private String getTableNameByRoutineLoadLabel(ConnectContext context, private void checkOperateLoadPrivilege(ConnectContext context, String dbName, String label) { GlobalStateMgr globalStateMgr = context.getGlobalStateMgr(); - Database db = globalStateMgr.getDb(dbName); + Database db = globalStateMgr.getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportSemanticException(ErrorCode.ERR_PRIVILEGE_DB_NOT_FOUND, dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java index 7a5f3ef978bcd..a0660b97cafcb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/BackupRestoreAnalyzer.java @@ -85,7 +85,7 @@ public Void visitBackupStatement(BackupStmt backupStmt, ConnectContext context) // If TableRefs is empty, it means that we do not specify any table in Backup stmt. // We should backup all table in current database. if (tableRefs.size() == 0) { - for (Table tbl : database.getTables()) { + for (Table tbl : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(database.getId())) { if (!Config.enable_backup_materialized_view && tbl.isMaterializedView()) { LOG.info("Skip backup materialized view: {} because " + "`Config.enable_backup_materialized_view=false`", tbl.getName()); @@ -364,7 +364,7 @@ public static String getDbName(String dbName, ConnectContext context) { } public static Database getDatabase(String dbName, ConnectContext context) { - Database db = context.getGlobalStateMgr().getDb(dbName); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } @@ -397,14 +397,14 @@ public static void analyzeTableRef(TableRef tableRef, String dbName, Database db tableName.setDb(dbName); PartitionNames partitionNames = tableRef.getPartitionNames(); - Table tbl = db.getTable(tableName.getTbl()); + Table tbl = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName.getTbl()); if (null == tbl) { throw new SemanticException(ErrorCode.ERR_WRONG_TABLE_NAME.formatErrorMsg(tableName.getTbl())); } String alias = tableRef.getAlias(); if (!tableName.getTbl().equalsIgnoreCase(alias)) { - Table tblAlias = db.getTable(alias); + Table tblAlias = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), alias); if (tblAlias != null && tbl != tblAlias) { ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, "table [" + alias + "] existed"); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CTASAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CTASAnalyzer.java index 6caf15d8b378d..c141c1776e7b2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CTASAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CTASAnalyzer.java @@ -36,7 +36,6 @@ import com.starrocks.sql.ast.PartitionDesc; import com.starrocks.sql.ast.QueryStatement; import com.starrocks.sql.ast.RangePartitionDesc; -import com.starrocks.sql.common.MetaUtils; import com.starrocks.sql.parser.ParsingException; import java.util.HashMap; @@ -88,7 +87,7 @@ public static void analyze(CreateTableAsSelectStmt createTableAsSelectStmt, Conn } TableName tableNameObject = createTableStmt.getDbTbl(); - MetaUtils.normalizationTableName(session, tableNameObject); + tableNameObject.normalization(session); CreateTableAnalyzer.analyzeEngineName(createTableStmt, tableNameObject.getCatalog()); for (int i = 0; i < allFields.size(); i++) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CancelAlterTableStatementAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CancelAlterTableStatementAnalyzer.java index 9ad265610606f..4819a30972202 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CancelAlterTableStatementAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CancelAlterTableStatementAnalyzer.java @@ -19,14 +19,13 @@ import com.starrocks.common.ErrorReport; import com.starrocks.qe.ConnectContext; import com.starrocks.sql.ast.CancelAlterTableStmt; -import com.starrocks.sql.common.MetaUtils; public class CancelAlterTableStatementAnalyzer { public static void analyze(CancelAlterTableStmt statement, ConnectContext context) { - MetaUtils.normalizationTableName(context, statement.getDbTableName()); + statement.getDbTableName().normalization(context); // Check db. - if (context.getGlobalStateMgr().getDb(statement.getDbName()) == null) { + if (context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDbName()) == null) { ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, statement.getDbName()); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java index 4f38a4b1b4945..b7d9e57ea7f06 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java @@ -78,7 +78,7 @@ public class CreateTableAnalyzer { public static void analyze(CreateTableStmt statement, ConnectContext context) { final TableName tableNameObject = statement.getDbTbl(); - MetaUtils.normalizationTableName(context, tableNameObject); + tableNameObject.normalization(context); final String catalogName = tableNameObject.getCatalog(); MetaUtils.checkCatalogExistAndReport(catalogName); @@ -86,11 +86,15 @@ public static void analyze(CreateTableStmt statement, ConnectContext context) { final String tableName = tableNameObject.getTbl(); FeNameFormat.checkTableName(tableName); - Database db = MetaUtils.getDatabase(catalogName, tableNameObject.getDb()); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, tableNameObject.getDb()); + if (db == null) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, tableNameObject.getDb()); + } if (statement instanceof CreateTemporaryTableStmt) { analyzeTemporaryTable(statement, context, catalogName, db, tableName); } else { - if (db.getTable(tableName) != null && !statement.isSetIfNotExists()) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName) != null && !statement.isSetIfNotExists()) { ErrorReport.reportSemanticException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableLikeAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableLikeAnalyzer.java index 7ad846c24e2ea..67cd34ebdbc78 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableLikeAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableLikeAnalyzer.java @@ -16,12 +16,12 @@ import com.google.common.collect.Lists; import com.starrocks.analysis.TableName; -import com.starrocks.catalog.Database; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; import com.starrocks.common.ErrorCode; import com.starrocks.common.ErrorReport; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.CreateTableLikeStmt; import com.starrocks.sql.ast.CreateTableStmt; import com.starrocks.sql.ast.CreateTemporaryTableLikeStmt; @@ -35,14 +35,17 @@ public class CreateTableLikeAnalyzer { public static void analyze(CreateTableLikeStmt stmt, ConnectContext context) { TableName existedDbTbl = stmt.getExistedDbTbl(); - MetaUtils.normalizationTableName(context, stmt.getDbTbl()); - MetaUtils.normalizationTableName(context, existedDbTbl); + stmt.getDbTbl().normalization(context); + existedDbTbl.normalization(context); String tableName = stmt.getTableName(); FeNameFormat.checkTableName(tableName); MetaUtils.checkNotSupportCatalog(existedDbTbl.getCatalog(), "CREATE TABLE LIKE"); - Database db = MetaUtils.getDatabase(context, existedDbTbl); - Table table = MetaUtils.getTable(existedDbTbl); + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(existedDbTbl.getCatalog(), + existedDbTbl.getDb(), existedDbTbl.getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName); + } List createTableStmt = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java index e5668f307b3d3..27b3913f28716 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java @@ -31,6 +31,7 @@ import com.starrocks.analysis.TableName; import com.starrocks.analysis.VariableExpr; import com.starrocks.catalog.Column; +import com.starrocks.catalog.Database; import com.starrocks.catalog.KeysType; import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; @@ -39,6 +40,7 @@ import com.starrocks.common.Config; import com.starrocks.load.Load; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.DeleteStmt; import com.starrocks.sql.ast.JoinRelation; import com.starrocks.sql.ast.LoadStmt; @@ -194,9 +196,13 @@ public static void analyze(DeleteStmt deleteStatement, ConnectContext session) { analyzeProperties(deleteStatement, session); TableName tableName = deleteStatement.getTableName(); - MetaUtils.normalizationTableName(session, tableName); + tableName.normalization(session); MetaUtils.checkNotSupportCatalog(tableName.getCatalog(), "DELETE"); - MetaUtils.getDatabase(session, tableName); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr() + .getDb(tableName.getCatalog(), tableName.getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", tableName.getCatalogAndDb()); + } Table table = MetaUtils.getSessionAwareTable(session, null, tableName); if (table instanceof MaterializedView) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DropStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DropStmtAnalyzer.java index 85dff3beb828a..7ae1aeaaa3d13 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DropStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DropStmtAnalyzer.java @@ -63,7 +63,7 @@ public void analyze(DdlStmt statement, ConnectContext session) { @Override public Void visitDropTableStatement(DropTableStmt statement, ConnectContext context) { - MetaUtils.normalizationTableName(context, statement.getTableNameObject()); + statement.getTableNameObject().normalization(context); // check catalog String catalogName = statement.getCatalogName(); @@ -116,7 +116,7 @@ public Void visitDropTableStatement(DropTableStmt statement, ConnectContext cont @Override public Void visitDropTemporaryTableStatement(DropTemporaryTableStmt statement, ConnectContext context) { statement.setSessionId(context.getSessionId()); - MetaUtils.normalizationTableName(context, statement.getTableNameObject()); + statement.getTableNameObject().normalization(context); // check catalog String catalogName = statement.getCatalogName(); @@ -166,7 +166,7 @@ public Void visitDropDbStatement(DropDbStmt statement, ConnectContext context) { context.getCurrentUserIdentity(), context.getCurrentRoleIds(), PrivilegeType.DROP.name(), ObjectType.DATABASE.name(), dbName); } else if (dbName.equalsIgnoreCase(SysDb.DATABASE_NAME)) { - Database db = GlobalStateMgr.getCurrentState().getDb(SysDb.DATABASE_NAME.toLowerCase()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(SysDb.DATABASE_NAME.toLowerCase()); if (db.getId() == SystemId.SYS_DB_ID) { AccessDeniedException.reportAccessDenied(context.getCurrentCatalog(), context.getCurrentUserIdentity(), context.getCurrentRoleIds(), @@ -197,7 +197,7 @@ public Void visitDropFunctionStatement(DropFunctionStmt statement, ConnectContex ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_FUNC_ERROR, funcDesc.toString()); } } else { - Database db = GlobalStateMgr.getCurrentState().getDb(functionName.getDb()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(functionName.getDb()); if (db != null) { func = db.getFunction(statement.getFunctionSearchDesc()); if (func == null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExportStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExportStmtAnalyzer.java index f6d801a6f3fd6..668665155e1af 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExportStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExportStmtAnalyzer.java @@ -68,7 +68,7 @@ public Void visitExportStatement(ExportStmt statement, ConnectContext context) { GlobalStateMgr mgr = context.getGlobalStateMgr(); TableName tableName = statement.getTableRef().getName(); // make sure catalog, db, table - MetaUtils.normalizationTableName(context, tableName); + tableName.normalization(context); Table table = MetaUtils.getSessionAwareTable(context, null, tableName); if (table.getType() == Table.TableType.OLAP && (((OlapTable) table).getState() == OlapTable.OlapTableState.RESTORE || diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java index 7aab75d74216b..3894dfb492714 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java @@ -1515,11 +1515,11 @@ public Void visitDictQueryExpr(DictQueryExpr node, Scope context) { throw new SemanticException("dict_mapping function first param table_name should be 'db.tbl' or 'tbl' format"); } - Database db = GlobalStateMgr.getCurrentState().getDb(tableName.getDb()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tableName.getDb()); if (db == null) { throw new SemanticException("Database %s is not found", tableName.getDb()); } - Table table = db.getTable(tableName.getTbl()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName.getTbl()); if (table == null) { throw new SemanticException("dict table %s is not found", tableName.getTbl()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/InsertAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/InsertAnalyzer.java index 77e999d911b13..d8269884cefea 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/InsertAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/InsertAnalyzer.java @@ -38,6 +38,7 @@ import com.starrocks.connector.hive.HiveWriteUtils; import com.starrocks.qe.ConnectContext; import com.starrocks.server.CatalogMgr; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.DefaultValueExpr; import com.starrocks.sql.ast.FileTableFunctionRelation; import com.starrocks.sql.ast.InsertStmt; @@ -378,14 +379,17 @@ private static Table getTargetTable(InsertStmt insertStmt, ConnectContext sessio return insertStmt.makeBlackHoleTable(); } - MetaUtils.normalizationTableName(session, insertStmt.getTableName()); + insertStmt.getTableName().normalization(session); String catalogName = insertStmt.getTableName().getCatalog(); String dbName = insertStmt.getTableName().getDb(); String tableName = insertStmt.getTableName().getTbl(); MetaUtils.checkCatalogExistAndReport(catalogName); - Database database = MetaUtils.getDatabase(catalogName, dbName); + Database database = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); + if (database == null) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } Table table = MetaUtils.getSessionAwareTable(session, database, insertStmt.getTableName()); if (table == null) { throw new SemanticException("Table %s is not found", tableName); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/LoadStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/LoadStmtAnalyzer.java index 731d4a71d22b9..6ffbf5c0b6be1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/LoadStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/LoadStmtAnalyzer.java @@ -124,14 +124,15 @@ private void analyzeDataDescriptions(LoadStmt statement) { if (etlJobType == EtlJobType.SPARK && database != null) { for (DataDescription dataDescription : dataDescriptions) { String tableName = dataDescription.getTableName(); - Database db = GlobalStateMgr.getCurrentState().getDb(database); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(database); if (db == null) { continue; } Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), tableName); if (table == null) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java index 8a676603772f2..e3288f563c800 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java @@ -80,7 +80,6 @@ import com.starrocks.sql.ast.SetOperationRelation; import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.ast.ViewRelation; -import com.starrocks.sql.common.MetaUtils; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Optimizer; import com.starrocks.sql.optimizer.base.ColumnRefFactory; @@ -242,7 +241,25 @@ public enum RefreshTimeUnit { public Void visitCreateMaterializedViewStatement(CreateMaterializedViewStatement statement, ConnectContext context) { final TableName tableNameObject = statement.getTableName(); - MetaUtils.normalizeMVName(context, tableNameObject); + /* + * Materialized view name is a little bit different from a normal table + * 1. Use default catalog if not specified, actually it only support default catalog until now + */ + if (com.google.common.base.Strings.isNullOrEmpty(tableNameObject.getCatalog())) { + tableNameObject.setCatalog(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME); + } + if (com.google.common.base.Strings.isNullOrEmpty(tableNameObject.getDb())) { + if (com.google.common.base.Strings.isNullOrEmpty(context.getDatabase())) { + throw new SemanticException("No database selected. " + + "You could set the database name through `.
` or `use ` statement"); + } + tableNameObject.setDb(context.getDatabase()); + } + + if (com.google.common.base.Strings.isNullOrEmpty(tableNameObject.getTbl())) { + throw new SemanticException("Table name cannot be empty"); + } + final String tableName = tableNameObject.getTbl(); FeNameFormat.checkTableName(tableName); QueryStatement queryStatement = statement.getQueryStatement(); @@ -280,7 +297,7 @@ public Void visitCreateMaterializedViewStatement(CreateMaterializedViewStatement "default_catalog through `set catalog ` statement", statement.getTableName().getPos()); } - Database db = context.getGlobalStateMgr().getDb(statement.getTableName().getDb()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getTableName().getDb()); if (db == null) { throw new SemanticException("Can not find database:" + statement.getTableName().getDb(), statement.getTableName().getPos()); @@ -1051,8 +1068,12 @@ public Void visitDropMaterializedViewStatement(DropMaterializedViewStmt stmt, Co @Override public Void visitAlterMaterializedViewStatement(AlterMaterializedViewStmt statement, ConnectContext context) { TableName mvName = statement.getMvName(); - MetaUtils.normalizationTableName(context, mvName); - Table table = MetaUtils.getTable(statement.getMvName()); + mvName.normalization(context); + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(statement.getMvName().getCatalog(), + statement.getMvName().getDb(), statement.getMvName().getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", mvName); + } if (!(table instanceof MaterializedView)) { throw new SemanticException(mvName.getTbl() + " is not async materialized view", mvName.getPos()); } @@ -1067,11 +1088,12 @@ public Void visitRefreshMaterializedViewStatement(RefreshMaterializedViewStateme ConnectContext context) { statement.getMvName().normalization(context); TableName mvName = statement.getMvName(); - Database db = context.getGlobalStateMgr().getDb(mvName.getDb()); + Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(mvName.getDb()); if (db == null) { throw new SemanticException("Can not find database:" + mvName.getDb(), mvName.getPos()); } - OlapTable table = (OlapTable) db.getTable(mvName.getTbl()); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), mvName.getTbl()); if (table == null) { throw new SemanticException("Can not find materialized view:" + mvName.getTbl(), mvName.getPos()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java index bdcb5d9d9d61a..fdaba853272ad 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/QueryAnalyzer.java @@ -418,7 +418,7 @@ private Relation resolveTableRef(Relation relation, Scope scope, Set } TableName resolveTableName = relation.getResolveTableName(); - MetaUtils.normalizationTableName(session, resolveTableName); + resolveTableName.normalization(session); if (aliasSet.contains(resolveTableName)) { ErrorReport.reportSemanticException(ErrorCode.ERR_NONUNIQ_TABLE, relation.getResolveTableName().getTbl()); @@ -1284,7 +1284,7 @@ public Scope visitNormalizedTableFunction(NormalizedTableFunctionRelation node, public Table resolveTable(TableRelation tableRelation) { TableName tableName = tableRelation.getName(); try { - MetaUtils.normalizationTableName(session, tableName); + tableName.normalization(session); String catalogName = tableName.getCatalog(); String dbName = tableName.getDb(); String tbName = tableName.getTbl(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverPartitionAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverPartitionAnalyzer.java index 858f9ef7835e8..27176fad835ec 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverPartitionAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverPartitionAnalyzer.java @@ -17,11 +17,10 @@ import com.starrocks.analysis.TableName; import com.starrocks.qe.ConnectContext; import com.starrocks.sql.ast.RecoverPartitionStmt; -import com.starrocks.sql.common.MetaUtils; public class RecoverPartitionAnalyzer { public static void analyze(RecoverPartitionStmt statement, ConnectContext context) { TableName tbl = statement.getDbTblName(); - MetaUtils.normalizationTableName(context, tbl); + tbl.normalization(context); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverTableAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverTableAnalyzer.java index b2272e83db05e..2632562c0eb60 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverTableAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RecoverTableAnalyzer.java @@ -17,12 +17,11 @@ import com.starrocks.analysis.TableName; import com.starrocks.qe.ConnectContext; import com.starrocks.sql.ast.RecoverTableStmt; -import com.starrocks.sql.common.MetaUtils; public class RecoverTableAnalyzer { public static void analyze(RecoverTableStmt statement, ConnectContext context) { TableName tableName = statement.getTableNameObject(); - MetaUtils.normalizationTableName(context, tableName); + tableName.normalization(context); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RefreshTableStatementAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RefreshTableStatementAnalyzer.java index cc910c60e7e32..16185ea03c2a9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RefreshTableStatementAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/RefreshTableStatementAnalyzer.java @@ -24,7 +24,6 @@ import com.starrocks.sql.ast.AstVisitor; import com.starrocks.sql.ast.DdlStmt; import com.starrocks.sql.ast.RefreshTableStmt; -import com.starrocks.sql.common.MetaUtils; public class RefreshTableStatementAnalyzer { public static void analyze(RefreshTableStmt statement, ConnectContext context) { @@ -45,7 +44,7 @@ public void analyze(DdlStmt statement, ConnectContext session) { @Override public Void visitRefreshTableStatement(RefreshTableStmt statement, ConnectContext context) { TableName tableName = statement.getTableName(); - MetaUtils.normalizationTableName(context, tableName); + tableName.normalization(context); String catalogName = tableName.getCatalog(); String dbName = tableName.getDb(); String tblName = tableName.getTbl(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ResourceGroupAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ResourceGroupAnalyzer.java index 85647a288b9a2..5cf7587c2a36a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ResourceGroupAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ResourceGroupAnalyzer.java @@ -84,7 +84,7 @@ public static ResourceGroupClassifier convertPredicateToClassifier(List databaseIds = new ArrayList<>(); for (String name : databases) { - Database db = GlobalStateMgr.getCurrentState().getDb(name); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(name); if (db == null) { throw new SemanticException(String.format("Specified database not exists: %s", name)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowAlterStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowAlterStmtAnalyzer.java index 80aae1f97f7a7..df71cb69fd80b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowAlterStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowAlterStmtAnalyzer.java @@ -68,7 +68,7 @@ public Void visitShowAlterStatement(ShowAlterStmt statement, ConnectContext cont private void handleShowAlterTable(ShowAlterStmt statement, ConnectContext context) throws SemanticException { // build proc path - @Nonnull Database db = context.getGlobalStateMgr().getDb(statement.getDbName()); + @Nonnull Database db = context.getGlobalStateMgr().getLocalMetastore().getDb(statement.getDbName()); ShowAlterStmt.AlterType type = statement.getType(); StringBuilder sb = new StringBuilder(); sb.append("/jobs/"); @@ -102,7 +102,7 @@ public void analyzeSyntax(ShowAlterStmt statement, ConnectContext context) { } statement.setDbName(dbName); // Check db. - if (context.getGlobalStateMgr().getDb(dbName) == null) { + if (context.getGlobalStateMgr().getLocalMetastore().getDb(dbName) == null) { ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java index 7c7159d91b72b..5c319e1176ebd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java @@ -332,7 +332,7 @@ public Void visitDescTableStmt(DescribeStmt node, ConnectContext context) { } private void descInternalCatalogTable(DescribeStmt node, ConnectContext context) { - Database db = GlobalStateMgr.getCurrentState().getDb(node.getDb()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(node.getDb()); if (db == null) { ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, node.getDb()); } @@ -348,7 +348,7 @@ private void descInternalCatalogTable(DescribeStmt node, ConnectContext context) } //if getTable not find table, may be is statement "desc materialized-view-name" if (table == null) { - for (Table tb : db.getTables()) { + for (Table tb : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (tb.getType() == Table.TableType.OLAP) { OlapTable olapTable = (OlapTable) tb; for (MaterializedIndexMeta mvMeta : olapTable.getVisibleIndexMetas()) { @@ -535,7 +535,7 @@ public Void visitShowPartitionsStatement(ShowPartitionsStmt statement, ConnectCo if (statement.getWhereClause() != null) { analyzeSubPredicate(filterMap, statement.getWhereClause()); } - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowTabletStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowTabletStmtAnalyzer.java index 8a416b0160618..4f0944a8539e3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowTabletStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowTabletStmtAnalyzer.java @@ -94,7 +94,7 @@ public Void visitShowTabletStatement(ShowTabletStmt statement, ConnectContext co // order by List orderByElements = statement.getOrderByElements(); if (orderByElements != null && !orderByElements.isEmpty()) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { throw new SemanticException("Database %s is not found", dbName); } @@ -103,7 +103,7 @@ public Void visitShowTabletStatement(ShowTabletStmt statement, ConnectContext co Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - table = db.getTable(tableName); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { throw new SemanticException("Table %s is not found", tableName); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/TruncateTableAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/TruncateTableAnalyzer.java index eefb61f4927ca..2715334fd4d77 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/TruncateTableAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/TruncateTableAnalyzer.java @@ -19,12 +19,11 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.sql.ast.PartitionNames; import com.starrocks.sql.ast.TruncateTableStmt; -import com.starrocks.sql.common.MetaUtils; public class TruncateTableAnalyzer { public static void analyze(TruncateTableStmt statement, ConnectContext context) { - MetaUtils.normalizationTableName(context, statement.getTblRef().getName()); + statement.getTblRef().getName().normalization(context); if (statement.getTblRef().hasExplicitAlias()) { throw new SemanticException("Not support truncate table with alias"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/UpdateAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/UpdateAnalyzer.java index 36b56904eb350..5f24949478cca 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/UpdateAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/UpdateAnalyzer.java @@ -23,11 +23,13 @@ import com.starrocks.analysis.StringLiteral; import com.starrocks.analysis.TableName; import com.starrocks.catalog.Column; +import com.starrocks.catalog.Database; import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; import com.starrocks.catalog.Type; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.SelectAnalyzer.RewriteAliasVisitor; import com.starrocks.sql.ast.ColumnAssignment; import com.starrocks.sql.ast.DefaultValueExpr; @@ -69,8 +71,12 @@ public static void analyze(UpdateStmt updateStmt, ConnectContext session) { analyzeProperties(updateStmt, session); TableName tableName = updateStmt.getTableName(); - MetaUtils.normalizationTableName(session, tableName); - MetaUtils.getDatabase(session, tableName); + tableName.normalization(session); + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr() + .getDb(tableName.getCatalog(), tableName.getDb()); + if (db == null) { + throw new SemanticException("Database %s is not found", tableName.getCatalogAndDb()); + } Table table = MetaUtils.getSessionAwareTable(session, null, tableName); if (table instanceof MaterializedView) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ViewAnalyzer.java index 5f63ed9dec044..1d1df0f360a3d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ViewAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ViewAnalyzer.java @@ -20,6 +20,7 @@ import com.starrocks.common.ErrorCode; import com.starrocks.common.ErrorReport; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.AlterClause; import com.starrocks.sql.ast.AlterViewClause; import com.starrocks.sql.ast.AlterViewStmt; @@ -28,7 +29,6 @@ import com.starrocks.sql.ast.CreateViewStmt; import com.starrocks.sql.ast.QueryRelation; import com.starrocks.sql.ast.StatementBase; -import com.starrocks.sql.common.MetaUtils; import java.util.HashSet; import java.util.List; @@ -67,7 +67,11 @@ public Void visitAlterViewStatement(AlterViewStmt stmt, ConnectContext context) final String tableName = stmt.getTableName().getTbl(); FeNameFormat.checkTableName(tableName); - Table table = MetaUtils.getTable(stmt.getTableName()); + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(stmt.getTableName().getCatalog(), + stmt.getTableName().getDb(), stmt.getTableName().getTbl()); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName); + } if (table.isConnectorView()) { throw new SemanticException("cannot alter connector view"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DataDescription.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DataDescription.java index 7beca40a5be50..273a775ffe762 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DataDescription.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DataDescription.java @@ -28,7 +28,6 @@ import com.starrocks.analysis.ParseNode; import com.starrocks.analysis.SlotRef; import com.starrocks.analysis.StringLiteral; -import com.starrocks.analysis.TableName; import com.starrocks.catalog.Column; import com.starrocks.catalog.FunctionSet; import com.starrocks.catalog.InternalCatalog; @@ -41,9 +40,10 @@ import com.starrocks.privilege.ObjectType; import com.starrocks.privilege.PrivilegeType; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.AstToSQLBuilder; import com.starrocks.sql.analyzer.Authorizer; -import com.starrocks.sql.common.MetaUtils; +import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.sql.parser.NodePosition; import com.starrocks.thrift.TNetworkAddress; import org.apache.logging.log4j.LogManager; @@ -673,7 +673,11 @@ public void analyze(String fullDbName) throws AnalysisException { } public void analyzeTable(String fullDbName) throws AnalysisException { - Table table = MetaUtils.getTable(ConnectContext.get(), new TableName(fullDbName, tableName)); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(fullDbName, tableName); + if (table == null) { + throw new SemanticException("Table %s is not found", tableName.toString()); + } + if (table instanceof MaterializedView) { throw new AnalysisException(String.format( "The data of '%s' cannot be inserted because '%s' is a materialized view," + diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ExportStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ExportStmt.java index 20cf7ec114fec..34bed282cc1b9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ExportStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ExportStmt.java @@ -178,11 +178,11 @@ public boolean isIncludeQueryId() { } public void checkTable(GlobalStateMgr globalStateMgr) { - Database db = globalStateMgr.getDb(tblName.getDb()); + Database db = globalStateMgr.getLocalMetastore().getDb(tblName.getDb()); if (db == null) { throw new SemanticException("Db does not exist. name: " + tblName.getDb()); } - Table table = db.getTable(tblName.getTbl()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName.getTbl()); if (table == null) { throw new SemanticException("Table[" + tblName.getTbl() + "] does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowAnalyzeStatusStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowAnalyzeStatusStmt.java index 6a8e3755081db..ff3f16ba9a6e1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowAnalyzeStatusStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowAnalyzeStatusStmt.java @@ -25,9 +25,9 @@ import com.starrocks.privilege.AccessDeniedException; import com.starrocks.qe.ConnectContext; import com.starrocks.qe.ShowResultSetMetaData; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.Authorizer; import com.starrocks.sql.analyzer.SemanticException; -import com.starrocks.sql.common.MetaUtils; import com.starrocks.sql.parser.NodePosition; import com.starrocks.statistic.AnalyzeStatus; import com.starrocks.statistic.StatisticUtils; @@ -74,8 +74,11 @@ public static List showAnalyzeStatus(ConnectContext context, Table table; // In new privilege framework(RBAC), user needs any action on the table to show analysis status for it. try { - table = MetaUtils.getTable(analyzeStatus.getCatalogName(), analyzeStatus.getDbName(), - analyzeStatus.getTableName()); + table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable( + analyzeStatus.getCatalogName(), analyzeStatus.getDbName(), analyzeStatus.getTableName()); + if (table == null) { + throw new SemanticException("Table %s is not found", analyzeStatus.getTableName()); + } Authorizer.checkAnyActionOnTableLikeObject(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), analyzeStatus.getDbName(), table); } catch (AccessDeniedException | SemanticException e) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowBasicStatsMetaStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowBasicStatsMetaStmt.java index 9c1fbcd88057b..1a9a5e214dfa5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowBasicStatsMetaStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowBasicStatsMetaStmt.java @@ -65,12 +65,12 @@ public static List showBasicStatsMeta(ConnectContext context, long tableId = basicStatsMeta.getTableId(); List columns = basicStatsMeta.getColumns(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("No found database: " + dbId); } row.set(0, db.getOriginName()); - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new MetaNotFoundException("No found table: " + tableId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowHistogramStatsMetaStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowHistogramStatsMetaStmt.java index c67f5c30e9d38..d63ffd565bc24 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowHistogramStatsMetaStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowHistogramStatsMetaStmt.java @@ -62,12 +62,12 @@ public static List showHistogramStatsMeta(ConnectContext context, long dbId = histogramStatsMeta.getDbId(); long tableId = histogramStatsMeta.getTableId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("No found database: " + dbId); } row.set(0, db.getOriginName()); - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new MetaNotFoundException("No found table: " + tableId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/pipe/ShowPipeStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/pipe/ShowPipeStmt.java index 8322c49c86a3d..388185a511012 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/pipe/ShowPipeStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/pipe/ShowPipeStmt.java @@ -69,7 +69,7 @@ public ShowPipeStmt(String dbName, String like, Expr where, List * NOTE: Must be consistent with the META_DATA */ public static void handleShow(List row, Pipe pipe) { - Optional db = GlobalStateMgr.getCurrentState().mayGetDb(pipe.getPipeId().getDbId()); + Optional db = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(pipe.getPipeId().getDbId()); row.add(db.map(Database::getFullName).orElse("")); row.add(String.valueOf(pipe.getPipeId().getId())); row.add(pipe.getName()); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/MetaUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/MetaUtils.java index 854711fc25446..8231c76e8834b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/MetaUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/MetaUtils.java @@ -24,7 +24,6 @@ import com.starrocks.catalog.ColumnId; import com.starrocks.catalog.Database; import com.starrocks.catalog.ExternalOlapTable; -import com.starrocks.catalog.InternalCatalog; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; import com.starrocks.common.ErrorCode; @@ -91,70 +90,6 @@ public static void checkNotSupportCatalog(String catalogName, String operation) } } - public static Database getDatabase(long dbId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - if (db == null) { - throw new SemanticException("Database %s is not found", dbId); - } - return db; - } - - public static Table getTable(long dbId, long tableId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); - if (db == null) { - throw new SemanticException("Database %s is not found", dbId); - } - Table table = db.getTable(tableId); - if (table == null) { - throw new SemanticException("Table %s is not found", tableId); - } - return table; - } - - public static Database getDatabase(ConnectContext session, TableName tableName) { - if (Strings.isNullOrEmpty(tableName.getCatalog())) { - tableName.setCatalog(session.getCurrentCatalog()); - } - Database db = session.getGlobalStateMgr().getMetadataMgr().getDb(tableName.getCatalog(), tableName.getDb()); - if (db == null) { - throw new SemanticException("Database %s is not found", tableName.getCatalogAndDb()); - } - return db; - } - - public static Database getDatabase(String catalogName, String dbName) { - Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); - if (db == null) { - ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_DB_ERROR, dbName); - } - return db; - } - - public static Table getTable(TableName tableName) { - if (Strings.isNullOrEmpty(tableName.getCatalog())) { - tableName.setCatalog(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME); - } - Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(tableName.getCatalog(), - tableName.getDb(), tableName.getTbl()); - if (table == null) { - throw new SemanticException("Table %s is not found", tableName); - } - return table; - } - - public static Table getTable(ConnectContext session, TableName tableName) { - if (Strings.isNullOrEmpty(tableName.getCatalog())) { - tableName.setCatalog(session.getCurrentCatalog()); - } - Table table = session.getGlobalStateMgr().getMetadataMgr().getTable(tableName.getCatalog(), - tableName.getDb(), tableName.getTbl()); - - if (table == null) { - throw new SemanticException("Table %s is not found", tableName.toString()); - } - return table; - } - // get table by tableName, unlike getTable, this interface is session aware, // which means if there is a temporary table with the same name, // use temporary table first, otherwise, treat it as a permanent table @@ -163,7 +98,13 @@ public static Table getSessionAwareTable(ConnectContext session, Database databa tableName.setCatalog(session.getCurrentCatalog()); } if (database == null) { - database = getDatabase(session, tableName); + if (Strings.isNullOrEmpty(tableName.getCatalog())) { + tableName.setCatalog(session.getCurrentCatalog()); + } + database = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(tableName.getCatalog(), tableName.getDb()); + if (database == null) { + throw new SemanticException("Database %s is not found", tableName.getCatalogAndDb()); + } } Table table = session.getGlobalStateMgr().getMetadataMgr().getTemporaryTable( @@ -179,54 +120,6 @@ public static Table getSessionAwareTable(ConnectContext session, Database databa return table; } - public static Table getTable(String catalogName, String dbName, String tableName) { - Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(catalogName, dbName, tableName); - if (table == null) { - throw new SemanticException("Table %s is not found", tableName); - } - return table; - } - - public static void normalizationTableName(ConnectContext connectContext, TableName tableName) { - if (Strings.isNullOrEmpty(tableName.getCatalog())) { - if (Strings.isNullOrEmpty(connectContext.getCurrentCatalog())) { - throw new SemanticException("No catalog selected"); - } - tableName.setCatalog(connectContext.getCurrentCatalog()); - } - if (Strings.isNullOrEmpty(tableName.getDb())) { - if (Strings.isNullOrEmpty(connectContext.getDatabase())) { - throw new SemanticException("No database selected"); - } - tableName.setDb(connectContext.getDatabase()); - } - - if (Strings.isNullOrEmpty(tableName.getTbl())) { - throw new SemanticException("Table name is null"); - } - } - - /** - * Materialized view name is a little bit different from a normal table - * 1. Use default catalog if not specified, actually it only support default catalog until now - */ - public static void normalizeMVName(ConnectContext connectContext, TableName tableName) { - if (Strings.isNullOrEmpty(tableName.getCatalog())) { - tableName.setCatalog(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME); - } - if (Strings.isNullOrEmpty(tableName.getDb())) { - if (Strings.isNullOrEmpty(connectContext.getDatabase())) { - throw new SemanticException("No database selected. " + - "You could set the database name through `.
` or `use ` statement"); - } - tableName.setDb(connectContext.getDatabase()); - } - - if (Strings.isNullOrEmpty(tableName.getTbl())) { - throw new SemanticException("Table name cannot be empty"); - } - } - public static Map parseColumnNameToDefineExpr(OriginStatement originStmt) { CreateMaterializedViewStmt stmt; @@ -267,12 +160,12 @@ public static ExternalOlapTable syncOLAPExternalTableMeta(ExternalOlapTable exte } public static boolean isPartitionExist(GlobalStateMgr stateMgr, long dbId, long tableId, long partitionId) { - Database db = stateMgr.getDb(dbId); + Database db = stateMgr.getLocalMetastore().getDb(dbId); if (db == null) { return false; } // lake table or lake materialized view - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) stateMgr.getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { return false; } @@ -326,7 +219,11 @@ public static List getColumnNamesByColumnIds(Map idToC } public static Column getColumnByColumnName(long dbId, long tableId, String columnName) { - Table table = getTable(dbId, tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(dbId, tableId); + if (table == null) { + throw new SemanticException("Table %s is not found", tableId); + } + Column column = table.getColumn(columnName); if (column == null) { throw new SemanticException(String.format("can not find column by name: %s", columnName)); @@ -343,7 +240,10 @@ public static Map buildNameToColumn(List schema) { } public static String getColumnNameByColumnId(long dbId, long tableId, ColumnId columnId) { - Table table = getTable(dbId, tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(dbId, tableId); + if (table == null) { + throw new SemanticException("Table %s is not found", tableId); + } Column column = table.getColumn(columnId); if (column == null) { throw new SemanticException(String.format("can not find column by column id: %s", columnId)); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/SqlWithIdUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/SqlWithIdUtils.java index 30230fd2d44f2..a0577f2624d61 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/SqlWithIdUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/SqlWithIdUtils.java @@ -80,7 +80,7 @@ public static StatementBase decode(String sql, ConnectContext context) { long dbId = Long.parseLong(dbStr.substring(0, dbStr.indexOf(COMMON_SUFFIX))); Database db = dbMap.get(dbId); if (db == null) { - db = GlobalStateMgr.getCurrentState().getDb(dbId); + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new SemanticException("Can not find db id: %s", dbId); } @@ -90,7 +90,7 @@ public static StatementBase decode(String sql, ConnectContext context) { long tableId = Long.parseLong(tableStr.substring(7, tableStr.indexOf(COMMON_SUFFIX))); Table table = tableMap.get(tableId); if (table == null) { - table = db.getTable(tableId); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new SemanticException("Can not find table id: %s in db: %s", tableId, db.getOriginName()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/SyncPartitionUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/SyncPartitionUtils.java index f3803b1ba2029..df2aa544f34dc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/SyncPartitionUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/SyncPartitionUtils.java @@ -716,11 +716,11 @@ private static void dropBaseVersionMetaForOlapTable(MaterializedView mv, String } Expr expr = mv.getPartitionRefTableExprs().get(0); - Database baseDb = GlobalStateMgr.getCurrentState().getDb(tableName.getDb()); + Database baseDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tableName.getDb()); if (baseDb == null) { return; } - Table baseTable = baseDb.getTable(tableName.getTbl()); + Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(baseDb.getFullName(), tableName.getTbl()); if (baseTable == null) { return; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java index d178092d794cb..0ebcfd138e1be 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java @@ -657,7 +657,7 @@ private Set getTableRelatedSyncMVs(OlapTable olapTable) { long dbId = indexMeta.getDbId(); String viewDefineSql = indexMeta.getViewDefineSql(); String mvName = olapTable.getIndexNameById(indexId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); // distribution info DistributionInfo baseTableDistributionInfo = olapTable.getDefaultDistributionInfo(); @@ -865,7 +865,7 @@ public static MaterializationContext buildMaterializationContext(OptimizerContex // Add mv info into dump info if (context.getDumpInfo() != null) { - String dbName = GlobalStateMgr.getCurrentState().getDb(mv.getDbId()).getFullName(); + String dbName = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mv.getDbId()).getFullName(); context.getDumpInfo().addTable(dbName, mv); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java index 03b17a91ff05f..f122ef102405f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java @@ -101,9 +101,9 @@ public static Table inspectExternalTable(TableName tableName) { } public static Pair inspectTable(TableName tableName) { - Database db = GlobalStateMgr.getCurrentState().mayGetDb(tableName.getDb()) + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(tableName.getDb()) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_DB_ERROR, tableName.getDb())); - Table table = db.tryGetTable(tableName.getTbl()) + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetTable(tableName.getDb(), tableName.getTbl()) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName)); ConnectContext connectContext = ConnectContext.get(); try { @@ -167,7 +167,7 @@ public static ConstantOperator inspectRelatedMv(ConstantOperator name) { Optional mayDb; Table table = inspectExternalTable(tableName); if (table.isNativeTableOrMaterializedView()) { - mayDb = GlobalStateMgr.getCurrentState().mayGetDb(tableName.getDb()); + mayDb = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(tableName.getDb()); } else { mayDb = Optional.empty(); } @@ -179,7 +179,7 @@ public static ConstantOperator inspectRelatedMv(ConstantOperator name) { Set relatedMvs = table.getRelatedMaterializedViews(); JsonArray array = new JsonArray(); for (MvId mv : SetUtils.emptyIfNull(relatedMvs)) { - String mvName = GlobalStateMgr.getCurrentState().mayGetTable(mv.getDbId(), mv.getId()) + String mvName = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetTable(mv.getDbId(), mv.getId()) .map(Table::getName) .orElse(null); JsonObject obj = new JsonObject(); @@ -246,7 +246,7 @@ public static ConstantOperator inspectAllPipes() { ConnectContext connectContext = ConnectContext.get(); authOperatorPrivilege(); String currentDb = connectContext.getDatabase(); - Database db = GlobalStateMgr.getCurrentState().mayGetDb(connectContext.getDatabase()) + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().mayGetDb(connectContext.getDatabase()) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_DB_ERROR, currentDb)); String json = GlobalStateMgr.getCurrentState().getPipeManager().getPipesOfDb(db.getId()); return ConstantOperator.createVarchar(json); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java index c04747dcf05e7..5fad4d24c465f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MaterializedViewTransparentRewriteRule.java @@ -136,9 +136,9 @@ private OptExpression doGetMvTransparentPlan(ConnectContext connectContext, LogicalOlapScanOperator olapScanOperator, OptExpression input) { // Fetch mv from catalog again since table from olap scan operator is copied. - Database db = GlobalStateMgr.getCurrentState().getDb(mvId.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvId.getDbId()); Preconditions.checkState(db != null, "Database not found: %s", mvId.getDbId()); - Table table = db.getTable(mvId.getId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId.getId()); Preconditions.checkState(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; MvPlanContext mvPlanContext = MvUtils.getMVPlanContext(connectContext, mv, true); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index ebd3a55947960..60b03db80bfb8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -174,12 +174,12 @@ public static void getRelatedMvs(ConnectContext connectContext, } Set
newMvs = Sets.newHashSet(); for (MvId mvId : newMvIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(mvId.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvId.getDbId()); if (db == null) { logMVPrepare("Cannot find database from mvId:{}", mvId); continue; } - Table table = db.getTable(mvId.getId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), mvId.getId()); if (table == null) { logMVPrepare("Cannot find materialized view from mvId:{}", mvId); continue; @@ -1151,7 +1151,8 @@ public static void inactiveRelatedMaterializedViews(Database db, } // inactive related asynchronous mvs for (MvId mvId : olapTable.getRelatedMaterializedViews()) { - MaterializedView mv = (MaterializedView) db.getTable(mvId.getId()); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), mvId.getId()); if (mv == null) { LOG.warn("Ignore materialized view {} does not exists", mvId); continue; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java index b1105eb59988f..8331dca6bbab9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java @@ -164,8 +164,8 @@ public boolean hasGlobalDict(long tableId, ColumnId columnName, long versionTime Set dbIds = ConnectContext.get().getCurrentSqlDbIds(); for (Long id : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(id); - if (db != null && db.getTable(tableId) != null) { + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(id); + if (db != null && GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId) != null) { columnIdentifier.setDbId(db.getId()); break; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java index 5d4506e88478d..fe49a46a9eb28 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java @@ -130,9 +130,9 @@ private List queryStatisticsData(ConnectContext context, long ta } private ColumnStatistic convert2ColumnStatistics(TStatisticData statisticData) throws AnalysisException { - Database db = GlobalStateMgr.getCurrentState().getDb(statisticData.dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(statisticData.dbId); MetaUtils.checkDbNullAndReport(db, String.valueOf(statisticData.dbId)); - Table table = db.getTable(statisticData.tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), statisticData.tableId); if (!(table instanceof OlapTable)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR, statisticData.tableId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnHistogramStatsCacheLoader.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnHistogramStatsCacheLoader.java index d95fdeb3245d5..c2921e0880a6e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnHistogramStatsCacheLoader.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnHistogramStatsCacheLoader.java @@ -118,9 +118,9 @@ public List queryHistogramStatistics(ConnectContext context, lon } private Histogram convert2Histogram(TStatisticData statisticData) throws AnalysisException { - Database db = GlobalStateMgr.getCurrentState().getDb(statisticData.dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(statisticData.dbId); MetaUtils.checkDbNullAndReport(db, String.valueOf(statisticData.dbId)); - Table table = db.getTable(statisticData.tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), statisticData.tableId); if (!(table instanceof OlapTable)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR, statisticData.tableId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java index 788d09c39fb3b..126fda67754df 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java @@ -267,7 +267,7 @@ public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext, // Create a fake table sink here, replaced it after created the MV PartitionInfo partitionInfo = LocalMetastore.buildPartitionInfo(createStmt); long mvId = GlobalStateMgr.getCurrentState().getNextId(); - long dbId = GlobalStateMgr.getCurrentState().getDb(createStmt.getTableName().getDb()).getId(); + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createStmt.getTableName().getDb()).getId(); MaterializedView view = GlobalStateMgr.getCurrentState().getMaterializedViewMgr() .createSinkTable(createStmt, partitionInfo, mvId, dbId); TupleDescriptor tupleDesc = buildTupleDesc(execPlan, view); diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/AnalyzeMgr.java b/fe/fe-core/src/main/java/com/starrocks/statistic/AnalyzeMgr.java index 75c80b1671f67..7079003a4781f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/AnalyzeMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/AnalyzeMgr.java @@ -43,7 +43,6 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.SemanticException; -import com.starrocks.sql.common.MetaUtils; import com.starrocks.transaction.InsertTxnCommitAttachment; import com.starrocks.transaction.TransactionState; import com.starrocks.transaction.TxnCommitAttachment; @@ -265,10 +264,8 @@ public void replayRemoveExternalBasicStatsMeta(ExternalBasicStatsMeta basicStats } public void refreshBasicStatisticsCache(Long dbId, Long tableId, List columns, boolean async) { - Table table; - try { - table = MetaUtils.getTable(dbId, tableId); - } catch (SemanticException e) { + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(dbId, tableId); + if (table == null) { return; } @@ -284,10 +281,9 @@ public void refreshBasicStatisticsCache(Long dbId, Long tableId, List co public void refreshConnectorTableBasicStatisticsCache(String catalogName, String dbName, String tableName, List columns, boolean async) { - Table table; - try { - table = MetaUtils.getTable(catalogName, dbName, tableName); - } catch (Exception e) { + + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(catalogName, dbName, tableName); + if (table == null) { return; } @@ -328,11 +324,11 @@ public void replayAddHistogramStatsMeta(HistogramStatsMeta histogramStatsMeta) { } public void refreshHistogramStatisticsCache(Long dbId, Long tableId, List columns, boolean async) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (null == db) { return; } - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (null == table) { return; } @@ -377,10 +373,8 @@ public void replayRemoveExternalHistogramStatsMeta(ExternalHistogramStatsMeta hi public void refreshConnectorTableHistogramStatisticsCache(String catalogName, String dbName, String tableName, List columns, boolean async) { - Table table; - try { - table = MetaUtils.getTable(catalogName, dbName, tableName); - } catch (Exception e) { + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(catalogName, dbName, tableName); + if (table == null) { return; } @@ -396,12 +390,12 @@ public void clearStatisticFromDroppedTable() { List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); Set tables = new HashSet<>(); for (Long dbId : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (null == db || StatisticUtils.statisticDatabaseBlackListCheck(db.getFullName())) { continue; } - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { /* * If the meta contains statistical information, but the data is empty, * it means that the table has been truncate or insert overwrite, and it is set to empty, @@ -475,9 +469,11 @@ private void clearStalePartitionStats() { && analyzeStatus.getStatus() == StatsConstants.ScheduleStatus.FINISH && Duration.between(endTime, lastCleanTime).toMinutes() < 30) { NativeAnalyzeStatus nativeAnalyzeStatus = (NativeAnalyzeStatus) analyzeStatus; - Database db = GlobalStateMgr.getCurrentState().getDb(nativeAnalyzeStatus.getDbId()); - if (db != null && db.getTable(nativeAnalyzeStatus.getTableId()) != null) { - tables.add(db.getTable(nativeAnalyzeStatus.getTableId())); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(nativeAnalyzeStatus.getDbId()); + if (db != null && GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), nativeAnalyzeStatus.getTableId()) != null) { + tables.add(GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), nativeAnalyzeStatus.getTableId())); } } } @@ -525,12 +521,12 @@ private void clearStaleStatsWhenStarted() { checkTableIds.clear(); List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (null == db || StatisticUtils.statisticDatabaseBlackListCheck(db.getFullName())) { continue; } - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { if (table == null || !(table.isOlapOrCloudNativeTable() || table.isMaterializedView())) { continue; } @@ -547,12 +543,12 @@ private void clearStaleStatsWhenStarted() { int exprLimit = Config.expr_children_limit / 2; for (Pair dbTableId : checkTableIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbTableId.first); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbTableId.first); if (null == db) { continue; } - Table table = db.getTable(dbTableId.second); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), dbTableId.second); if (table == null) { continue; } @@ -676,7 +672,7 @@ public ExecutorService getAnalyzeTaskThreadPool() { } public void updateLoadRows(TransactionState transactionState) { - Database db = GlobalStateMgr.getCurrentState().getDb(transactionState.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(transactionState.getDbId()); if (null == db || StatisticUtils.statisticDatabaseBlackListCheck(db.getFullName())) { return; } @@ -703,7 +699,8 @@ public void updateLoadRows(TransactionState transactionState) { long tableId = transactionState.getTableIdList().get(0); long loadRows = ((InsertTxnCommitAttachment) attachment).getLoadedRows(); if (loadRows == 0) { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); loadRows = table != null ? table.getRowCount() : 0; } updateBasicStatsMeta(db.getId(), tableId, loadRows); diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/BasicStatsMeta.java b/fe/fe-core/src/main/java/com/starrocks/statistic/BasicStatsMeta.java index b723f46973bfe..844c6f2438304 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/BasicStatsMeta.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/BasicStatsMeta.java @@ -123,8 +123,8 @@ public Map getProperties() { } public double getHealthy() { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); - OlapTable table = (OlapTable) database.getTable(tableId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); long totalPartitionCount = table.getPartitions().size(); long tableRowCount = 1L; diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeJob.java b/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeJob.java index 3eb615fc8afbd..81bc4f23e84f7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeJob.java @@ -105,7 +105,7 @@ public String getCatalogName() { @Override public String getDbName() throws MetaNotFoundException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("No found database: " + dbId); } @@ -114,11 +114,11 @@ public String getDbName() throws MetaNotFoundException { @Override public String getTableName() throws MetaNotFoundException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("No found database: " + dbId); } - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { throw new MetaNotFoundException("No found table: " + tableId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeStatus.java b/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeStatus.java index aef4182e1fccf..11035a63e1575 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeStatus.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/NativeAnalyzeStatus.java @@ -27,7 +27,6 @@ import com.starrocks.qe.ShowResultSet; import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.SemanticException; -import com.starrocks.sql.common.MetaUtils; import java.io.DataInput; import java.io.DataOutput; @@ -114,7 +113,7 @@ public String getCatalogName() { @Override public String getDbName() throws MetaNotFoundException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("No found database: " + dbId); } @@ -123,11 +122,11 @@ public String getDbName() throws MetaNotFoundException { @Override public String getTableName() throws MetaNotFoundException { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { throw new MetaNotFoundException("No found database: " + dbId); } - Table table = db.getTable(tableId); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(dbId, tableId); if (table == null) { throw new MetaNotFoundException("No found table: " + tableId); } @@ -205,7 +204,7 @@ public ShowResultSet toShowResult() { if (dbId == StatsConstants.DEFAULT_ALL_ID) { dbName = "*"; } else { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); dbName = db.getOriginName(); } String tableName; @@ -213,7 +212,11 @@ public ShowResultSet toShowResult() { tableName = "*"; } else { try { - tableName = MetaUtils.getTable(dbId, tableId).getName(); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(dbId, tableId); + if (table == null) { + throw new SemanticException("Table %s is not found", tableId); + } + tableName = table.getName(); } catch (SemanticException e) { tableName = ""; status = StatsConstants.ScheduleStatus.FAILED; diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java index 4066a9f8f3d4b..69a4f2329ed1e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java @@ -82,19 +82,19 @@ public List queryStatisticSync(ConnectContext context, Long dbId if (dbId == null) { List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long id : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(id); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(id); if (db == null) { continue; } - table = db.getTable(tableId); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { continue; } break; } } else { - Database database = GlobalStateMgr.getCurrentState().getDb(dbId); - table = database.getTable(tableId); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); } if (table == null) { @@ -187,8 +187,16 @@ public static Pair, Status> queryDictSync(Long dbId, Long t return Pair.create(Collections.emptyList(), Status.OK); } - Database db = MetaUtils.getDatabase(dbId); - Table table = MetaUtils.getTable(dbId, tableId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); + if (db == null) { + throw new SemanticException("Database %s is not found", dbId); + } + + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); + if (table == null) { + throw new SemanticException("Table %s is not found", tableId); + } + if (!(table.isOlapOrCloudNativeTable() || table.isMaterializedView())) { throw new SemanticException("Table '%s' is not a OLAP table or LAKE table or Materialize View", table.getName()); diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java index 63bebb94035fc..d58477592d2fc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java @@ -235,8 +235,8 @@ public static boolean statisticTableBlackListCheck(long tableId) { } for (String dbName : COLLECT_DATABASES_BLACKLIST) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - if (null != db && null != db.getTable(tableId)) { + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + if (null != db && null != GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId)) { return true; } } @@ -248,7 +248,7 @@ public static boolean checkStatisticTableStateNormal() { if (FeConstants.runningUnitTest) { return true; } - Database db = GlobalStateMgr.getCurrentState().getDb(StatsConstants.STATISTICS_DB_NAME); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(StatsConstants.STATISTICS_DB_NAME); List tableNameList = Lists.newArrayList(StatsConstants.SAMPLE_STATISTICS_TABLE_NAME, StatsConstants.FULL_STATISTICS_TABLE_NAME, StatsConstants.HISTOGRAM_STATISTICS_TABLE_NAME, StatsConstants.EXTERNAL_FULL_STATISTICS_TABLE_NAME); @@ -260,7 +260,7 @@ public static boolean checkStatisticTableStateNormal() { for (String tableName : tableNameList) { // check table - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { return false; } diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java index 1ebeb99005a0f..43f6bb5b19351 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java @@ -59,33 +59,34 @@ public static List buildStatisticsCollectJob(NativeAnalyze List dbIds = GlobalStateMgr.getCurrentState().getLocalMetastore().getDbIds(); for (Long dbId : dbIds) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (null == db || StatisticUtils.statisticDatabaseBlackListCheck(db.getFullName())) { continue; } - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { createJob(statsJobs, nativeAnalyzeJob, db, table, null, null); } } } else if (StatsConstants.DEFAULT_ALL_ID == nativeAnalyzeJob.getTableId() && StatsConstants.DEFAULT_ALL_ID != nativeAnalyzeJob.getDbId()) { // all table - Database db = GlobalStateMgr.getCurrentState().getDb(nativeAnalyzeJob.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(nativeAnalyzeJob.getDbId()); if (null == db) { return Collections.emptyList(); } - for (Table table : db.getTables()) { + for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId())) { createJob(statsJobs, nativeAnalyzeJob, db, table, null, null); } } else { // database or table is null mean database/table has been dropped - Database db = GlobalStateMgr.getCurrentState().getDb(nativeAnalyzeJob.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(nativeAnalyzeJob.getDbId()); if (db == null) { return Collections.emptyList(); } - createJob(statsJobs, nativeAnalyzeJob, db, db.getTable(nativeAnalyzeJob.getTableId()), + createJob(statsJobs, nativeAnalyzeJob, db, GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), nativeAnalyzeJob.getTableId()), nativeAnalyzeJob.getColumns(), nativeAnalyzeJob.getColumnTypes()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java index 9a4e7bd10f6ef..ca983ef3c1620 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsMetaManager.java @@ -61,7 +61,7 @@ public StatisticsMetaManager() { } private boolean checkDatabaseExist() { - return GlobalStateMgr.getCurrentState().getDb(StatsConstants.STATISTICS_DB_NAME) != null; + return GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(StatsConstants.STATISTICS_DB_NAME) != null; } private boolean createDatabase() { @@ -78,9 +78,9 @@ private boolean createDatabase() { } private boolean checkTableExist(String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(StatsConstants.STATISTICS_DB_NAME); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(StatsConstants.STATISTICS_DB_NAME); Preconditions.checkState(db != null); - return db.getTable(tableName) != null; + return GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName) != null; } private boolean checkReplicateNormal(String tableName) { @@ -92,9 +92,9 @@ private boolean checkReplicateNormal(String tableName) { return true; } - Database db = GlobalStateMgr.getCurrentState().getDb(StatsConstants.STATISTICS_DB_NAME); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(StatsConstants.STATISTICS_DB_NAME); Preconditions.checkState(db != null); - OlapTable table = (OlapTable) db.getTable(tableName); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); Preconditions.checkState(table != null); if (table.isCloudNativeTableOrMaterializedView()) { return true; diff --git a/fe/fe-core/src/main/java/com/starrocks/system/SystemInfoService.java b/fe/fe-core/src/main/java/com/starrocks/system/SystemInfoService.java index 4173102cf8504..b619b4f2f92ad 100644 --- a/fe/fe-core/src/main/java/com/starrocks/system/SystemInfoService.java +++ b/fe/fe-core/src/main/java/com/starrocks/system/SystemInfoService.java @@ -421,11 +421,11 @@ protected void checkWhenNotForceDrop(Backend droppedBackend) { GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletIdsByBackendId(droppedBackend.getId()); List dbs = globalStateMgr.getLocalMetastore().getDbIds(); - dbs.stream().map(globalStateMgr::getDb).forEach(db -> { + dbs.stream().map(dbId -> globalStateMgr.getLocalMetastore().getDb(dbId)).forEach(db -> { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - db.getTables().stream() + GlobalStateMgr.getCurrentState().getLocalMetastore().getTables(db.getId()).stream() .filter(Table::isOlapTableOrMaterializedView) .map(table -> (OlapTable) table) .filter(table -> table.getTableProperty().getReplicationNum() == 1) @@ -953,7 +953,7 @@ public void updateBackendReportVersion(long backendId, long newReportVersion, lo if (node instanceof Backend) { AtomicLong atomicLong; if ((atomicLong = idToReportVersionRef.get(backendId)) != null) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db != null) { updateReportVersionIncrementally(atomicLong, newReportVersion); LOG.debug("update backend {} report version: {}, db: {}", backendId, newReportVersion, dbId); diff --git a/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java b/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java index 492cd560eb7ae..22d6e3ef77122 100644 --- a/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java @@ -301,12 +301,12 @@ public TAlterTabletReqV2 toThrift() { * And because alter request report success, it means that we can increase replica's version to X. */ public void handleFinishAlterTask() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(getDbId()); if (db == null) { throw new MetaNotFoundException("database " + getDbId() + " does not exist"); } - OlapTable tbl = (OlapTable) db.getTable(getTableId()); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), getTableId()); if (tbl == null) { throw new MetaNotFoundException("tbl " + getTableId() + " does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/task/ExportPendingTask.java b/fe/fe-core/src/main/java/com/starrocks/task/ExportPendingTask.java index 58738543f73c4..0a1122f75eba5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/task/ExportPendingTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/task/ExportPendingTask.java @@ -80,7 +80,7 @@ protected void exec() { } long dbId = job.getDbId(); - db = GlobalStateMgr.getCurrentState().getDb(dbId); + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { job.cancelInternal(ExportFailMsg.CancelType.RUN_FAIL, "database does not exist"); return; diff --git a/fe/fe-core/src/main/java/com/starrocks/task/PublishVersionTask.java b/fe/fe-core/src/main/java/com/starrocks/task/PublishVersionTask.java index 9a306ace4b999..25f455c27ad22 100644 --- a/fe/fe-core/src/main/java/com/starrocks/task/PublishVersionTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/task/PublishVersionTask.java @@ -189,7 +189,7 @@ public void updateReplicaVersions(List tabletVersions) { LOG.warn("backend not found or no replicas on backend, backendid={}", backendId); return; } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.warn("db not found dbid={}", dbId); return; diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java index de568f0864af0..95fd4f3bee873 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java @@ -269,7 +269,7 @@ public void prepareTransaction(long transactionId, List tablet Preconditions.checkNotNull(tabletFailInfos, "tabletFailInfos is null"); // 1. check status // the caller method already own db lock, we do not obtain db lock here - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (null == db) { throw new MetaNotFoundException("could not find db [" + dbId + "]"); } @@ -389,7 +389,7 @@ public void prepareTransaction(long transactionId, List tablet public VisibleStateWaiter commitPreparedTransaction(long transactionId) throws UserException { // 1. check status // the caller method already own db lock, we do not obtain db lock here - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (null == db) { throw new MetaNotFoundException("could not find db [" + dbId + "]"); } @@ -419,7 +419,7 @@ public VisibleStateWaiter commitPreparedTransaction(long transactionId) throws U txnSpan.addEvent("commit_start"); for (Long tableId : transactionState.getTableIdList()) { - Table table = db.getTable(tableId); + Table table = globalStateMgr.getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { // this can happen when tableId == -1 (tablet being dropping) // or table really not exist. @@ -551,12 +551,12 @@ public void abortTransaction(long transactionId, boolean abortPrepared, String r LOG.info("transaction:[{}] successfully rollback", transactionState); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { return; } for (Long tableId : transactionState.getTableIdList()) { - Table table = db.getTable(tableId); + Table table = globalStateMgr.getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { continue; } @@ -875,7 +875,7 @@ public List getReadyToPublishTxnListBatch() { // for each tablet of load txn, if most replicas version publish successed // the trasaction can be treated as successful and can be finished public boolean canTxnFinished(TransactionState txn, Set errReplicas, Set unfinishedBackends) { - Database db = globalStateMgr.getDb(txn.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(txn.getDbId()); if (db == null) { return true; } @@ -888,7 +888,7 @@ public boolean canTxnFinished(TransactionState txn, Set errReplicas, Set errorReplicaIds) thr errorReplicaIds.addAll(originalErrorReplicas); } - Database db = globalStateMgr.getDb(transactionState.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(transactionState.getDbId()); if (db == null) { transactionState.writeLock(); try { @@ -1018,7 +1018,8 @@ public void finishTransaction(long transactionId, Set errorReplicaIds) thr Set droppedTableIds = Sets.newHashSet(); for (TableCommitInfo tableCommitInfo : transactionState.getIdToTableCommitInfos().values()) { long tableId = tableCommitInfo.getTableId(); - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableId); // table maybe dropped between commit and publish, ignore this error if (table == null) { droppedTableIds.add(tableId); @@ -1230,7 +1231,7 @@ protected void unprotectedCommitPreparedTransaction(TransactionState transaction while (tableCommitInfoIterator.hasNext()) { TableCommitInfo tableCommitInfo = tableCommitInfoIterator.next(); long tableId = tableCommitInfo.getTableId(); - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) globalStateMgr.getLocalMetastore().getTable(db.getId(), tableId); // table maybe dropped between commit and publish, ignore this error if (table == null) { transactionState.removeTable(tableId); @@ -1614,7 +1615,7 @@ public List> getSingleTranInfo(long dbId, long txnId) throws Analys List> infos = new ArrayList>(); readLock(); try { - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { throw new AnalysisException("Database[" + dbId + "] does not exist"); } @@ -1657,7 +1658,7 @@ protected void checkRunningTxnExceedLimit(TransactionState.LoadJobSourceType sou private void updateCatalogAfterCommitted(TransactionState transactionState, Database db) { for (TableCommitInfo tableCommitInfo : transactionState.getIdToTableCommitInfos().values()) { long tableId = tableCommitInfo.getTableId(); - Table table = db.getTable(tableId); + Table table = globalStateMgr.getLocalMetastore().getTable(db.getId(), tableId); TransactionLogApplier applier = txnLogApplierFactory.create(table); applier.applyCommitLog(transactionState, tableCommitInfo); } @@ -1665,7 +1666,7 @@ private void updateCatalogAfterCommitted(TransactionState transactionState, Data private boolean updateCatalogAfterVisible(TransactionState transactionState, Database db) { for (TableCommitInfo tableCommitInfo : transactionState.getIdToTableCommitInfos().values()) { - Table table = db.getTable(tableCommitInfo.getTableId()); + Table table = globalStateMgr.getLocalMetastore().getTable(db.getId(), tableCommitInfo.getTableId()); // table may be dropped by force after transaction committed // so that it will be a visible edit log after drop table if (table == null) { @@ -1684,7 +1685,8 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat // the write lock of database has been hold private boolean updateCatalogAfterVisibleBatch(TransactionStateBatch transactionStateBatch, Database db) { - Table table = db.getTable(transactionStateBatch.getTableId()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), transactionStateBatch.getTableId()); if (table == null) { return true; } @@ -1771,7 +1773,7 @@ public void replayUpsertTransactionState(TransactionState transactionState) { } // set transaction status will call txn state change listener transactionState.replaySetTransactionStatus(); - Database db = globalStateMgr.getDb(transactionState.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(transactionState.getDbId()); if (transactionState.getTransactionStatus() == TransactionStatus.COMMITTED) { LOG.info("replay a committed transaction {}", transactionState); updateCatalogAfterCommitted(transactionState, db); @@ -1793,7 +1795,7 @@ public void replayUpsertTransactionStateBatch(TransactionStateBatch transactionS writeLock(); try { LOG.info("replay a transaction state batch{}", transactionStateBatch); - Database db = globalStateMgr.getDb(transactionStateBatch.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(transactionStateBatch.getDbId()); updateCatalogAfterVisibleBatch(transactionStateBatch, db); unprotectSetTransactionStateBatch(transactionStateBatch, true); @@ -1832,7 +1834,7 @@ GlobalStateMgr getGlobalStateMgr() { } public void finishTransactionNew(TransactionState transactionState, Set publishErrorReplicas) throws UserException { - Database db = globalStateMgr.getDb(transactionState.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(transactionState.getDbId()); if (db == null) { transactionState.writeLock(); try { @@ -1901,7 +1903,7 @@ public void finishTransactionNew(TransactionState transactionState, Set pu } public void finishTransactionBatch(TransactionStateBatch stateBatch, Set errorReplicaIds) { - Database db = globalStateMgr.getDb(stateBatch.getDbId()); + Database db = globalStateMgr.getLocalMetastore().getDb(stateBatch.getDbId()); if (db == null) { stateBatch.writeLock(); try { @@ -1983,7 +1985,7 @@ private List populateTransactionStateListeners(@NotNul throws TransactionException { List stateListeners = Lists.newArrayList(); for (Long tableId : transactionState.getTableIdList()) { - Table table = database.getTable(tableId); + Table table = globalStateMgr.getLocalMetastore().getTable(database.getId(), tableId); if (table == null) { // this can happen when tableId == -1 (tablet being dropping) // or table really not exist. @@ -2013,7 +2015,7 @@ public TTransactionStatus getTxnStatus(long txnId) { } private void checkDatabaseDataQuota() throws AnalysisException { - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { throw new AnalysisException("Database[" + dbId + "] does not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java b/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java index 30a42c4c9f282..f66b2a01e1231 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java @@ -283,7 +283,7 @@ public void prepareTransaction(long dbId, long transactionId, List> getDbInfo() { for (long dbId : dbIds) { List info = new ArrayList(); info.add(dbId); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnLogApplier.java b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnLogApplier.java index fd44dddf30b5e..a2ee47df23366 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnLogApplier.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnLogApplier.java @@ -90,7 +90,7 @@ public void applyCommitLog(TransactionState txnState, TableCommitInfo commitInfo public void applyVisibleLog(TransactionState txnState, TableCommitInfo commitInfo, Database db) { Set errorReplicaIds = txnState.getErrorReplicas(); long tableId = table.getId(); - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { LOG.warn("table {} is dropped, ignore", tableId); return; diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java index 4822927400ccc..a840c51c6a068 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java @@ -292,7 +292,7 @@ public void preWriteCommitLog(TransactionState txnState) { public void postAbort(TransactionState txnState, List finishedTablets, List failedTablets) { txnState.clearAutomaticPartitionSnapshot(); - Database db = GlobalStateMgr.getCurrentState().getDb(txnState.getDbId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(txnState.getDbId()); if (db != null) { Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, txnState.getTableIdList(), LockType.READ); diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/PublishVersionDaemon.java b/fe/fe-core/src/main/java/com/starrocks/transaction/PublishVersionDaemon.java index 4594c9e8d9c5a..54b971dacf777 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/PublishVersionDaemon.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/PublishVersionDaemon.java @@ -427,7 +427,7 @@ private CompletableFuture publishLakeTransactionAsync(TransactionState txn GlobalTransactionMgr globalTransactionMgr = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr(); long txnId = txnState.getTransactionId(); long dbId = txnState.getDbId(); - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.info("the database of transaction {} has been deleted", txnId); try { @@ -482,7 +482,7 @@ public boolean publishPartitionBatch(Database db, long tableId, PartitionPublish // version -> shadowTablets long warehouseId = WarehouseManager.DEFAULT_WAREHOUSE_ID; try { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { // table has been dropped return true; @@ -655,7 +655,7 @@ private CompletableFuture publishLakeTransactionBatchAsync(TransactionStat } } - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { LOG.info("the database of transaction batch {} has been deleted", txnStateBatch); @@ -768,7 +768,7 @@ private boolean publishPartition(@NotNull Database db, @NotNull TableCommitInfo Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(tableId), LockType.READ); try { - OlapTable table = (OlapTable) db.getTable(tableId); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); if (table == null) { txnState.removeTable(tableCommitInfo.getTableId()); LOG.info("Removed non-exist table {} from transaction {}. txn_id={}", tableId, txnLabel, txnId); diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionChecker.java b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionChecker.java index 5a6a2b39383de..ac5758eef1bc5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionChecker.java @@ -23,6 +23,7 @@ import com.starrocks.catalog.Tablet; import com.starrocks.common.util.concurrent.lock.LockType; import com.starrocks.common.util.concurrent.lock.Locker; +import com.starrocks.server.GlobalStateMgr; import java.util.ArrayList; import java.util.List; @@ -97,7 +98,8 @@ void debugInfo(StringBuilder sb) { public static TransactionChecker create(TransactionState txn, Database db) { List partitions = new ArrayList<>(); for (TableCommitInfo tableCommitInfo : txn.getIdToTableCommitInfos().values()) { - OlapTable table = (OlapTable) db.getTable(tableCommitInfo.getTableId()); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tableCommitInfo.getTableId()); if (table == null || table.isCloudNativeTableOrMaterializedView()) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java index 43f5f7974cde6..0d9348b5362c1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java @@ -918,7 +918,7 @@ public boolean allPublishTasksFinishedOrQuorumWaitTimeout(Set publishError public boolean checkCanFinish() { // finishChecker may require refresh if table/partition is dropped, or index is changed caused by Alter job - Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); if (db == null) { // consider txn finished if db is dropped return true; diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/UpdateDbUsedDataQuotaDaemon.java b/fe/fe-core/src/main/java/com/starrocks/transaction/UpdateDbUsedDataQuotaDaemon.java index 340852bdc3696..43fdfc3de90b1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/UpdateDbUsedDataQuotaDaemon.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/UpdateDbUsedDataQuotaDaemon.java @@ -61,7 +61,7 @@ private void updateAllDatabaseUsedDataQuota() { List dbIdList = globalStateMgr.getLocalMetastore().getDbIds(); GlobalTransactionMgr globalTransactionMgr = globalStateMgr.getGlobalTransactionMgr(); for (Long dbId : dbIdList) { - Database db = globalStateMgr.getDb(dbId); + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); if (db == null) { LOG.warn("Database [" + dbId + "] doese not exist, skip to update database used data quota"); continue; diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/AlterJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/AlterJobV2Test.java index 2172b981049cb..3f896604efd7d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/AlterJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/AlterJobV2Test.java @@ -79,14 +79,14 @@ public static void beforeClass() throws Exception { UtFrameUtils.setDefaultConfigForAsyncMVTest(connectContext); starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.schema_change_test(k1 int, k2 int, k3 int) " + - "distributed by hash(k1) buckets 3 properties('replication_num' = '1');") - .withTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) " + - "distributed by hash(k1) buckets 3 properties('replication_num' = '1');") - .withTable("CREATE TABLE test.properties_change_test(k1 int, v1 int) " + - "primary key(k1) distributed by hash(k1) properties('replication_num' = '1');") - .withTable("CREATE TABLE modify_column_test(k1 int, k2 int, k3 int) ENGINE = OLAP " + - "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); + .withTable("CREATE TABLE test.schema_change_test(k1 int, k2 int, k3 int) " + + "distributed by hash(k1) buckets 3 properties('replication_num' = '1');") + .withTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) " + + "distributed by hash(k1) buckets 3 properties('replication_num' = '1');") + .withTable("CREATE TABLE test.properties_change_test(k1 int, v1 int) " + + "primary key(k1) distributed by hash(k1) properties('replication_num' = '1');") + .withTable("CREATE TABLE modify_column_test(k1 int, k2 int, k3 int) ENGINE = OLAP " + + "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); } @AfterClass @@ -116,7 +116,7 @@ public void testSchemaChange() throws Exception { // 3. check show alter table column String showAlterStmtStr = "show alter table column from test;"; ShowAlterStmt showAlterStmt = - (ShowAlterStmt) UtFrameUtils.parseStmtWithNewParser(showAlterStmtStr, connectContext); + (ShowAlterStmt) UtFrameUtils.parseStmtWithNewParser(showAlterStmtStr, connectContext); ShowResultSet showResultSet = ShowExecutor.execute(showAlterStmt, connectContext); System.out.println(showResultSet.getMetaData()); System.out.println(showResultSet.getResultRows()); @@ -133,7 +133,7 @@ public void testRollup() throws Exception { for (AlterJobV2 alterJobV2 : alterJobs.values()) { while (!alterJobV2.getJobState().isFinalState()) { System.out.println( - "alter job " + alterJobV2.getJobId() + " is running. state: " + alterJobV2.getJobState()); + "alter job " + alterJobV2.getJobId() + " is running. state: " + alterJobV2.getJobState()); Thread.sleep(1000); } System.out.println("alter job " + alterJobV2.getJobId() + " is done. state: " + alterJobV2.getJobState()); @@ -142,7 +142,7 @@ public void testRollup() throws Exception { // 3. check show alter table column String showAlterStmtStr = "show alter table rollup from test;"; ShowAlterStmt showAlterStmt = - (ShowAlterStmt) UtFrameUtils.parseStmtWithNewParser(showAlterStmtStr, connectContext); + (ShowAlterStmt) UtFrameUtils.parseStmtWithNewParser(showAlterStmtStr, connectContext); ShowResultSet showResultSet = ShowExecutor.execute(showAlterStmt, connectContext); System.out.println(showResultSet.getMetaData()); System.out.println(showResultSet.getResultRows()); @@ -160,7 +160,7 @@ public void testModifyTableProperties() throws Exception { // 3. check enable persistent index String showCreateTableStr = "show create table test.properties_change_test;"; ShowCreateTableStmt showCreateTableStmt = - (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateTableStr, connectContext); + (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateTableStr, connectContext); ShowResultSet showResultSet = ShowExecutor.execute(showCreateTableStmt, connectContext); System.out.println(showResultSet.getMetaData()); System.out.println(showResultSet.getResultRows()); @@ -175,7 +175,7 @@ public void testModifyTableProperties() throws Exception { // 5. check enable persistent index showCreateTableStmt = - (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateTableStr, connectContext); + (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateTableStr, connectContext); showResultSet = ShowExecutor.execute(showCreateTableStmt, connectContext); System.out.println(showResultSet.getMetaData()); System.out.println(showResultSet.getResultRows()); @@ -185,10 +185,10 @@ public void testModifyTableProperties() throws Exception { public void testModifyRelatedColumnWithMv() { try { String sql = "CREATE MATERIALIZED VIEW test.mv2 DISTRIBUTED BY HASH(k1) " + - " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') AS SELECT k1, k2 FROM modify_column_test"; + " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') AS SELECT k1, k2 FROM modify_column_test"; StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore() - .createMaterializedView((CreateMaterializedViewStatement) statementBase); + .createMaterializedView((CreateMaterializedViewStatement) statementBase); // modify column which define in mv String alterStmtStr = "alter table test.modify_column_test modify column k2 varchar(10)"; @@ -196,7 +196,8 @@ public void testModifyRelatedColumnWithMv() { DDLStmtExecutor.execute(alterTableStmt, connectContext); waitForSchemaChangeAlterJobFinish(); - MaterializedView mv2 = (MaterializedView) GlobalStateMgr.getCurrentState().getDb("test").getTable("mv2"); + MaterializedView mv2 = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("mv2"); Assert.assertFalse(mv2.isActive()); } catch (Exception e) { e.printStackTrace(); @@ -209,21 +210,21 @@ public void testModifyRelatedColumnWithMv() { public void testModifyWithSelectStarMV1() throws Exception { try { starRocksAssert.withTable("CREATE TABLE modify_column_test3(k1 int, k2 int, k3 int) ENGINE = OLAP " + - "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); + "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); String sql = "CREATE MATERIALIZED VIEW test.mv3 DISTRIBUTED BY HASH(k1) " + - " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') " + - "AS SELECT * FROM modify_column_test3"; + " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') " + + "AS SELECT * FROM modify_column_test3"; StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore() - .createMaterializedView((CreateMaterializedViewStatement) statementBase); + .createMaterializedView((CreateMaterializedViewStatement) statementBase); String alterStmtStr = "alter table test.modify_column_test3 modify column k2 varchar(20)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterStmtStr, connectContext); DDLStmtExecutor.execute(alterTableStmt, connectContext); waitForSchemaChangeAlterJobFinish(); - MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState() - .getDb("test").getTable("mv3"); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getDb("test").getTable("mv3"); Assert.assertTrue(!mv.isActive()); } finally { starRocksAssert.dropTable("modify_column_test3"); @@ -235,13 +236,13 @@ public void testModifyWithSelectStarMV1() throws Exception { public void testModifyWithSelectStarMV2() throws Exception { try { starRocksAssert.withTable("CREATE TABLE testModifyWithSelectStarMV2(k1 int, k2 int, k3 int) ENGINE = OLAP " + - "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); + "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); String sql = "CREATE MATERIALIZED VIEW test.mv6 DISTRIBUTED BY HASH(k1) " + - " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') " + - "AS SELECT * FROM testModifyWithSelectStarMV2"; + " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') " + + "AS SELECT * FROM testModifyWithSelectStarMV2"; StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore() - .createMaterializedView((CreateMaterializedViewStatement) statementBase); + .createMaterializedView((CreateMaterializedViewStatement) statementBase); String alterStmtStr = "alter table test.testModifyWithSelectStarMV2 add column k4 bigint"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterStmtStr, connectContext); @@ -249,7 +250,7 @@ public void testModifyWithSelectStarMV2() throws Exception { waitForSchemaChangeAlterJobFinish(); MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState() - .getDb("test").getTable("mv6"); + .getLocalMetastore().getDb("test").getTable("mv6"); Assert.assertTrue(mv.isActive()); } catch (Exception e) { e.printStackTrace(); @@ -264,13 +265,13 @@ public void testModifyWithSelectStarMV2() throws Exception { public void testModifyWithSelectStarMV3() throws Exception { try { starRocksAssert.withTable("CREATE TABLE modify_column_test5(k1 int, k2 int, k3 int) ENGINE = OLAP " + - "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); + "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); String sql = "CREATE MATERIALIZED VIEW test.mv5 DISTRIBUTED BY HASH(k1) " + - " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') " + - "AS SELECT * FROM modify_column_test5"; + " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') " + + "AS SELECT * FROM modify_column_test5"; StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore() - .createMaterializedView((CreateMaterializedViewStatement) statementBase); + .createMaterializedView((CreateMaterializedViewStatement) statementBase); String alterStmtStr = "alter table test.modify_column_test5 drop column k2"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterStmtStr, connectContext); @@ -278,7 +279,7 @@ public void testModifyWithSelectStarMV3() throws Exception { waitForSchemaChangeAlterJobFinish(); MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState() - .getDb("test").getTable("mv5"); + .getLocalMetastore().getDb("test").getTable("mv5"); Assert.assertTrue(!mv.isActive()); } catch (Exception e) { Assert.fail(); @@ -291,23 +292,24 @@ public void testModifyWithSelectStarMV3() throws Exception { public void testModifyWithExpr() throws Exception { try { starRocksAssert.withTable("CREATE TABLE modify_column_test4(k1 int, k2 int, k3 int) ENGINE = OLAP " + - "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); + "DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) properties('replication_num' = '1');"); String sql = "CREATE MATERIALIZED VIEW test.mv4 DISTRIBUTED BY HASH(k1) " + - " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') AS SELECT k1, k2 + 1 FROM modify_column_test4"; + " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1')" + + " AS SELECT k1, k2 + 1 FROM modify_column_test4"; StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore() - .createMaterializedView((CreateMaterializedViewStatement) statementBase); + .createMaterializedView((CreateMaterializedViewStatement) statementBase); { // modify column which not define in mv String alterStmtStr = "alter table test.modify_column_test4 modify column k3 varchar(10)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterStmtStr, - connectContext); + connectContext); DDLStmtExecutor.execute(alterTableStmt, connectContext); waitForSchemaChangeAlterJobFinish(); MaterializedView mv = (MaterializedView) GlobalStateMgr - .getCurrentState().getDb("test").getTable("mv4"); + .getCurrentState().getLocalMetastore().getDb("test").getTable("mv4"); Assert.assertTrue(mv.isActive()); } @@ -315,12 +317,12 @@ public void testModifyWithExpr() throws Exception { // modify column which define in mv String alterStmtStr = "alter table test.modify_column_test4 modify column k2 varchar(30) "; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterStmtStr, - connectContext); + connectContext); DDLStmtExecutor.execute(alterTableStmt, connectContext); waitForSchemaChangeAlterJobFinish(); MaterializedView mv = (MaterializedView) GlobalStateMgr - .getCurrentState().getDb("test").getTable("mv4"); + .getCurrentState().getLocalMetastore().getDb("test").getTable("mv4"); Assert.assertFalse(mv.isActive()); System.out.println(mv.getInactiveReason()); Assert.assertTrue(mv.getInactiveReason().contains("base table schema changed for columns: k2")); @@ -337,10 +339,10 @@ public void testModifyWithExpr() throws Exception { public void testModifyUnRelatedColumnWithMv() { try { String sql = "CREATE MATERIALIZED VIEW test.mv1 DISTRIBUTED BY HASH(k1) " + - " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') AS SELECT k1, k2 FROM modify_column_test"; + " BUCKETS 10 REFRESH ASYNC properties('replication_num' = '1') AS SELECT k1, k2 FROM modify_column_test"; StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore() - .createMaterializedView((CreateMaterializedViewStatement) statementBase); + .createMaterializedView((CreateMaterializedViewStatement) statementBase); // modify column which not define in mv String alterStmtStr = "alter table test.modify_column_test modify column k3 varchar(10)"; @@ -348,7 +350,8 @@ public void testModifyUnRelatedColumnWithMv() { DDLStmtExecutor.execute(alterTableStmt, connectContext); waitForSchemaChangeAlterJobFinish(); - MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getDb("test").getTable("mv1"); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("mv1"); Assert.assertTrue(mv.isActive()); } catch (Exception e) { e.printStackTrace(); @@ -359,7 +362,7 @@ public void testModifyUnRelatedColumnWithMv() { @Test public void test() throws Exception { starRocksAssert.withTable("CREATE TABLE test.schema_change_test_load(k1 int, k2 int, k3 int) " + - "distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); + "distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); String alterStmtStr = "alter table test.schema_change_test_load add column k4 int default '1'"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterStmtStr, connectContext); @@ -377,7 +380,7 @@ public void test() throws Exception { srMetaBlockReader.close(); AlterJobV2 alterJobV2New = - GlobalStateMgr.getCurrentState().getSchemaChangeHandler().getAlterJobsV2().get(alterJobV2Old.jobId); + GlobalStateMgr.getCurrentState().getSchemaChangeHandler().getAlterJobsV2().get(alterJobV2Old.jobId); Assert.assertEquals(alterJobV2Old.jobId, alterJobV2New.jobId); } diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTableTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTableTest.java index 2f91d20bea36d..b481dc1bd495c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTableTest.java @@ -71,25 +71,25 @@ public static void beforeClass() throws Exception { @Test(expected = AnalysisException.class) public void testAlterTableBucketSize() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_alter_bucket_size (\n" + - "event_day DATE,\n" + - "site_id INT DEFAULT '10',\n" + - "city_code VARCHAR(100),\n" + - "user_name VARCHAR(32) DEFAULT '',\n" + - "pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY RANGE(event_day)(\n" + - "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + - "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + - "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + - "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + - ")\n" + - "DISTRIBUTED BY RANDOM\n" + - "PROPERTIES(\n" + - "\t\"replication_num\" = \"1\",\n" + - " \"storage_medium\" = \"SSD\",\n" + - " \"storage_cooldown_ttl\" = \"1 day\"\n" + - ");"); + "event_day DATE,\n" + + "site_id INT DEFAULT '10',\n" + + "city_code VARCHAR(100),\n" + + "user_name VARCHAR(32) DEFAULT '',\n" + + "pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY RANGE(event_day)(\n" + + "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + + "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + + "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + + "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n" + + "PROPERTIES(\n" + + "\t\"replication_num\" = \"1\",\n" + + " \"storage_medium\" = \"SSD\",\n" + + " \"storage_cooldown_ttl\" = \"1 day\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE test_alter_bucket_size SET (\"bucket_size\" = \"-1\");"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); @@ -99,31 +99,31 @@ public void testAlterTableBucketSize() throws Exception { @Test public void testAlterTableStorageCoolDownTTL() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_alter_cool_down_ttl (\n" + - "event_day DATE,\n" + - "site_id INT DEFAULT '10',\n" + - "city_code VARCHAR(100),\n" + - "user_name VARCHAR(32) DEFAULT '',\n" + - "pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY RANGE(event_day)(\n" + - "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + - "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + - "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + - "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - "\t\"replication_num\" = \"1\",\n" + - " \"storage_medium\" = \"SSD\",\n" + - " \"storage_cooldown_ttl\" = \"1 day\"\n" + - ");"); + "event_day DATE,\n" + + "site_id INT DEFAULT '10',\n" + + "city_code VARCHAR(100),\n" + + "user_name VARCHAR(32) DEFAULT '',\n" + + "pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY RANGE(event_day)(\n" + + "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + + "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + + "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + + "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + "\t\"replication_num\" = \"1\",\n" + + " \"storage_medium\" = \"SSD\",\n" + + " \"storage_cooldown_ttl\" = \"1 day\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE test_alter_cool_down_ttl SET (\"storage_cooldown_ttl\" = \"3 day\");"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterTable(ctx, alterTableStmt); - Table table = GlobalStateMgr.getCurrentState().getDb("test").getTable("test_alter_cool_down_ttl"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("test_alter_cool_down_ttl"); OlapTable olapTable = (OlapTable) table; String storageCooldownTtl = olapTable.getTableProperty().getProperties().get("storage_cooldown_ttl"); Assert.assertEquals("3 day", storageCooldownTtl); @@ -134,25 +134,25 @@ public void testAlterTableStorageCoolDownTTL() throws Exception { @Test(expected = AnalysisException.class) public void testAlterTableStorageCoolDownTTLExcept() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_alter_cool_down_ttl_2 (\n" + - "event_day DATE,\n" + - "site_id INT DEFAULT '10',\n" + - "city_code VARCHAR(100),\n" + - "user_name VARCHAR(32) DEFAULT '',\n" + - "pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY RANGE(event_day)(\n" + - "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + - "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + - "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + - "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - "\t\"replication_num\" = \"1\",\n" + - " \"storage_medium\" = \"SSD\",\n" + - " \"storage_cooldown_ttl\" = \"1 day\"\n" + - ");"); + "event_day DATE,\n" + + "site_id INT DEFAULT '10',\n" + + "city_code VARCHAR(100),\n" + + "user_name VARCHAR(32) DEFAULT '',\n" + + "pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY RANGE(event_day)(\n" + + "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + + "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + + "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + + "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + "\t\"replication_num\" = \"1\",\n" + + " \"storage_medium\" = \"SSD\",\n" + + " \"storage_cooldown_ttl\" = \"1 day\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE test_alter_cool_down_ttl_2 SET (\"storage_cooldown_ttl\" = \"abc\");"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); @@ -162,27 +162,28 @@ public void testAlterTableStorageCoolDownTTLExcept() throws Exception { @Test public void testAlterTableStorageCoolDownTTLPartition() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_alter_cool_down_ttl_partition (\n" + - "event_day DATE,\n" + - "site_id INT DEFAULT '10',\n" + - "city_code VARCHAR(100),\n" + - "user_name VARCHAR(32) DEFAULT '',\n" + - "pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY RANGE(event_day)(\n" + - "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + - "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + - "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + - "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - "\t\"replication_num\" = \"1\",\n" + - " \"storage_medium\" = \"SSD\",\n" + - " \"storage_cooldown_ttl\" = \"1 day\"\n" + - ");"); + "event_day DATE,\n" + + "site_id INT DEFAULT '10',\n" + + "city_code VARCHAR(100),\n" + + "user_name VARCHAR(32) DEFAULT '',\n" + + "pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY RANGE(event_day)(\n" + + "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + + "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + + "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + + "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + "\t\"replication_num\" = \"1\",\n" + + " \"storage_medium\" = \"SSD\",\n" + + " \"storage_cooldown_ttl\" = \"1 day\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); - Table table = GlobalStateMgr.getCurrentState().getDb("test").getTable("test_alter_cool_down_ttl_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_alter_cool_down_ttl_partition"); OlapTable olapTable = (OlapTable) table; RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); DataProperty p20200321 = rangePartitionInfo.getDataProperty(olapTable.getPartition("p20200321").getId()); @@ -195,7 +196,7 @@ public void testAlterTableStorageCoolDownTTLPartition() throws Exception { Assert.assertEquals("9999-12-31 23:59:59", TimeUtils.longToTimeString(p20200324.getCooldownTimeMs())); String sql = "ALTER TABLE test_alter_cool_down_ttl_partition\n" + - "MODIFY PARTITION (*) SET(\"storage_cooldown_ttl\" = \"2 day\", \"storage_medium\" = \"SSD\");"; + "MODIFY PARTITION (*) SET(\"storage_cooldown_ttl\" = \"2 day\", \"storage_medium\" = \"SSD\");"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); AlterJobExecutor alterJobExecutor = new AlterJobExecutor(); alterJobExecutor.process(alterTableStmt, ctx); @@ -214,33 +215,34 @@ public void testAlterTableStorageCoolDownTTLPartition() throws Exception { @Test public void testAlterTablePartitionTTLInvalid() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_partition_live_number (\n" + - "event_day DATE,\n" + - "site_id INT DEFAULT '10',\n" + - "city_code VARCHAR(100),\n" + - "user_name VARCHAR(32) DEFAULT '',\n" + - "pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY RANGE(event_day)(\n" + - "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + - "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + - "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + - "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - "\t\"replication_num\" = \"1\",\n" + - " \"storage_medium\" = \"SSD\",\n" + - " \"partition_live_number\" = \"2\"\n" + - ");"); + "event_day DATE,\n" + + "site_id INT DEFAULT '10',\n" + + "city_code VARCHAR(100),\n" + + "user_name VARCHAR(32) DEFAULT '',\n" + + "pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY RANGE(event_day)(\n" + + "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + + "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + + "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + + "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + "\t\"replication_num\" = \"1\",\n" + + " \"storage_medium\" = \"SSD\",\n" + + " \"partition_live_number\" = \"2\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE test_partition_live_number SET(\"partition_live_number\" = \"-1\");"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterTable(ctx, alterTableStmt); Set> ttlPartitionInfo = GlobalStateMgr.getCurrentState() - .getDynamicPartitionScheduler().getTtlPartitionInfo(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("test_partition_live_number"); + .getDynamicPartitionScheduler().getTtlPartitionInfo(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition_live_number"); Assert.assertFalse(ttlPartitionInfo.contains(new Pair<>(db.getId(), table.getId()))); sql = "ALTER TABLE test_partition_live_number SET(\"partition_live_number\" = \"1\");"; alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); @@ -251,65 +253,67 @@ public void testAlterTablePartitionTTLInvalid() throws Exception { @Test public void testAlterTablePartitionStorageMedium() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_partition_storage_medium (\n" + - "event_day DATE,\n" + - "site_id INT DEFAULT '10',\n" + - "city_code VARCHAR(100),\n" + - "user_name VARCHAR(32) DEFAULT '',\n" + - "pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY RANGE(event_day)(\n" + - "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + - "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + - "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + - "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - "\"replication_num\" = \"1\"\n" + - ");"); + "event_day DATE,\n" + + "site_id INT DEFAULT '10',\n" + + "city_code VARCHAR(100),\n" + + "user_name VARCHAR(32) DEFAULT '',\n" + + "pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY RANGE(event_day)(\n" + + "PARTITION p20200321 VALUES LESS THAN (\"2020-03-22\"),\n" + + "PARTITION p20200322 VALUES LESS THAN (\"2020-03-23\"),\n" + + "PARTITION p20200323 VALUES LESS THAN (\"2020-03-24\"),\n" + + "PARTITION p20200324 VALUES LESS THAN MAXVALUE\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + "\"replication_num\" = \"1\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE test_partition_storage_medium SET(\"default.storage_medium\" = \"SSD\");"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterTable(ctx, alterTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable olapTable = (OlapTable) db.getTable("test_partition_storage_medium"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_partition_storage_medium"); Assert.assertTrue(olapTable.getStorageMedium().equals("SSD")); } @Test public void testAlterTableStorageType() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE test_storage_type (\n" + - "event_day DATE,\n" + - "site_id INT DEFAULT '10',\n" + - "city_code VARCHAR(100),\n" + - "user_name VARCHAR(32) DEFAULT '',\n" + - "pv BIGINT DEFAULT '2'\n" + - ")\n" + - "PRIMARY KEY(event_day, site_id, city_code, user_name)\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_type\" = \"column_with_row\"\n" + - ");"); + "event_day DATE,\n" + + "site_id INT DEFAULT '10',\n" + + "city_code VARCHAR(100),\n" + + "user_name VARCHAR(32) DEFAULT '',\n" + + "pv BIGINT DEFAULT '2'\n" + + ")\n" + + "PRIMARY KEY(event_day, site_id, city_code, user_name)\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_type\" = \"column_with_row\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String sql1 = "ALTER TABLE test_storage_type SET(\"storage_type\" = \"column\");"; AnalysisException e1 = - Assert.assertThrows(AnalysisException.class, () -> UtFrameUtils.parseStmtWithNewParser(sql1, ctx)); + Assert.assertThrows(AnalysisException.class, () -> UtFrameUtils.parseStmtWithNewParser(sql1, ctx)); Assert.assertTrue(e1.getMessage().contains("Can't change storage type")); String sql2 = "ALTER TABLE test_storage_type SET(\"storage_type\" = \"column_with_row\");"; AnalysisException e2 = - Assert.assertThrows(AnalysisException.class, () -> UtFrameUtils.parseStmtWithNewParser(sql2, ctx)); + Assert.assertThrows(AnalysisException.class, () -> UtFrameUtils.parseStmtWithNewParser(sql2, ctx)); Assert.assertTrue(e2.getMessage().contains("Can't change storage type")); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable olapTable = (OlapTable) db.getTable("test_storage_type"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_storage_type"); Assert.assertTrue(olapTable.getStorageType().equals(TStorageType.COLUMN_WITH_ROW)); } public void testAlterTableLocationProp() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // add label to backend SystemInfoService systemInfoService = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); @@ -317,22 +321,22 @@ public void testAlterTableLocationProp() throws Exception { List backendIds = systemInfoService.getBackendIds(); Backend backend = systemInfoService.getBackend(backendIds.get(0)); String modifyBackendPropSqlStr = "alter system modify backend '" + backend.getHost() + - ":" + backend.getHeartbeatPort() + "' set ('" + - AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:rack1')"; + ":" + backend.getHeartbeatPort() + "' set ('" + + AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:rack1')"; DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(modifyBackendPropSqlStr, connectContext), - connectContext); + connectContext); String sql = "CREATE TABLE test.`test_location_alter` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; starRocksAssert.withTable(sql); UtFrameUtils.PseudoJournalReplayer.resetFollowerJournalQueue(); @@ -341,48 +345,50 @@ public void testAlterTableLocationProp() throws Exception { // ** test alter table location to rack:* sql = "ALTER TABLE test.`test_location_alter` SET ('" + - PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:*');"; + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:*');"; DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(sql, connectContext), - connectContext); - Assert.assertEquals("rack", ((OlapTable) testDb.getTable("test_location_alter")) - .getLocation().keySet().stream().findFirst().get()); - Assert.assertEquals("*", ((OlapTable) testDb.getTable("test_location_alter")) - .getLocation().get("rack").stream().findFirst().get()); + connectContext); + Assert.assertEquals("rack", ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_alter")) + .getLocation().keySet().stream().findFirst().get()); + Assert.assertEquals("*", ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_alter")) + .getLocation().get("rack").stream().findFirst().get()); // ** test alter table location to nil sql = "ALTER TABLE test.`test_location_alter` SET ('" + - PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = '');"; + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = '');"; DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(sql, connectContext), - connectContext); - Assert.assertNull(((OlapTable) testDb.getTable("test_location_alter")) - .getLocation()); + connectContext); + Assert.assertNull(((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_alter")) + .getLocation()); // ** test replay from edit log: alter to rack:* LocalMetastore localMetastoreFollower = new LocalMetastore(GlobalStateMgr.getCurrentState(), null, null); localMetastoreFollower.load(initialImage.getMetaBlockReader()); ModifyTablePropertyOperationLog info = (ModifyTablePropertyOperationLog) - UtFrameUtils.PseudoJournalReplayer.replayNextJournal(OperationType.OP_ALTER_TABLE_PROPERTIES); + UtFrameUtils.PseudoJournalReplayer.replayNextJournal(OperationType.OP_ALTER_TABLE_PROPERTIES); localMetastoreFollower.replayModifyTableProperty(OperationType.OP_ALTER_TABLE_PROPERTIES, info); OlapTable olapTable = (OlapTable) localMetastoreFollower.getDb("test") - .getTable("test_location_alter"); + .getTable("test_location_alter"); System.out.println(olapTable.getLocation()); Assert.assertEquals(1, olapTable.getLocation().size()); Assert.assertTrue(olapTable.getLocation().containsKey("rack")); - // ** test replay from edit log: alter to nil info = (ModifyTablePropertyOperationLog) - UtFrameUtils.PseudoJournalReplayer.replayNextJournal(OperationType.OP_ALTER_TABLE_PROPERTIES); + UtFrameUtils.PseudoJournalReplayer.replayNextJournal(OperationType.OP_ALTER_TABLE_PROPERTIES); localMetastoreFollower.replayModifyTableProperty(OperationType.OP_ALTER_TABLE_PROPERTIES, info); olapTable = (OlapTable) localMetastoreFollower.getDb("test") - .getTable("test_location_alter"); + .getTable("test_location_alter"); System.out.println(olapTable.getLocation()); Assert.assertNull(olapTable.getLocation()); } @Test public void testAlterColocateTableLocationProp() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // add label to backend SystemInfoService systemInfoService = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); @@ -390,29 +396,30 @@ public void testAlterColocateTableLocationProp() throws Exception { List backendIds = systemInfoService.getBackendIds(); Backend backend = systemInfoService.getBackend(backendIds.get(0)); String modifyBackendPropSqlStr = "alter system modify backend '" + backend.getHost() + - ":" + backend.getHeartbeatPort() + "' set ('" + - AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:rack1')"; + ":" + backend.getHeartbeatPort() + "' set ('" + + AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:rack1')"; DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(modifyBackendPropSqlStr, connectContext), - connectContext); + connectContext); String sql = "CREATE TABLE test.`test_location_colocate_alter1` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\",\n" + - " \"colocate_with\" = \"cg1\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"colocate_with\" = \"cg1\"\n" + + ");"; starRocksAssert.withTable(sql); - OlapTable olapTable = (OlapTable) testDb.getTable("test_location_colocate_alter1"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_colocate_alter1"); Assert.assertNull(olapTable.getLocation()); sql = "ALTER TABLE test.`test_location_colocate_alter1` SET ('" + - PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:*');"; + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:*');"; try { DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(sql, connectContext), connectContext); } catch (DdlException e) { @@ -422,7 +429,7 @@ public void testAlterColocateTableLocationProp() throws Exception { @Test public void testAlterLocationPropTableToColocate() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // add label to backend SystemInfoService systemInfoService = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); @@ -430,28 +437,29 @@ public void testAlterLocationPropTableToColocate() throws Exception { List backendIds = systemInfoService.getBackendIds(); Backend backend = systemInfoService.getBackend(backendIds.get(0)); String modifyBackendPropSqlStr = "alter system modify backend '" + backend.getHost() + - ":" + backend.getHeartbeatPort() + "' set ('" + - AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:rack1')"; + ":" + backend.getHeartbeatPort() + "' set ('" + + AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:rack1')"; DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(modifyBackendPropSqlStr, connectContext), - connectContext); + connectContext); String sql = "CREATE TABLE test.`test_location_colocate_alter2` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ");"; starRocksAssert.withTable(sql); - OlapTable olapTable = (OlapTable) testDb.getTable("test_location_colocate_alter2"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_colocate_alter2"); Assert.assertTrue(olapTable.getLocation().containsKey("*")); sql = "ALTER TABLE test.`test_location_colocate_alter1` SET ('" + - PropertyAnalyzer.PROPERTIES_COLOCATE_WITH + "' = 'cg1');"; + PropertyAnalyzer.PROPERTIES_COLOCATE_WITH + "' = 'cg1');"; try { DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(sql, connectContext), connectContext); } catch (DdlException e) { diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java index bcd73343f8d72..f04612e80dc1b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java @@ -144,107 +144,107 @@ public static void beforeClass() throws Exception { starRocksAssert = new StarRocksAssert(connectContext); starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - - .withTable("CREATE TABLE test.tbl2\n" + - "(\n" + - " k1 date,\n" + - " v1 int sum\n" + - ")\n" + - "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - - .withTable("CREATE TABLE test.tbl3\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - - .withTable("CREATE TABLE test.tbl4\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01'),\n" + - " PARTITION p3 values less than('2020-04-01'),\n" + - " PARTITION p4 values less than('2020-05-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES" + - "(" + - " 'replication_num' = '1',\n" + - " 'in_memory' = 'false',\n" + - " 'storage_medium' = 'SSD',\n" + - " 'storage_cooldown_time' = '9999-12-31 00:00:00'\n" + - ");") - .withTable("CREATE TABLE t_recharge_detail(\n" + - " id bigint ,\n" + - " user_id bigint ,\n" + - " recharge_money decimal(32,2) , \n" + - " province varchar(20) not null,\n" + - " dt varchar(20) not null\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (dt,province) (\n" + - " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\")),\n" + - " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10 \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\"\n" + - ");") - .withTable("CREATE TABLE test.site_access_date_trunc (\n" + - " event_day DATETIME NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - " \"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_time_slice (\n" + - " event_day datetime,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY time_slice(event_day, interval 1 day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\n" + - " \"partition_live_number\" = \"3\",\n" + - " \"replication_num\" = \"1\"\n" + - ");"); + .withTable("CREATE TABLE test.tbl1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + + .withTable("CREATE TABLE test.tbl2\n" + + "(\n" + + " k1 date,\n" + + " v1 int sum\n" + + ")\n" + + "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + + .withTable("CREATE TABLE test.tbl3\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + + .withTable("CREATE TABLE test.tbl4\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01'),\n" + + " PARTITION p3 values less than('2020-04-01'),\n" + + " PARTITION p4 values less than('2020-05-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES" + + "(" + + " 'replication_num' = '1',\n" + + " 'in_memory' = 'false',\n" + + " 'storage_medium' = 'SSD',\n" + + " 'storage_cooldown_time' = '9999-12-31 00:00:00'\n" + + ");") + .withTable("CREATE TABLE t_recharge_detail(\n" + + " id bigint ,\n" + + " user_id bigint ,\n" + + " recharge_money decimal(32,2) , \n" + + " province varchar(20) not null,\n" + + " dt varchar(20) not null\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (dt,province) (\n" + + " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\")),\n" + + " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\"\n" + + ");") + .withTable("CREATE TABLE test.site_access_date_trunc (\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + " \"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_time_slice (\n" + + " event_day datetime,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY time_slice(event_day, interval 1 day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\n" + + " \"partition_live_number\" = \"3\",\n" + + " \"replication_num\" = \"1\"\n" + + ");"); } @AfterClass @@ -276,19 +276,19 @@ private static void createTable(String sql) throws Exception { private static void createMaterializedView(String sql) throws Exception { CreateMaterializedViewStatement createMaterializedViewStatement = - (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().createMaterializedView(createMaterializedViewStatement); } private static void dropMaterializedView(String sql) throws Exception { DropMaterializedViewStmt dropMaterializedViewStmt = - (DropMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + (DropMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().dropMaterializedView(dropMaterializedViewStmt); } private static void alterMaterializedView(String sql, boolean expectedException) throws Exception { AlterMaterializedViewStmt alterMaterializedViewStmt = - (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); try { GlobalStateMgr.getCurrentState().getLocalMetastore().alterMaterializedView(alterMaterializedViewStmt); if (expectedException) { @@ -304,12 +304,12 @@ private static void alterMaterializedView(String sql, boolean expectedException) private static void refreshMaterializedView(String sql) throws Exception { RefreshMaterializedViewStatement refreshMaterializedViewStatement = - (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); try { GlobalStateMgr.getCurrentState().getLocalMetastore() - .refreshMaterializedView(refreshMaterializedViewStatement.getMvName().getDb(), - refreshMaterializedViewStatement.getMvName().getTbl(), false, null, - Constants.TaskRunPriority.LOWEST.value(), false, true); + .refreshMaterializedView(refreshMaterializedViewStatement.getMvName().getDb(), + refreshMaterializedViewStatement.getMvName().getTbl(), false, null, + Constants.TaskRunPriority.LOWEST.value(), false, true); } catch (Exception e) { e.printStackTrace(); Assert.fail(); @@ -318,7 +318,7 @@ private static void refreshMaterializedView(String sql) throws Exception { private static void cancelRefreshMaterializedView(String sql, boolean expectedException) throws Exception { CancelRefreshMaterializedViewStmt cancelRefresh = - (CancelRefreshMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + (CancelRefreshMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); try { GlobalStateMgr.getCurrentState().getLocalMetastore().cancelRefreshMaterializedView(cancelRefresh); if (expectedException) { @@ -359,36 +359,36 @@ private static void alterTableWithNewParserAndExceptionMsg(String sql, String ms @Test public void testRenameMaterializedView() throws Exception { starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE test.testTable1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.testTable1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String sql = "create materialized view mv1 " + - "partition by k1 " + - "distributed by hash(k2) " + - "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select k1, k2 from test.testTable1;"; + "partition by k1 " + + "distributed by hash(k2) " + + "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select k1, k2 from test.testTable1;"; createMaterializedView(sql); String alterStmt = "alter materialized view test.mv1 rename mv2"; alterMaterializedView(alterStmt, false); - MaterializedView materializedView = (MaterializedView) GlobalStateMgr.getCurrentState(). - getDb("test").getTable("mv2"); + MaterializedView materializedView = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore(). + getDb("test").getTable("mv2"); TaskManager taskManager = GlobalStateMgr.getCurrentState().getTaskManager(); Task task = taskManager.getTask(TaskBuilder.getMvTaskName(materializedView.getId())); Assert.assertEquals("insert overwrite `mv2` SELECT `test`.`testTable1`.`k1`, `test`.`testTable1`.`k2`\n" + - "FROM `test`.`testTable1`", task.getDefinition()); + "FROM `test`.`testTable1`", task.getDefinition()); ConnectContext.get().setCurrentUserIdentity(UserIdentity.ROOT); ConnectContext.get().setCurrentRoleIds(UserIdentity.ROOT); dropMaterializedView("drop materialized view test.mv2"); @@ -397,45 +397,45 @@ public void testRenameMaterializedView() throws Exception { @Test public void testCouldNotFindMaterializedView() throws Exception { starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE test.testTable1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withTable("CREATE TABLE test.testTable2\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.testTable1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withTable("CREATE TABLE test.testTable2\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String sql = "create materialized view mv1 " + - "partition by k1 " + - "distributed by hash(k2) " + - "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select k1, k2 from test.testTable1;"; + "partition by k1 " + + "distributed by hash(k2) " + + "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select k1, k2 from test.testTable1;"; createMaterializedView(sql); starRocksAssert.getCtx().setCurrentRoleIds(GlobalStateMgr.getCurrentState().getAuthorizationMgr().getRoleIdsByUser( - starRocksAssert.getCtx().getCurrentUserIdentity())); + starRocksAssert.getCtx().getCurrentUserIdentity())); dropMaterializedView("drop materialized view test.mv1"); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test").getTable("testTable1"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("testTable1"); // this for mock olapTable.getIndexNameById(mvIdx.getId()) == Null table.deleteIndexInfo("testTable1"); try { @@ -449,19 +449,19 @@ public void testCouldNotFindMaterializedView() throws Exception { @Test public void testRenameTable() throws Exception { starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE test.testRenameTable1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.testRenameTable1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String alterStmt = "alter table test.testRenameTable1 rename testRenameTable2"; alterTableWithNewParser(alterStmt, false); } @@ -469,27 +469,27 @@ public void testRenameTable() throws Exception { @Test public void testChangeMaterializedViewRefreshScheme() throws Exception { starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE test.testTable2\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.testTable2\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String sql = "create materialized view mv1 " + - "partition by k1 " + - "distributed by hash(k2) " + - "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select k1, k2 from test.testTable2;"; + "partition by k1 " + + "distributed by hash(k2) " + + "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select k1, k2 from test.testTable2;"; createMaterializedView(sql); String alterStmt = "alter materialized view mv1 refresh async EVERY(INTERVAL 1 minute)"; alterMaterializedView(alterStmt, false); @@ -503,61 +503,61 @@ public void testChangeMaterializedViewRefreshScheme() throws Exception { @Test public void testRefreshMaterializedView() throws Exception { starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE test.testTable3\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.testTable3\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String sql = "create materialized view mv1 " + - "partition by k1 " + - "distributed by hash(k2) " + - "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select k1, k2 from test.testTable3;"; + "partition by k1 " + + "distributed by hash(k2) " + + "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select k1, k2 from test.testTable3;"; createMaterializedView(sql); String alterStmt = "refresh materialized view test.mv1"; refreshMaterializedView(alterStmt); starRocksAssert.getCtx().setCurrentRoleIds(GlobalStateMgr.getCurrentState().getAuthorizationMgr().getRoleIdsByUser( - starRocksAssert.getCtx().getCurrentUserIdentity())); + starRocksAssert.getCtx().getCurrentUserIdentity())); dropMaterializedView("drop materialized view test.mv1"); } @Test public void testCancelRefreshMaterializedView() throws Exception { starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE test.testTable4\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.testTable4\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String sql = "create materialized view mv1 " + - "partition by k1 " + - "distributed by hash(k2) " + - "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select k1, k2 from test.testTable4;"; + "partition by k1 " + + "distributed by hash(k2) " + + "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select k1, k2 from test.testTable4;"; starRocksAssert.getCtx().setCurrentRoleIds(GlobalStateMgr.getCurrentState().getAuthorizationMgr().getRoleIdsByUser( - starRocksAssert.getCtx().getCurrentUserIdentity())); + starRocksAssert.getCtx().getCurrentUserIdentity())); createMaterializedView(sql); String alterStmt = "refresh materialized view test.mv1"; refreshMaterializedView(alterStmt); @@ -567,12 +567,12 @@ public void testCancelRefreshMaterializedView() throws Exception { @Test public void testConflictAlterOperations() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("tbl1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); String stmt = - "alter table test.tbl1 add partition p3 values less than('2020-04-01'), " + - "add partition p4 values less than('2020-05-01')"; + "alter table test.tbl1 add partition p3 values less than('2020-04-01'), " + + "add partition p4 values less than('2020-05-01')"; alterTableWithNewParser(stmt, true); stmt = "alter table test.tbl1 add partition p3 values less than('2020-04-01'), drop partition p4"; @@ -600,12 +600,12 @@ public void testConflictAlterOperations() throws Exception { // enable dynamic partition // not adding the `start` property so that it won't drop the origin partition p1, p2 and p3 stmt = "alter table test.tbl1 set (\n" + - "'dynamic_partition.enable' = 'true',\n" + - "'dynamic_partition.time_unit' = 'DAY',\n" + - "'dynamic_partition.end' = '3',\n" + - "'dynamic_partition.prefix' = 'p',\n" + - "'dynamic_partition.buckets' = '3'\n" + - " );"; + "'dynamic_partition.enable' = 'true',\n" + + "'dynamic_partition.time_unit' = 'DAY',\n" + + "'dynamic_partition.end' = '3',\n" + + "'dynamic_partition.prefix' = 'p',\n" + + "'dynamic_partition.buckets' = '3'\n" + + " );"; alterTableWithNewParser(stmt, false); Assert.assertTrue(tbl.getTableProperty().getDynamicPartitionProperty().isEnabled()); @@ -613,12 +613,12 @@ public void testConflictAlterOperations() throws Exception { // add partition when dynamic partition is enable stmt = "alter table test.tbl1 add partition p3 values less than('2020-04-01') " + - "distributed by hash(k2) buckets 4 PROPERTIES ('replication_num' = '1')"; + "distributed by hash(k2) buckets 4 PROPERTIES ('replication_num' = '1')"; alterTableWithNewParser(stmt, true); // add temp partition when dynamic partition is enable stmt = "alter table test.tbl1 add temporary partition tp3 values less than('2020-04-01') " + - "distributed by hash(k2) buckets 4 PROPERTIES ('replication_num' = '1')"; + "distributed by hash(k2) buckets 4 PROPERTIES ('replication_num' = '1')"; alterTableWithNewParser(stmt, false); Assert.assertEquals(1, tbl.getTempPartitions().size()); @@ -629,7 +629,7 @@ public void testConflictAlterOperations() throws Exception { // add partition when dynamic partition is disable stmt = "alter table test.tbl1 add partition p3 values less than('2020-04-01') " + - "distributed by hash(k2) buckets 4"; + "distributed by hash(k2) buckets 4"; alterTableWithNewParser(stmt, false); // set table's default replication num @@ -646,25 +646,25 @@ public void testConflictAlterOperations() throws Exception { Assert.assertEquals(Short.valueOf("1"), Short.valueOf(tbl.getPartitionInfo().getReplicationNum(p1.getId()))); // set un-partitioned table's real replication num - OlapTable tbl2 = (OlapTable) db.getTable("tbl2"); + OlapTable tbl2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl2"); Partition partition = tbl2.getPartition(tbl2.getName()); Assert.assertEquals(Short.valueOf("1"), - Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); + Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); // partition replication num and table default replication num are updated at the same time in unpartitioned table stmt = "alter table test.tbl2 set ('replication_num' = '3');"; alterTableWithNewParser(stmt, false); Assert.assertEquals(Short.valueOf("3"), - Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); + Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); Assert.assertEquals(Short.valueOf("3"), tbl2.getDefaultReplicationNum()); stmt = "alter table test.tbl2 set ('default.replication_num' = '2');"; alterTableWithNewParser(stmt, false); Assert.assertEquals(Short.valueOf("2"), - Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); + Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); Assert.assertEquals(Short.valueOf("2"), tbl2.getDefaultReplicationNum()); stmt = "alter table test.tbl2 modify partition tbl2 set ('replication_num' = '1');"; alterTableWithNewParser(stmt, false); Assert.assertEquals(Short.valueOf("1"), - Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); + Short.valueOf(tbl2.getPartitionInfo().getReplicationNum(partition.getId()))); Assert.assertEquals(Short.valueOf("1"), tbl2.getDefaultReplicationNum()); Thread.sleep(5000); // sleep to wait dynamic partition scheduler run @@ -677,8 +677,8 @@ public void testConflictAlterOperations() throws Exception { alterTableWithNewParser(stmt, false); stmt = "alter table test.tbl1 " + - "add TEMPORARY partition p5 values [('2020-04-10'), ('2020-05-10')) ('replication_num' = '1') " + - "DISTRIBUTED BY HASH(k2) BUCKETS 3 PROPERTIES('replication_num' = '1');"; + "add TEMPORARY partition p5 values [('2020-04-10'), ('2020-05-10')) ('replication_num' = '1') " + + "DISTRIBUTED BY HASH(k2) BUCKETS 3 PROPERTIES('replication_num' = '1');"; alterTableWithNewParser(stmt, false); //rename table stmt = "alter table test.tbl1 rename newTableName"; @@ -688,8 +688,8 @@ public void testConflictAlterOperations() throws Exception { // test batch update range partitions' properties @Test public void testBatchUpdatePartitionProperties() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl4 = (OlapTable) db.getTable("tbl4"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl4 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl4"); Partition p1 = tbl4.getPartition("p1"); Partition p2 = tbl4.getPartition("p2"); Partition p3 = tbl4.getPartition("p3"); @@ -700,12 +700,12 @@ public void testBatchUpdatePartitionProperties() throws Exception { List partitionList = Lists.newArrayList(p1, p2, p4); for (Partition partition : partitionList) { Assert.assertEquals(Short.valueOf("1"), - Short.valueOf(tbl4.getPartitionInfo().getReplicationNum(partition.getId()))); + Short.valueOf(tbl4.getPartitionInfo().getReplicationNum(partition.getId()))); } alterTableWithNewParser(stmt, false); for (Partition partition : partitionList) { Assert.assertEquals(Short.valueOf("3"), - Short.valueOf(tbl4.getPartitionInfo().getReplicationNum(partition.getId()))); + Short.valueOf(tbl4.getPartitionInfo().getReplicationNum(partition.getId()))); } Assert.assertEquals(Short.valueOf("1"), Short.valueOf(tbl4.getPartitionInfo().getReplicationNum(p3.getId()))); @@ -743,7 +743,7 @@ public void testBatchUpdatePartitionProperties() throws Exception { alterTableWithNewParser(stmt, false); for (Partition partition : partitionList) { Assert.assertEquals(Short.valueOf("1"), - Short.valueOf(tbl4.getPartitionInfo().getReplicationNum(partition.getId()))); + Short.valueOf(tbl4.getPartitionInfo().getReplicationNum(partition.getId()))); } } @@ -763,8 +763,8 @@ public void testDynamicPartitionDropAndAdd() throws Exception { alterTable(stmt, false); Thread.sleep(5000); // sleep to wait dynamic partition scheduler run - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("tbl3"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl3"); Assert.assertEquals(4, tbl.getPartitionNames().size()); Assert.assertNull(tbl.getPartition("p1")); Assert.assertNull(tbl.getPartition("p2")); @@ -778,11 +778,11 @@ private void waitSchemaChangeJobDone(boolean rollupJob, OlapTable tb) throws Int for (AlterJobV2 alterJobV2 : alterJobs.values()) { while (!alterJobV2.getJobState().isFinalState()) { System.out.println( - "alter job " + alterJobV2.getJobId() + " is running. state: " + alterJobV2.getJobState()); + "alter job " + alterJobV2.getJobId() + " is running. state: " + alterJobV2.getJobState()); Thread.sleep(1000); } System.out.println(alterJobV2.getType() + " alter job " + alterJobV2.getJobId() + " is done. state: " + - alterJobV2.getJobState()); + alterJobV2.getJobState()); Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); } checkTableStateToNormal(tb); @@ -792,36 +792,36 @@ private void waitSchemaChangeJobDone(boolean rollupJob, OlapTable tb) throws Int public void testSetDynamicPropertiesInNormalTable() throws Exception { String tableName = "no_dynamic_table"; String createOlapTblStmt = "CREATE TABLE test.`" + tableName + "` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");"; + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"; createTable(createOlapTblStmt); String alterStmt = "alter table test." + tableName + " set (\"dynamic_partition.enable\" = \"true\");"; alterTableWithNewParserAndExceptionMsg(alterStmt, "Table test.no_dynamic_table is not a dynamic partition table."); // test set dynamic properties in a no dynamic partition table String stmt = "alter table test." + tableName + " set (\n" + - "'dynamic_partition.enable' = 'true',\n" + - "'dynamic_partition.time_unit' = 'DAY',\n" + - "'dynamic_partition.start' = '-3',\n" + - "'dynamic_partition.end' = '3',\n" + - "'dynamic_partition.prefix' = 'p',\n" + - "'dynamic_partition.buckets' = '3'\n" + - " );"; + "'dynamic_partition.enable' = 'true',\n" + + "'dynamic_partition.time_unit' = 'DAY',\n" + + "'dynamic_partition.start' = '-3',\n" + + "'dynamic_partition.end' = '3',\n" + + "'dynamic_partition.prefix' = 'p',\n" + + "'dynamic_partition.buckets' = '3'\n" + + " );"; alterTableWithNewParser(stmt, false); } @@ -829,30 +829,30 @@ public void testSetDynamicPropertiesInNormalTable() throws Exception { public void testSetDynamicPropertiesInDynamicPartitionTable() throws Exception { String tableName = "dynamic_table"; String createOlapTblStmt = "CREATE TABLE test.`" + tableName + "` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; createTable(createOlapTblStmt); String alterStmt1 = "alter table test." + tableName + " set (\"dynamic_partition.enable\" = \"false\");"; @@ -873,32 +873,32 @@ public void testSetDynamicPropertiesInDynamicPartitionTable() throws Exception { public void testDynamicPartitionTableMetaFailed() throws Exception { String tableName = "dynamic_table_test"; String createOlapTblStmt = "CREATE TABLE test.`" + tableName + "` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; createTable(createOlapTblStmt); - OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test").getTable(tableName); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable(tableName); olapTable.getTableProperty().getProperties().remove("dynamic_partition.end"); olapTable.getTableProperty().gsonPostProcess(); } @@ -906,52 +906,52 @@ public void testDynamicPartitionTableMetaFailed() throws Exception { @Test public void testSwapTable() throws Exception { String stmt1 = "CREATE TABLE test.replace1\n" + - "(\n" + - " k1 int, k2 int, k3 int sum\n" + - ")\n" + - "AGGREGATE KEY(k1, k2)\n" + - "DISTRIBUTED BY HASH(k1) BUCKETS 10\n" + - "rollup (\n" + - "r1(k1),\n" + - "r2(k2, k3)\n" + - ")\n" + - "PROPERTIES(\"replication_num\" = \"1\");"; + "(\n" + + " k1 int, k2 int, k3 int sum\n" + + ")\n" + + "AGGREGATE KEY(k1, k2)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 10\n" + + "rollup (\n" + + "r1(k1),\n" + + "r2(k2, k3)\n" + + ")\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; String stmt2 = "CREATE TABLE test.r1\n" + - "(\n" + - " k1 int, k2 int\n" + - ")\n" + - "DISTRIBUTED BY HASH(k1) BUCKETS 11\n" + - "PROPERTIES(\"replication_num\" = \"1\");"; + "(\n" + + " k1 int, k2 int\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 11\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; String stmt3 = "CREATE TABLE test.replace2\n" + - "(\n" + - " k1 int, k2 int\n" + - ")\n" + - "DISTRIBUTED BY HASH(k1) BUCKETS 11\n" + - "PROPERTIES(\"replication_num\" = \"1\");"; + "(\n" + + " k1 int, k2 int\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 11\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; String stmt4 = "CREATE TABLE test.replace3\n" + - "(\n" + - " k1 int, k2 int, k3 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - "\tPARTITION p1 values less than(\"100\"),\n" + - "\tPARTITION p2 values less than(\"200\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + - "rollup (\n" + - "r3(k1),\n" + - "r4(k2, k3)\n" + - ")\n" + - "PROPERTIES(\"replication_num\" = \"1\");"; + "(\n" + + " k1 int, k2 int, k3 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + "\tPARTITION p1 values less than(\"100\"),\n" + + "\tPARTITION p2 values less than(\"200\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "rollup (\n" + + "r3(k1),\n" + + "r4(k2, k3)\n" + + ")\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; createTable(stmt1); createTable(stmt2); createTable(stmt3); createTable(stmt4); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // name conflict String replaceStmt = "ALTER TABLE test.replace1 SWAP WITH r1"; @@ -959,25 +959,27 @@ public void testSwapTable() throws Exception { // replace1 with replace2 replaceStmt = "ALTER TABLE test.replace1 SWAP WITH replace2"; - OlapTable replace1 = (OlapTable) db.getTable("replace1"); - OlapTable replace2 = (OlapTable) db.getTable("replace2"); + OlapTable replace1 = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "replace1"); + OlapTable replace2 = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "replace2"); Assert.assertEquals(3, - replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) + .size()); Assert.assertEquals(1, - replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) + .size()); alterTableWithNewParser(replaceStmt, false); - replace1 = (OlapTable) db.getTable("replace1"); - replace2 = (OlapTable) db.getTable("replace2"); + replace1 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "replace1"); + replace2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "replace2"); Assert.assertEquals(1, - replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) + .size()); Assert.assertEquals(3, - replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) - .size()); + replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE) + .size()); Assert.assertEquals("replace1", replace1.getIndexNameById(replace1.getBaseIndexId())); Assert.assertEquals("replace2", replace2.getIndexNameById(replace2.getBaseIndexId())); } @@ -989,34 +991,34 @@ public void testCatalogAddPartitionsDay() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20140104\") EVERY (INTERVAL 1 DAY)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20140104\") EVERY (INTERVAL 1 DAY)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition ADD\n" + - " PARTITIONS START (\"2017-01-03\") END (\"2017-01-07\") EVERY (interval 1 day)"; + " PARTITIONS START (\"2017-01-03\") END (\"2017-01-07\") EVERY (interval 1 day)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertNotNull(table.getPartition("p20170103")); Assert.assertNotNull(table.getPartition("p20170104")); @@ -1037,34 +1039,34 @@ public void testAddPhysicalPartition() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "DISTRIBUTED BY RANDOM BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "DISTRIBUTED BY RANDOM BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Optional partition = table.getPartitions().stream().findFirst(); Assert.assertTrue(partition.isPresent()); Assert.assertEquals(table.getPhysicalPartitions().size(), 1); GlobalStateMgr.getCurrentState().getLocalMetastore().addSubPartitions(db, table, partition.get(), 1, - WarehouseManager.DEFAULT_WAREHOUSE_ID); + WarehouseManager.DEFAULT_WAREHOUSE_ID); Assert.assertEquals(partition.get().getSubPartitions().size(), 2); Assert.assertEquals(table.getPhysicalPartitions().size(), 2); GlobalStateMgr.getCurrentState().getLocalMetastore().addSubPartitions(db, table, partition.get(), 2, - WarehouseManager.DEFAULT_WAREHOUSE_ID); + WarehouseManager.DEFAULT_WAREHOUSE_ID); Assert.assertEquals(partition.get().getSubPartitions().size(), 4); Assert.assertEquals(table.getPhysicalPartitions().size(), 4); @@ -1090,33 +1092,33 @@ public void testAddRangePhysicalPartition() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20140104\") EVERY (INTERVAL 1 DAY)\n" + - ")\n" + - "DISTRIBUTED BY RANDOM BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20140104\") EVERY (INTERVAL 1 DAY)\n" + + ")\n" + + "DISTRIBUTED BY RANDOM BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertEquals(table.getPhysicalPartitions().size(), 3); Partition partition = table.getPartition("p20140101"); Assert.assertNotNull(partition); GlobalStateMgr.getCurrentState().getLocalMetastore().addSubPartitions(db, table, partition, 1, - WarehouseManager.DEFAULT_WAREHOUSE_ID); + WarehouseManager.DEFAULT_WAREHOUSE_ID); Assert.assertEquals(table.getPhysicalPartitions().size(), 4); Assert.assertEquals(partition.getSubPartitions().size(), 2); @@ -1124,7 +1126,7 @@ public void testAddRangePhysicalPartition() throws Exception { Assert.assertNotNull(partition); GlobalStateMgr.getCurrentState().getLocalMetastore().addSubPartitions(db, table, partition, 2, - WarehouseManager.DEFAULT_WAREHOUSE_ID); + WarehouseManager.DEFAULT_WAREHOUSE_ID); Assert.assertEquals(table.getPhysicalPartitions().size(), 6); Assert.assertEquals(partition.getSubPartitions().size(), 3); @@ -1140,29 +1142,29 @@ public void testAddPhysicalPartitionForHash() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Optional partition = table.getPartitions().stream().findFirst(); Assert.assertTrue(partition.isPresent()); Assert.assertEquals(table.getPhysicalPartitions().size(), 1); GlobalStateMgr.getCurrentState().getLocalMetastore().addSubPartitions(db, table, partition.get(), 1, - WarehouseManager.DEFAULT_WAREHOUSE_ID); + WarehouseManager.DEFAULT_WAREHOUSE_ID); } @Test @@ -1192,33 +1194,33 @@ public void testAddBackend() throws Exception { public void testCatalogAddPartitions5Day() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition ADD\n" + - " PARTITIONS START (\"2017-01-03\") END (\"2017-01-15\") EVERY (interval 5 day)"; + " PARTITIONS START (\"2017-01-03\") END (\"2017-01-15\") EVERY (interval 5 day)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertNotNull(table.getPartition("p20170103")); Assert.assertNotNull(table.getPartition("p20170108")); @@ -1233,31 +1235,31 @@ public void testCatalogAddPartitions5Day() throws Exception { public void testCatalogAddPartitionsDayConflictException() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition_exception (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20140104\") EVERY (INTERVAL 1 DAY)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20140104\") EVERY (INTERVAL 1 DAY)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition_exception ADD\n" + - " PARTITIONS START (\"2014-01-01\") END (\"2014-01-04\") EVERY (interval 1 day)"; + " PARTITIONS START (\"2014-01-01\") END (\"2014-01-04\") EVERY (interval 1 day)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exception", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exception", addPartitionClause); } @Test @@ -1265,33 +1267,33 @@ public void testCatalogAddPartitionsWeekWithoutCheck() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); Config.enable_create_partial_partition_in_batch = true; String createSQL = "CREATE TABLE test.test_partition_week (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition_week ADD\n" + - " PARTITIONS START (\"2017-03-25\") END (\"2017-04-10\") EVERY (interval 1 week)"; + " PARTITIONS START (\"2017-03-25\") END (\"2017-04-10\") EVERY (interval 1 week)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_week", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_week", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition_week"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition_week"); Assert.assertNotNull(table.getPartition("p2017_12")); Assert.assertNotNull(table.getPartition("p2017_13")); @@ -1307,33 +1309,33 @@ public void testCatalogAddPartitionsWeekWithoutCheck() throws Exception { public void testCatalogAddPartitionsWeekWithCheck() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition_week (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition_week ADD\n" + - " PARTITIONS START (\"2017-03-20\") END (\"2017-04-10\") EVERY (interval 1 week)"; + " PARTITIONS START (\"2017-03-20\") END (\"2017-04-10\") EVERY (interval 1 week)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_week", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_week", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition_week"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition_week"); Assert.assertNotNull(table.getPartition("p2017_12")); Assert.assertNotNull(table.getPartition("p2017_13")); @@ -1351,33 +1353,33 @@ public void testCatalogAddPartitionsMonth() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition ADD\n" + - " PARTITIONS START (\"2017-01-01\") END (\"2017-04-01\") EVERY (interval 1 month)"; + " PARTITIONS START (\"2017-01-01\") END (\"2017-04-01\") EVERY (interval 1 month)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertNotNull(table.getPartition("p201701")); Assert.assertNotNull(table.getPartition("p201702")); @@ -1396,33 +1398,33 @@ public void testCatalogAddPartitionsYear() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition ADD\n" + - " PARTITIONS START (\"2017-01-01\") END (\"2020-01-01\") EVERY (interval 1 YEAR)"; + " PARTITIONS START (\"2017-01-01\") END (\"2020-01-01\") EVERY (interval 1 YEAR)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertNotNull(table.getPartition("p2017")); Assert.assertNotNull(table.getPartition("p2018")); @@ -1441,33 +1443,33 @@ public void testCatalogAddPartitionsNumber() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_partition (\n" + - " k2 INT,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 INT,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition ADD\n" + - " PARTITIONS START (\"1\") END (\"4\") EVERY (1)"; + " PARTITIONS START (\"1\") END (\"4\") EVERY (1)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertNotNull(table.getPartition("p1")); Assert.assertNotNull(table.getPartition("p2")); @@ -1486,39 +1488,39 @@ public void testCatalogAddPartitionsAtomicRange() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); try { String alterSQL = "ALTER TABLE test_partition ADD\n" + - " PARTITIONS START (\"2014-01-01\") END (\"2014-01-06\") EVERY (interval 1 day);"; + " PARTITIONS START (\"2014-01-01\") END (\"2014-01-06\") EVERY (interval 1 day);"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); Assert.fail(); } catch (AnalysisException ex) { } - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertNull(table.getPartition("p20140101")); Assert.assertNull(table.getPartition("p20140102")); @@ -1534,39 +1536,39 @@ public void testCatalogAddPartitionsZeroDay() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test_partition_0day (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140104\") END (\"20150104\") EVERY (INTERVAL 1 YEAR)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140104\") END (\"20150104\") EVERY (INTERVAL 1 YEAR)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); try { String alterSQL = "ALTER TABLE test_partition_0day ADD\n" + - " PARTITIONS START (\"2014-01-01\") END (\"2014-01-06\") EVERY (interval 0 day);"; + " PARTITIONS START (\"2014-01-01\") END (\"2014-01-06\") EVERY (interval 0 day);"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_0day", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_0day", addPartitionClause); Assert.fail(); } catch (AnalysisException ex) { } - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition_0day"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition_0day"); Assert.assertNull(table.getPartition("p20140101")); Assert.assertNull(table.getPartition("p20140102")); @@ -1585,39 +1587,39 @@ public void testCatalogAddPartitionsWithoutPartitions() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test_partition (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition ADD\n" + - " START (\"2015-01-01\") END (\"2015-01-06\") EVERY (interval 1 day);"; + " START (\"2015-01-01\") END (\"2015-01-06\") EVERY (interval 1 day);"; try { AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); Assert.fail(); } catch (AnalysisException ex) { } - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); Assert.assertNull(table.getPartition("p20140101")); Assert.assertNull(table.getPartition("p20140102")); @@ -1632,41 +1634,41 @@ public void testCatalogAddPartitionsWithoutPartitions() throws Exception { public void testCatalogAddPartitionsIfNotExist() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test_partition_exists (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = - "ALTER TABLE test_partition_exists ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-01')"; + "ALTER TABLE test_partition_exists ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-01')"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists", addPartitionClause); String alterSQL2 = - "ALTER TABLE test_partition_exists ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-02')"; + "ALTER TABLE test_partition_exists ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-02')"; AlterTableStmt alterTableStmt2 = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL2, ctx); AddPartitionClause addPartitionClause2 = (AddPartitionClause) alterTableStmt2.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists", addPartitionClause2); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists", addPartitionClause2); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition_exists"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition_exists"); Assert.assertEquals(2, table.getPartitions().size()); @@ -1679,41 +1681,41 @@ public void testCatalogAddPartitionsIfNotExist() throws Exception { public void testCatalogAddPartitionsSameNameShouldNotThrowError() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test_partition_exists2 (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = - "ALTER TABLE test_partition_exists2 ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-01')"; + "ALTER TABLE test_partition_exists2 ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-01')"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists2", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists2", addPartitionClause); String alterSQL2 = - "ALTER TABLE test_partition_exists2 ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-01')"; + "ALTER TABLE test_partition_exists2 ADD PARTITION IF NOT EXISTS p20210701 VALUES LESS THAN ('2021-07-01')"; AlterTableStmt alterTableStmt2 = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL2, ctx); AddPartitionClause addPartitionClause2 = (AddPartitionClause) alterTableStmt2.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists2", addPartitionClause2); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists2", addPartitionClause2); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition_exists2"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition_exists2"); Assert.assertEquals(2, table.getPartitions().size()); @@ -1726,39 +1728,39 @@ public void testCatalogAddPartitionsSameNameShouldNotThrowError() throws Excepti public void testCatalogAddPartitionsShouldThrowError() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test_partition_exists3 (\n" + - " k2 DATE,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(k2, k3)\n" + - "PARTITION BY RANGE (k2) (\n" + - " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " k2 DATE,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k2, k3)\n" + + "PARTITION BY RANGE (k2) (\n" + + " START (\"20140101\") END (\"20150101\") EVERY (INTERVAL 1 YEAR)\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_partition_exists3 ADD PARTITION p20210701 VALUES LESS THAN ('2021-07-01')"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists3", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists3", addPartitionClause); String alterSQL2 = "ALTER TABLE test_partition_exists3 ADD PARTITION p20210701 VALUES LESS THAN ('2021-07-01')"; AlterTableStmt alterTableStmt2 = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL2, ctx); AddPartitionClause addPartitionClause2 = (AddPartitionClause) alterTableStmt2.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists3", addPartitionClause2); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_exists3", addPartitionClause2); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition_exists3"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition_exists3"); Assert.assertEquals(2, ((OlapTable) table).getPartitions().size()); @@ -1771,14 +1773,14 @@ public void testCatalogAddPartitionsShouldThrowError() throws Exception { public void testRenameDb() throws Exception { String createUserSql = "CREATE USER 'testuser' IDENTIFIED BY ''"; CreateUserStmt createUserStmt = - (CreateUserStmt) UtFrameUtils.parseStmtWithNewParser(createUserSql, starRocksAssert.getCtx()); + (CreateUserStmt) UtFrameUtils.parseStmtWithNewParser(createUserSql, starRocksAssert.getCtx()); AuthenticationMgr authenticationManager = - starRocksAssert.getCtx().getGlobalStateMgr().getAuthenticationMgr(); + starRocksAssert.getCtx().getGlobalStateMgr().getAuthenticationMgr(); authenticationManager.createUser(createUserStmt); String sql = "grant ALTER on database test to testuser"; DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(sql, starRocksAssert.getCtx()), - starRocksAssert.getCtx()); + starRocksAssert.getCtx()); UserIdentity testUser = new UserIdentity("testuser", "%"); testUser.analyze(); @@ -1786,13 +1788,13 @@ public void testRenameDb() throws Exception { starRocksAssert.getCtx().setQualifiedUser("testuser"); starRocksAssert.getCtx().setCurrentUserIdentity(testUser); starRocksAssert.getCtx().setCurrentRoleIds( - GlobalStateMgr.getCurrentState().getAuthorizationMgr().getRoleIdsByUser(testUser)); + GlobalStateMgr.getCurrentState().getAuthorizationMgr().getRoleIdsByUser(testUser)); starRocksAssert.getCtx().setRemoteIP("%"); starRocksAssert.withDatabase("test_to_rename"); String renameDb = "alter database test_to_rename rename test_to_rename_2"; AlterDatabaseRenameStatement renameDbStmt = - (AlterDatabaseRenameStatement) UtFrameUtils.parseStmtWithNewParser(renameDb, starRocksAssert.getCtx()); + (AlterDatabaseRenameStatement) UtFrameUtils.parseStmtWithNewParser(renameDb, starRocksAssert.getCtx()); DDLStmtExecutor.execute(renameDbStmt, starRocksAssert.getCtx()); } @@ -1800,38 +1802,38 @@ public void testRenameDb() throws Exception { public void testAddMultiItemListPartition() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10) not null,\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (dt,province) (\n" + - " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\")),\n" + - " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\")) \n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (dt,province) (\n" + + " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\")),\n" + + " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\")) \n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List values = Lists.newArrayList("2022-04-01", "shandong"); List> multiValues = Lists.newArrayList(); multiValues.add(values); PartitionDesc partitionDesc = new MultiItemListPartitionDesc(false, "p3", multiValues, new HashMap<>()); AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDesc, null, new HashMap<>(), false); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_partition"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_partition"); AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(table); analyzer.analyze(Util.getOrCreateConnectContext(), addPartitionClause); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); ListPartitionInfo partitionInfo = (ListPartitionInfo) table.getPartitionInfo(); Map>> idToValues = partitionInfo.getIdToMultiValues(); @@ -1850,15 +1852,15 @@ public void testAddMultiItemListPartition() throws Exception { public void testModifyPartitionBucket() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE modify_bucket (\n" + - " chuangyi varchar(65533) NULL COMMENT \"创意\",\n" + - " guanggao varchar(65533) NULL COMMENT \"广告\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(chuangyi, guanggao)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(chuangyi, guanggao) BUCKETS 3\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");"; + " chuangyi varchar(65533) NULL COMMENT \"创意\",\n" + + " guanggao varchar(65533) NULL COMMENT \"广告\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(chuangyi, guanggao)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(chuangyi, guanggao) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); String stmt = "alter table modify_bucket set (\"dynamic_partition.buckets\" = \"10\");\n"; @@ -1870,34 +1872,35 @@ public void testModifyPartitionBucket() throws Exception { public void testAddSingleItemListPartition() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (province) (\n" + - " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + - " PARTITION p2 VALUES IN (\"guangdong\") \n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + + " PARTITION p2 VALUES IN (\"guangdong\") \n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List values = Lists.newArrayList("shanxi", "shanghai"); PartitionDesc partitionDesc = new SingleItemListPartitionDesc(false, "p3", values, new HashMap<>()); AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDesc, null, new HashMap<>(), false); - OlapTable table = (OlapTable) db.getTable("test_partition"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition"); AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(table); analyzer.analyze(Util.getOrCreateConnectContext(), addPartitionClause); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition", addPartitionClause); ListPartitionInfo partitionInfo = (ListPartitionInfo) table.getPartitionInfo(); Map> idToValues = partitionInfo.getIdToValues(); @@ -1915,25 +1918,26 @@ public void testAddSingleItemListPartition() throws Exception { public void testSingleItemPartitionPersistInfo() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (province) (\n" + - " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") \n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") \n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_partition"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition"); ListPartitionInfo partitionInfo = (ListPartitionInfo) table.getPartitionInfo(); long dbId = db.getId(); @@ -1946,8 +1950,8 @@ public void testSingleItemPartitionPersistInfo() throws Exception { boolean isInMemory = partitionInfo.getIsInMemory(partitionId); boolean isTempPartition = false; ListPartitionPersistInfo partitionPersistInfoOut = new ListPartitionPersistInfo(dbId, tableId, partition, - dataProperty, replicationNum, isInMemory, isTempPartition, values, new ArrayList<>(), - partitionInfo.getDataCacheInfo(partitionId)); + dataProperty, replicationNum, isInMemory, isTempPartition, values, new ArrayList<>(), + partitionInfo.getDataCacheInfo(partitionId)); // write log File file = new File("./test_serial.log"); @@ -1992,25 +1996,26 @@ public void testSingleItemPartitionPersistInfo() throws Exception { public void testMultiItemPartitionPersistInfo() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10) not null,\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (dt , province) (\n" + - " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (dt , province) (\n" + + " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_partition"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition"); ListPartitionInfo partitionInfo = (ListPartitionInfo) table.getPartitionInfo(); long dbId = db.getId(); @@ -2023,8 +2028,8 @@ public void testMultiItemPartitionPersistInfo() throws Exception { boolean isInMemory = partitionInfo.getIsInMemory(partitionId); boolean isTempPartition = false; ListPartitionPersistInfo partitionPersistInfoOut = new ListPartitionPersistInfo(dbId, tableId, partition, - dataProperty, replicationNum, isInMemory, isTempPartition, new ArrayList<>(), multiValues, - partitionInfo.getDataCacheInfo(partitionId)); + dataProperty, replicationNum, isInMemory, isTempPartition, new ArrayList<>(), multiValues, + partitionInfo.getDataCacheInfo(partitionId)); // write log File file = new File("./test_serial.log"); @@ -2073,61 +2078,61 @@ public void testMultiItemPartitionPersistInfo() throws Exception { public void testAddSingleListPartitionSamePartitionNameShouldThrowError() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition_1 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (province) (\n" + - " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + - " PARTITION p2 VALUES IN (\"guangdong\") \n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + + " PARTITION p2 VALUES IN (\"guangdong\") \n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List values = Lists.newArrayList("shanxi", "heilongjiang"); PartitionDesc partitionDesc = new SingleItemListPartitionDesc(false, "p1", values, new HashMap<>()); AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDesc, null, new HashMap<>(), false); - Table table = db.getTable("test_partition_1"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition_1"); AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(table); analyzer.analyze(Util.getOrCreateConnectContext(), addPartitionClause); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_1", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_1", addPartitionClause); } @Test(expected = SemanticException.class) public void testAddMultiListPartitionSamePartitionNameShouldThrowError() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition_2 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10) not null,\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (dt,province) (\n" + - " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\")),\n" + - " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\")) \n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (dt,province) (\n" + + " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\")),\n" + + " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\")) \n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List values1 = Lists.newArrayList("2022-04-01", "beijing"); List values2 = Lists.newArrayList("2022-04-01", "chongqing"); @@ -2137,41 +2142,41 @@ public void testAddMultiListPartitionSamePartitionNameShouldThrowError() throws PartitionDesc partitionDesc = new MultiItemListPartitionDesc(false, "p1", multiValues, new HashMap<>()); AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDesc, null, new HashMap<>(), false); - Table table = db.getTable("test_partition_2"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition_2"); AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(table); analyzer.analyze(Util.getOrCreateConnectContext(), addPartitionClause); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_2", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_partition_2", addPartitionClause); } @Test(expected = SemanticException.class) public void testAddSingleListPartitionSamePartitionValueShouldThrowError() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition_3 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (province) (\n" + - " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + - " PARTITION p2 VALUES IN (\"guangdong\") \n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + + " PARTITION p2 VALUES IN (\"guangdong\") \n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List values = Lists.newArrayList("beijing", "chongqing"); PartitionDesc partitionDesc = new SingleItemListPartitionDesc(false, "p3", values, new HashMap<>()); AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDesc, null, new HashMap<>(), false); - Table table = db.getTable("test_partition_3"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition_3"); AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(table); analyzer.analyze(Util.getOrCreateConnectContext(), addPartitionClause); } @@ -2180,25 +2185,25 @@ public void testAddSingleListPartitionSamePartitionValueShouldThrowError() throw public void testAddMultiItemListPartitionSamePartitionValueShouldThrowError() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.test_partition_4 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10) not null,\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "ENGINE=olap\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (dt, province) (\n" + - " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\")),\n" + - " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\")) \n" + - ")\n" + - "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (dt, province) (\n" + + " PARTITION p1 VALUES IN ((\"2022-04-01\", \"beijing\"),(\"2022-04-01\", \"chongqing\")),\n" + + " PARTITION p2 VALUES IN ((\"2022-04-01\", \"shanghai\")) \n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 10\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List values = Lists.newArrayList("2022-04-01", "shanghai"); List> multiValues = Lists.newArrayList(); @@ -2206,7 +2211,7 @@ public void testAddMultiItemListPartitionSamePartitionValueShouldThrowError() th PartitionDesc partitionDesc = new MultiItemListPartitionDesc(false, "p3", multiValues, new HashMap<>()); AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDesc, null, new HashMap<>(), false); - Table table = db.getTable("test_partition_4"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_partition_4"); AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(table); analyzer.analyze(Util.getOrCreateConnectContext(), addPartitionClause); } @@ -2214,16 +2219,16 @@ public void testAddMultiItemListPartitionSamePartitionValueShouldThrowError() th @Test public void testCatalogAddColumn() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl1\n" + - "(\n" + - " k1 date,\n" + - " v1 int \n" + - ")\n" + - "DUPLICATE KEY(`k1`)" + - "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("tbl1"); + .withTable("CREATE TABLE test.tbl1\n" + + "(\n" + + " k1 date,\n" + + " v1 int \n" + + ")\n" + + "DUPLICATE KEY(`k1`)" + + "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); String stmt = "alter table test.tbl1 add column k2 int"; alterTableWithNewParser(stmt, false); @@ -2245,7 +2250,7 @@ public void testCatalogAddColumn() throws Exception { @Test public void testCatalogAddColumns() throws Exception { String stmt = "alter table test.tbl1 add column (`col1` int(11) not null default \"0\" comment \"\", " - + "`col2` int(11) not null default \"0\" comment \"\") in `testTable`;"; + + "`col2` int(11) not null default \"0\" comment \"\") in `testTable`;"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); AddColumnsClause clause = (AddColumnsClause) alterTableStmt.getAlterClauseList().get(0); Assert.assertEquals(2, clause.getColumns().size()); @@ -2253,7 +2258,7 @@ public void testCatalogAddColumns() throws Exception { Assert.assertEquals("testTable", clause.getRollupName()); stmt = "alter table test.tbl1 add column (`col1` int(11) not null default \"0\" comment \"\", " - + "`col2` int(11) not null default \"0\" comment \"\");"; + + "`col2` int(11) not null default \"0\" comment \"\");"; alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); clause = (AddColumnsClause) alterTableStmt.getAlterClauseList().get(0); Assert.assertEquals(null, clause.getRollupName()); @@ -2264,35 +2269,35 @@ public void testCreateTemporaryPartitionInBatch() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); starRocksAssert.withDatabase("test2"); String createSQL = "CREATE TABLE test2.site_access(\n" + - " event_day datetime,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day)(\n" + - " START (\"2023-03-27\") END (\"2023-03-30\") EVERY (INTERVAL 1 day)\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\n" + - " \"replication_num\" = \"1\"\n" + - ");"; + " event_day datetime,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day)(\n" + + " START (\"2023-03-27\") END (\"2023-03-30\") EVERY (INTERVAL 1 day)\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\n" + + " \"replication_num\" = \"1\"\n" + + ");"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test2"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test2"); String sql = "alter table test2.site_access add TEMPORARY partitions " + - "START (\"2023-03-27\") END (\"2023-03-30\") EVERY (INTERVAL 1 day);"; + "START (\"2023-03-27\") END (\"2023-03-30\") EVERY (INTERVAL 1 day);"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, starRocksAssert.getCtx()); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "site_access", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "site_access", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test2") - .getTable("site_access"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test2") + .getTable("site_access"); OlapTable olapTable = (OlapTable) table; PartitionInfo partitionInfo = olapTable.getPartitionInfo(); RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; @@ -2303,14 +2308,14 @@ public void testCreateTemporaryPartitionInBatch() throws Exception { @Test public void testCatalogDropColumn() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.tbl1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String stmt = "alter table test.tbl1 drop column k2 from `testRollup`"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); DropColumnClause clause = (DropColumnClause) alterTableStmt.getAlterClauseList().get(0); @@ -2411,9 +2416,9 @@ public Table getTable(String tableName) { TruncatePartitionClause clause = new TruncatePartitionClause(partitionNames); cList.add(clause); AlterJobMgr alter = new AlterJobMgr( - new SchemaChangeHandler(), - new MaterializedViewHandler(), - new SystemHandler()); + new SchemaChangeHandler(), + new MaterializedViewHandler(), + new SystemHandler()); TableName tableName = new TableName("test_db", "test_table"); AlterTableStmt stmt = new AlterTableStmt(tableName, cList); DDLStmtExecutor.execute(stmt, starRocksAssert.getCtx()); @@ -2439,7 +2444,7 @@ public void testAutoPartitionTableUnsupported() throws Exception { public void testAutoPartitionTableUnsupported2() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE site_access_time_slice\n" + - "ADD PARTITIONS START (\"2022-05-01\") END (\"2022-05-03\") EVERY (INTERVAL 1 day)"; + "ADD PARTITIONS START (\"2022-05-01\") END (\"2022-05-03\") EVERY (INTERVAL 1 day)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterTable(ctx, alterTableStmt); } @@ -2448,7 +2453,7 @@ public void testAutoPartitionTableUnsupported2() throws Exception { public void testAutoPartitionTableUnsupported3() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE site_access_date_trunc\n" + - "ADD PARTITIONS START (\"2022-05-01\") END (\"2022-05-03\") EVERY (INTERVAL 2 day)"; + "ADD PARTITIONS START (\"2022-05-01\") END (\"2022-05-03\") EVERY (INTERVAL 2 day)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterTable(ctx, alterTableStmt); } @@ -2456,43 +2461,44 @@ public void testAutoPartitionTableUnsupported3() throws Exception { @Test public void testAlterMvWithResourceGroup() throws Exception { starRocksAssert.executeResourceGroupDdlSql("create resource group if not exists mv_rg" + - " with (" + - " 'cpu_core_limit' = '10'," + - " 'mem_limit' = '20%'," + - " 'concurrency_limit' = '11'," + - " 'type' = 'mv'" + - " );"); + " with (" + + " 'cpu_core_limit' = '10'," + + " 'mem_limit' = '20%'," + + " 'concurrency_limit' = '11'," + + " 'type' = 'mv'" + + " );"); starRocksAssert.useDatabase("test") - .withMaterializedView("CREATE MATERIALIZED VIEW `mv2` (a comment \"a1\", b comment \"b2\", c)\n" + - "COMMENT \"MATERIALIZED_VIEW\"\n" + - "DISTRIBUTED BY HASH(a) BUCKETS 12\n" + - "REFRESH ASYNC\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"resource_group\" = \"mv_rg\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT k1, k2, v1 from test.tbl1"); - MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getDb("test").getTable("mv2"); + .withMaterializedView("CREATE MATERIALIZED VIEW `mv2` (a comment \"a1\", b comment \"b2\", c)\n" + + "COMMENT \"MATERIALIZED_VIEW\"\n" + + "DISTRIBUTED BY HASH(a) BUCKETS 12\n" + + "REFRESH ASYNC\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"resource_group\" = \"mv_rg\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT k1, k2, v1 from test.tbl1"); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("mv2"); Assert.assertEquals("mv_rg", mv.getTableProperty().getResourceGroup()); ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER MATERIALIZED VIEW mv2\n" + - "set (\"resource_group\" =\"\" )"; + "set (\"resource_group\" =\"\" )"; AlterMaterializedViewStmt alterTableStmt = (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterMaterializedView(alterTableStmt); Assert.assertEquals("", mv.getTableProperty().getResourceGroup()); sql = "ALTER MATERIALIZED VIEW mv2\n" + - "set (\"resource_group\" =\"not_exist_rg\" )"; + "set (\"resource_group\" =\"not_exist_rg\" )"; AlterMaterializedViewStmt alterTableStmt2 = - (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); Assert.assertThrows("resource_group not_exist_rg does not exist.", - SemanticException.class, - () -> GlobalStateMgr.getCurrentState().getLocalMetastore().alterMaterializedView(alterTableStmt2)); + SemanticException.class, + () -> GlobalStateMgr.getCurrentState().getLocalMetastore().alterMaterializedView(alterTableStmt2)); sql = "ALTER MATERIALIZED VIEW mv2\n" + - "set (\"resource_group\" =\"mv_rg\" )"; + "set (\"resource_group\" =\"mv_rg\" )"; AlterMaterializedViewStmt alterTableStmt3 = - (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterMaterializedView(alterTableStmt3); Assert.assertEquals("mv_rg", mv.getTableProperty().getResourceGroup()); @@ -2504,7 +2510,7 @@ public Warehouse getWarehouse(String warehouseName) { }; sql = "ALTER MATERIALIZED VIEW mv2 set (\"warehouse\" = \"w1\")"; AlterMaterializedViewStmt alterTableStmt4 = - (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().alterMaterializedView(alterTableStmt4); Assert.assertEquals(1L, mv.getWarehouseId()); } @@ -2512,19 +2518,19 @@ public Warehouse getWarehouse(String warehouseName) { @Test(expected = ErrorReportException.class) public void testAlterListPartitionUseBatchBuildPartition() throws Exception { starRocksAssert.useDatabase("test").withTable("CREATE TABLE t2 (\n" + - " dt datetime not null,\n" + - " user_id bigint not null,\n" + - " recharge_money decimal(32,2) not null, \n" + - " province varchar(20) not null,\n" + - " id varchar(20) not null\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(dt)\n" + - "PARTITION BY (dt)\n" + - "DISTRIBUTED BY HASH(`dt`) BUCKETS 10 \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\"\n" + - ");"); + " dt datetime not null,\n" + + " user_id bigint not null,\n" + + " recharge_money decimal(32,2) not null, \n" + + " province varchar(20) not null,\n" + + " id varchar(20) not null\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(dt)\n" + + "PARTITION BY (dt)\n" + + "DISTRIBUTED BY HASH(`dt`) BUCKETS 10 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String sql = "ALTER TABLE t2 ADD PARTITIONS START (\"2021-01-04\") END (\"2021-01-06\") EVERY (INTERVAL 1 DAY);"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); @@ -2538,25 +2544,25 @@ public void testAlterForeignKey() throws Exception { { // inner table starRocksAssert.useDatabase("test").withMaterializedView("create materialized view if not exists `fk_mv_1` " + - "refresh manual " + - "as " + - "select t1.event_day, t1.site_id, t2.user_name " + - "from site_access_date_trunc t1 join site_access_time_slice t2 " + - "on t1.site_id = t2.site_id"); + "refresh manual " + + "as " + + "select t1.event_day, t1.site_id, t2.user_name " + + "from site_access_date_trunc t1 join site_access_time_slice t2 " + + "on t1.site_id = t2.site_id"); connectContext.executeSql("alter materialized view fk_mv_1 set " + - "( 'unique_constraints'='site_access_date_trunc.site_id'); "); + "( 'unique_constraints'='site_access_date_trunc.site_id'); "); connectContext.executeSql("alter materialized view fk_mv_1 set " + - "( 'foreign_key_constraints'='site_access_time_slice(site_id)" + - " REFERENCES site_access_date_trunc(site_id)'); "); + "( 'foreign_key_constraints'='site_access_time_slice(site_id)" + + " REFERENCES site_access_date_trunc(site_id)'); "); while (true) { ModifyTablePropertyOperationLog modifyMvLog = - (ModifyTablePropertyOperationLog) UtFrameUtils.PseudoJournalReplayer. - replayNextJournal(OperationType.OP_ALTER_MATERIALIZED_VIEW_PROPERTIES); + (ModifyTablePropertyOperationLog) UtFrameUtils.PseudoJournalReplayer. + replayNextJournal(OperationType.OP_ALTER_MATERIALIZED_VIEW_PROPERTIES); Assert.assertNotNull(modifyMvLog); if (modifyMvLog.getProperties().containsKey("foreign_key_constraints")) { Assert.assertEquals("default_catalog.10001.10133(site_id) " + - "REFERENCES default_catalog.10001.10118(site_id)", - modifyMvLog.getProperties().get("foreign_key_constraints")); + "REFERENCES default_catalog.10001.10118(site_id)", + modifyMvLog.getProperties().get("foreign_key_constraints")); break; } } @@ -2565,24 +2571,24 @@ public void testAlterForeignKey() throws Exception { { // external table starRocksAssert.withMaterializedView("create materialized view if not exists `fk_mv_2` " + - "refresh manual " + - "as " + - "select t1.l_orderkey, t1.l_partkey, t2.o_totalprice " + - "from hive0.tpch.lineitem t1 join hive0.tpch.orders t2 " + - "on t1.l_orderkey = t2.o_orderkey"); + "refresh manual " + + "as " + + "select t1.l_orderkey, t1.l_partkey, t2.o_totalprice " + + "from hive0.tpch.lineitem t1 join hive0.tpch.orders t2 " + + "on t1.l_orderkey = t2.o_orderkey"); connectContext.executeSql("alter materialized view fk_mv_2 set " + - "( 'unique_constraints'='hive0.tpch.orders.o_orderkey'); "); + "( 'unique_constraints'='hive0.tpch.orders.o_orderkey'); "); connectContext.executeSql("alter materialized view fk_mv_2 set " + - "( 'foreign_key_constraints'='hive0.tpch.lineitem(l_orderkey) " + - "REFERENCES hive0.tpch.orders(o_orderkey)'); "); + "( 'foreign_key_constraints'='hive0.tpch.lineitem(l_orderkey) " + + "REFERENCES hive0.tpch.orders(o_orderkey)'); "); while (true) { ModifyTablePropertyOperationLog modifyMvLog = - (ModifyTablePropertyOperationLog) UtFrameUtils.PseudoJournalReplayer. - replayNextJournal(OperationType.OP_ALTER_MATERIALIZED_VIEW_PROPERTIES); + (ModifyTablePropertyOperationLog) UtFrameUtils.PseudoJournalReplayer. + replayNextJournal(OperationType.OP_ALTER_MATERIALIZED_VIEW_PROPERTIES); Assert.assertNotNull(modifyMvLog); if (modifyMvLog.getProperties().containsKey("foreign_key_constraints")) { Assert.assertEquals("hive0.tpch.lineitem:0(l_orderkey) REFERENCES hive0.tpch.orders:0(o_orderkey)", - modifyMvLog.getProperties().get("foreign_key_constraints")); + modifyMvLog.getProperties().get("foreign_key_constraints")); break; } } diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java index c020c02f4c412..455a5a9cd2322 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/BatchRollupJobTest.java @@ -92,9 +92,9 @@ public void testBatchRollup() throws Exception { Map alterJobs = GlobalStateMgr.getCurrentState().getRollupHandler().getAlterJobsV2(); Assert.assertEquals(3, alterJobs.size()); - Database db = GlobalStateMgr.getCurrentState().getDb("db1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1"); Assert.assertNotNull(db); - OlapTable tbl = (OlapTable) db.getTable("tbl1"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Assert.assertNotNull(tbl); // 3 rollup jobs may be finished in the loop, so only check the final state at last. @@ -138,9 +138,9 @@ public void testCancelBatchRollup() throws Exception { Assert.assertEquals(3, alterJobs.size()); List jobIds = Lists.newArrayList(alterJobs.keySet()); - Database db = GlobalStateMgr.getCurrentState().getDb("db1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1"); Assert.assertNotNull(db); - OlapTable tbl = (OlapTable) db.getTable("tbl2"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl2"); Assert.assertNotNull(tbl); Assert.assertEquals(OlapTableState.ROLLUP, tbl.getState()); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/CompactionHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/CompactionHandlerTest.java index a49e5d6735764..f2304f9533597 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/CompactionHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/CompactionHandlerTest.java @@ -52,25 +52,26 @@ public static void beforeClass() throws Exception { connectContext = UtFrameUtils.createDefaultCtx(); starRocksAssert = new StarRocksAssert(connectContext); starRocksAssert.withDatabase(GlobalStateMgrTestUtil.testDb1) - .useDatabase(GlobalStateMgrTestUtil.testDb1); + .useDatabase(GlobalStateMgrTestUtil.testDb1); starRocksAssert.withTable("CREATE TABLE testTable1\n" + - "(\n" + - " v1 date,\n" + - " v2 int,\n" + - " v3 int\n" + - ")\n" + - "DUPLICATE KEY(`v1`)\n" + - "PARTITION BY RANGE(v1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(v1) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); - - db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + "(\n" + + " v1 date,\n" + + " v2 int,\n" + + " v3 int\n" + + ")\n" + + "DUPLICATE KEY(`v1`)\n" + + "PARTITION BY RANGE(v1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(v1) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); + + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); } @After diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterDataCachePartitionDurationTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterDataCachePartitionDurationTest.java index 51cf2c8caacee..733f17281e57f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterDataCachePartitionDurationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterDataCachePartitionDurationTest.java @@ -50,7 +50,7 @@ import com.starrocks.persist.ModifyTablePropertyOperationLog; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; -import com.starrocks.server.MetadataMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.sql.ast.AlterTableStmt; import com.starrocks.sql.ast.ModifyTablePropertiesClause; import com.starrocks.thrift.TStorageMedium; @@ -178,14 +178,14 @@ public void testModifyDataCachePartitionDurationAbourtMonths() throws Exception properties.put(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION, "7 months"); ModifyTablePropertiesClause modify = new ModifyTablePropertiesClause(properties); - new MockUp() { + new MockUp() { @Mock - public Database getDb(String catalogName, String dbName) { + public Database getDb(String dbName) { return db; } @Mock - public Table getTable(String catalogName, String dbName, String tblName) { + public Table getTable(String dbName, String tblName) { return table; } }; @@ -213,14 +213,14 @@ public void testModifyDataCachePartitionDurationAbourtDays() throws Exception { properties.put(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION, "2 days"); ModifyTablePropertiesClause modify = new ModifyTablePropertiesClause(properties); - new MockUp() { + new MockUp() { @Mock - public Database getDb(String catalogName, String dbName) { + public Database getDb(String dbName) { return db; } @Mock - public Table getTable(String catalogName, String dbName, String tblName) { + public Table getTable(String dbName, String tblName) { return table; } }; @@ -243,14 +243,14 @@ public void testModifyDataCachePartitionDurationAboutHours() throws Exception { Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION, "4 hours"); ModifyTablePropertiesClause modify = new ModifyTablePropertiesClause(properties); - new MockUp() { + new MockUp() { @Mock - public Database getDb(String catalogName, String dbName) { + public Database getDb(String dbName) { return db; } @Mock - public Table getTable(String catalogName, String dbName, String tblName) { + public Table getTable(String dbName, String tblName) { return table; } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java index f1d9bddfae5eb..0b225ef82b230 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAlterMetaJobTest.java @@ -78,11 +78,11 @@ public void setUp() throws Exception { db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB_NAME); table = createTable(connectContext, - "CREATE TABLE t0(c0 INT) PRIMARY KEY(c0) DISTRIBUTED BY HASH(c0) BUCKETS 1 " + - "PROPERTIES('enable_persistent_index'='false')"); + "CREATE TABLE t0(c0 INT) PRIMARY KEY(c0) DISTRIBUTED BY HASH(c0) BUCKETS 1 " + + "PROPERTIES('enable_persistent_index'='false')"); Assert.assertFalse(table.enablePersistentIndex()); job = new LakeTableAlterMetaJob(GlobalStateMgr.getCurrentState().getNextId(), db.getId(), table.getId(), - table.getName(), 60 * 1000, TTabletMetaType.ENABLE_PERSISTENT_INDEX, true); + table.getName(), 60 * 1000, TTabletMetaType.ENABLE_PERSISTENT_INDEX, true); } @After @@ -97,8 +97,9 @@ public void tearDown() throws DdlException, MetaNotFoundException { private static LakeTable createTable(ConnectContext connectContext, String sql) throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().createTable(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb(createTableStmt.getDbName()); - return (LakeTable) db.getTable(createTableStmt.getTableName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createTableStmt.getDbName()); + return (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), createTableStmt.getTableName()); } @Test @@ -121,7 +122,7 @@ public void testUpdatePartitonMetaFailed() { @Mock public Warehouse getWarehouse(long warehouseId) { return new DefaultWarehouse(WarehouseManager.DEFAULT_WAREHOUSE_ID, - WarehouseManager.DEFAULT_WAREHOUSE_NAME); + WarehouseManager.DEFAULT_WAREHOUSE_NAME); } @Mock @@ -208,8 +209,8 @@ public void testReplay() { Assert.assertEquals(AlterJobV2.JobState.FINISHED, job.getJobState()); LakeTableAlterMetaJob replayAlterMetaJob = new LakeTableAlterMetaJob(job.jobId, - job.dbId, job.tableId, job.tableName, - job.timeoutMs, TTabletMetaType.ENABLE_PERSISTENT_INDEX, true); + job.dbId, job.tableId, job.tableName, + job.timeoutMs, TTabletMetaType.ENABLE_PERSISTENT_INDEX, true); Table partitionIndexMap = job.getPartitionIndexMap(); Map commitVersionMap = job.getCommitVersionMap(); @@ -249,7 +250,7 @@ public void testUpdateTabletMetaInfoTaskToThrift() throws AlterCancelException { tabletSet.add(1L); MarkedCountDownLatch> latch = new MarkedCountDownLatch<>(1); TabletMetadataUpdateAgentTask task = TabletMetadataUpdateAgentTaskFactory.createEnablePersistentIndexUpdateTask( - backend, tabletSet, true); + backend, tabletSet, true); task.setLatch(latch); task.setTxnId(txnId); TUpdateTabletMetaInfoReq result = task.toThrift(); @@ -264,6 +265,6 @@ public void testSetPropertyNotSupport() { ModifyTablePropertiesClause modify = new ModifyTablePropertiesClause(properties); SchemaChangeHandler schemaChangeHandler = new SchemaChangeHandler(); Assertions.assertThrows(DdlException.class, - () -> schemaChangeHandler.createAlterMetaJob(modify, db, table)); + () -> schemaChangeHandler.createAlterMetaJob(modify, db, table)); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java index d7d2b9d56bcce..70d7fcae77fad 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableAsyncFastSchemaChangeJobTest.java @@ -57,8 +57,9 @@ public static void setUp() throws Exception { private static LakeTable createTable(ConnectContext connectContext, String sql) throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().createTable(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb(createTableStmt.getDbName()); - LakeTable table = (LakeTable) db.getTable(createTableStmt.getTableName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createTableStmt.getDbName()); + LakeTable table = (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), createTableStmt.getTableName()); table.addRelatedMaterializedView(new MvId(db.getId(), GlobalStateMgr.getCurrentState().getNextId())); return table; } @@ -95,7 +96,7 @@ private AlterJobV2 mustAlterTable(Table table, String sql) throws Exception { @Test public void test() throws Exception { LakeTable table = createTable(connectContext, "CREATE TABLE t0(c0 INT) DUPLICATE KEY(c0) DISTRIBUTED BY HASH(c0) " + - "BUCKETS 2 PROPERTIES('fast_schema_evolution'='true')"); + "BUCKETS 2 PROPERTIES('fast_schema_evolution'='true')"); long oldSchemaId = table.getIndexIdToMeta().get(table.getBaseIndexId()).getSchemaId(); { @@ -136,7 +137,7 @@ public void test() throws Exception { @Test public void testGetInfo() throws Exception { LakeTable table = createTable(connectContext, "CREATE TABLE t1(c0 INT) DUPLICATE KEY(c0) DISTRIBUTED BY HASH(c0) " + - "BUCKETS 2 PROPERTIES('fast_schema_evolution'='true')"); + "BUCKETS 2 PROPERTIES('fast_schema_evolution'='true')"); AlterJobV2 job = mustAlterTable(table, "ALTER TABLE t1 ADD COLUMN c1 BIGINT"); List> infoList = new ArrayList<>(); job.getInfo(infoList); @@ -151,7 +152,7 @@ public void testGetInfo() throws Exception { Assert.assertEquals(table.getBaseIndexId(), info.get(5)); Assert.assertEquals(table.getBaseIndexId(), info.get(6)); Assert.assertEquals(String.format("%d:0", table.getIndexIdToMeta().get(table.getBaseIndexId()).getSchemaVersion()), - info.get(7)); + info.get(7)); Assert.assertEquals(job.getTransactionId().get(), info.get(8)); Assert.assertEquals(job.getJobState().name(), info.get(9)); Assert.assertEquals(job.errMsg, info.get(10)); @@ -161,14 +162,14 @@ public void testGetInfo() throws Exception { @Test public void testReplay() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(DB_NAME); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB_NAME); LakeTable table = createTable(connectContext, "CREATE TABLE t3(c0 INT) DUPLICATE KEY(c0) DISTRIBUTED BY HASH(c0) " + - "BUCKETS 2 PROPERTIES('fast_schema_evolution'='true')"); + "BUCKETS 2 PROPERTIES('fast_schema_evolution'='true')"); SchemaChangeHandler handler = GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler(); AlterTableStmt stmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser("ALTER TABLE t3 ADD COLUMN c1 INT", - connectContext); + connectContext); LakeTableAsyncFastSchemaChangeJob job = (LakeTableAsyncFastSchemaChangeJob) - handler.analyzeAndCreateJob(stmt.getAlterClauseList(), db, table); + handler.analyzeAndCreateJob(stmt.getAlterClauseList(), db, table); Assert.assertNotNull(job); Assert.assertEquals(AlterJobV2.JobState.PENDING, job.getJobState()); Assert.assertEquals(OlapTable.OlapTableState.NORMAL, table.getState()); @@ -209,7 +210,7 @@ public void testReplay() throws Exception { @Test public void testSortKey() throws Exception { LakeTable table = createTable(connectContext, "CREATE TABLE t_test_sort_key(c0 INT, c1 BIGINT) PRIMARY KEY(c0) " + - "DISTRIBUTED BY HASH(c0) BUCKETS 2 ORDER BY(c1)"); + "DISTRIBUTED BY HASH(c0) BUCKETS 2 ORDER BY(c1)"); long oldSchemaId = table.getIndexIdToMeta().get(table.getBaseIndexId()).getSchemaId(); { @@ -242,7 +243,7 @@ public void testSortKey() throws Exception { @Test public void testSortKeyIndexesChanged() throws Exception { LakeTable table = createTable(connectContext, "CREATE TABLE t_test_sort_key_index_changed" + - "(c0 INT, c1 BIGINT, c2 BIGINT) PRIMARY KEY(c0) DISTRIBUTED BY HASH(c0) BUCKETS 2 ORDER BY(c2)"); + "(c0 INT, c1 BIGINT, c2 BIGINT) PRIMARY KEY(c0) DISTRIBUTED BY HASH(c0) BUCKETS 2 ORDER BY(c2)"); long oldSchemaId = table.getIndexIdToMeta().get(table.getBaseIndexId()).getSchemaId(); { @@ -263,13 +264,13 @@ public void testSortKeyIndexesChanged() throws Exception { @Test public void testModifyColumnType() throws Exception { LakeTable table = createTable(connectContext, "CREATE TABLE t_modify_type" + - "(c0 INT, c1 INT, c2 VARCHAR(5), c3 DATE)" + - "DUPLICATE KEY(c0) DISTRIBUTED BY HASH(c0) " + - "BUCKETS 1 PROPERTIES('fast_schema_evolution'='true')"); + "(c0 INT, c1 INT, c2 VARCHAR(5), c3 DATE)" + + "DUPLICATE KEY(c0) DISTRIBUTED BY HASH(c0) " + + "BUCKETS 1 PROPERTIES('fast_schema_evolution'='true')"); long oldSchemaId = table.getIndexIdToMeta().get(table.getBaseIndexId()).getSchemaId(); { mustAlterTable(table, "ALTER TABLE t_modify_type MODIFY COLUMN c1 BIGINT, MODIFY COLUMN c2 VARCHAR(10)," + - "MODIFY COLUMN c3 DATETIME"); + "MODIFY COLUMN c3 DATETIME"); List columns = table.getBaseSchema(); Assert.assertEquals(4, columns.size()); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java index fcb22ae831cdf..f3455325b8285 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java @@ -76,8 +76,9 @@ public static void setUp() throws Exception { private static LakeTable createTable(ConnectContext connectContext, String sql) throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().createTable(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb(createTableStmt.getDbName()); - return (LakeTable) db.getTable(createTableStmt.getTableName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createTableStmt.getDbName()); + return (LakeTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), createTableStmt.getTableName()); } private static void alterTable(ConnectContext connectContext, String sql) throws Exception { @@ -101,9 +102,9 @@ public void before() throws Exception { CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseStmtWithNewParser(createDbStmtStr, connectContext); GlobalStateMgr.getCurrentState().getMetadata().createDb(createDbStmt.getFullDbName()); connectContext.setDatabase(DB_NAME); - db = GlobalStateMgr.getServingState().getDb(DB_NAME); + db = GlobalStateMgr.getServingState().getLocalMetastore().getDb(DB_NAME); table = createTable(connectContext, "CREATE TABLE t0(c0 INT) duplicate key(c0) distributed by hash(c0) buckets " - + NUM_BUCKETS); + + NUM_BUCKETS); Config.enable_fast_schema_evolution_in_share_data_mode = false; alterTable(connectContext, "ALTER TABLE t0 ADD COLUMN c1 DOUBLE"); schemaChangeJob = getAlterJob(table); @@ -152,7 +153,7 @@ public void writeEditLog(LakeTableSchemaChangeJob job) { @Mock public Warehouse getWarehouse(long warehouseId) { return new DefaultWarehouse(WarehouseManager.DEFAULT_WAREHOUSE_ID, - WarehouseManager.DEFAULT_WAREHOUSE_NAME); + WarehouseManager.DEFAULT_WAREHOUSE_NAME); } @Mock @@ -413,7 +414,7 @@ public void sendAgentTask(AgentBatchTask batchTask) { Assert.assertNotNull(partition); Assert.assertEquals(3, partition.getNextVersion()); List shadowIndexes = - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); + partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); Assert.assertEquals(1, shadowIndexes.size()); // Does not support cancel job in FINISHED_REWRITING state. @@ -435,8 +436,8 @@ public void testPublishVersion() throws AlterCancelException { @Mock public void publishVersion(@NotNull List tablets, TxnInfoPB txnInfo, long baseVersion, long newVersion, long warehouseId) - throws - RpcException { + throws + RpcException { throw new RpcException("publish version failed", "127.0.0.1"); } }; @@ -468,7 +469,7 @@ public void sendAgentTask(AgentBatchTask batchTask) { Assert.assertEquals(AlterJobV2.JobState.FINISHED_REWRITING, schemaChangeJob.getJobState()); List shadowIndexes = - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); + partition.getMaterializedIndices(MaterializedIndex.IndexExtState.SHADOW); Assert.assertEquals(1, shadowIndexes.size()); // The partition's visible version has not catch up with the commit version of this schema change job now. @@ -520,7 +521,7 @@ public void publishVersion(@NotNull List tablets, TxnInfoPB txnInfo, lon Assert.assertEquals(0, shadowIndexes.size()); List normalIndexes = - partition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE); + partition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE); Assert.assertEquals(1, normalIndexes.size()); MaterializedIndex normalIndex = normalIndexes.get(0); @@ -557,7 +558,7 @@ public boolean isPreviousLoadFinished(long dbId, long tableId, long txnId) throw schemaChangeJob.runPendingJob(); }); Assert.assertTrue(exception.getMessage().contains( - "concurrent transaction detected while adding shadow index, please re-run the alter table command")); + "concurrent transaction detected while adding shadow index, please re-run the alter table command")); Assert.assertEquals(AlterJobV2.JobState.PENDING, schemaChangeJob.getJobState()); Assert.assertEquals(10101L, schemaChangeJob.getWatershedTxnId()); @@ -571,14 +572,14 @@ public void testShow() { @Mock public Warehouse getWarehouseAllowNull(long warehouseId) { return new DefaultWarehouse(WarehouseManager.DEFAULT_WAREHOUSE_ID, - WarehouseManager.DEFAULT_WAREHOUSE_NAME); + WarehouseManager.DEFAULT_WAREHOUSE_NAME); } }; SchemaChangeHandler schemaChangeHandler = new SchemaChangeHandler(); LakeTableSchemaChangeJob alterJobV2 = - new LakeTableSchemaChangeJob(12345L, db.getId(), table.getId(), table.getName(), 10); + new LakeTableSchemaChangeJob(12345L, db.getId(), table.getId(), table.getName(), 10); alterJobV2.addIndexSchema(1L, 2L, "a", (short) 1, Lists.newArrayList()); schemaChangeHandler.addAlterJobV2(alterJobV2); @@ -603,7 +604,7 @@ public void testCancelPendingJobWithFlag() throws Exception { schemaChangeJob.setIsCancelling(true); schemaChangeJob.runPendingJob(); schemaChangeJob.setIsCancelling(false); - + schemaChangeJob.setWaitingCreatingReplica(true); schemaChangeJob.cancel(""); schemaChangeJob.setWaitingCreatingReplica(false); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java index 4f2d36dc56730..6a3975c4be2d7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/OnlineOptimizeJobV2Test.java @@ -75,8 +75,9 @@ public void clear() { @Test public void testOptimizeTable() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); @@ -88,8 +89,9 @@ public void testOptimizeTable() throws Exception { @Test public void testOptimizeTableFinish() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); @@ -119,8 +121,9 @@ public void testOptimizeTableFinish() throws Exception { @Test public void testOptimizeTableFailed() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); @@ -157,15 +160,16 @@ public void testOptimizeTableFailed() throws Exception { Assert.assertEquals(JobState.CANCELLED, optimizeJob.getJobState()); OnlineOptimizeJobV2 replayOptimizeJob = new OnlineOptimizeJobV2( - optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); + optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); replayOptimizeJob.replay(optimizeJob); } @Test public void testSchemaChangeWhileTabletNotStable() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); @@ -214,8 +218,9 @@ public void testSerializeOfOptimizeJob() throws IOException { @Test public void testOptimizeReplay() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); @@ -223,7 +228,7 @@ public void testOptimizeReplay() throws Exception { OnlineOptimizeJobV2 optimizeJob = (OnlineOptimizeJobV2) alterJobsV2.values().stream().findAny().get(); OnlineOptimizeJobV2 replayOptimizeJob = new OnlineOptimizeJobV2( - optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); + optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); replayOptimizeJob.replay(optimizeJob); Assert.assertEquals(JobState.PENDING, replayOptimizeJob.getJobState()); @@ -260,8 +265,9 @@ public void testOptimizeReplay() throws Exception { @Test public void testOptimizeReplayPartialSuccess() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable("testTable2"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "testTable2"); String stmt = "alter table testTable2 distributed by hash(v1)"; AlterTableStmt alterStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); @@ -270,9 +276,8 @@ public void testOptimizeReplayPartialSuccess() throws Exception { Assert.assertEquals(1, alterJobsV2.size()); OnlineOptimizeJobV2 optimizeJob = (OnlineOptimizeJobV2) alterJobsV2.values().stream().findAny().get(); - OnlineOptimizeJobV2 replayOptimizeJob = new OnlineOptimizeJobV2( - optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); + optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); replayOptimizeJob.replay(optimizeJob); Assert.assertEquals(JobState.PENDING, replayOptimizeJob.getJobState()); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java index 5ffb4a1d87c14..15d8d060c7482 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/OptimizeJobV2Test.java @@ -147,8 +147,9 @@ public void testOptimizeParser() throws Exception { @Test public void testOptimizeTable() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); @@ -160,8 +161,9 @@ public void testOptimizeTable() throws Exception { @Test public void testOptimizeTableFinish() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); @@ -201,8 +203,9 @@ public void testOptimizeTableFinish() throws Exception { @Test public void testOptimizeTableFailed() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); @@ -239,15 +242,16 @@ public void testOptimizeTableFailed() throws Exception { Assert.assertEquals(JobState.CANCELLED, optimizeJob.getJobState()); OptimizeJobV2 replayOptimizeJob = new OptimizeJobV2( - optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); + optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); replayOptimizeJob.replay(optimizeJob); } @Test public void testSchemaChangeWhileTabletNotStable() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); @@ -296,8 +300,9 @@ public void testSerializeOfOptimizeJob() throws IOException { @Test public void testOptimizeReplay() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable7); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable7); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); @@ -305,7 +310,7 @@ public void testOptimizeReplay() throws Exception { OptimizeJobV2 optimizeJob = (OptimizeJobV2) alterJobsV2.values().stream().findAny().get(); OptimizeJobV2 replayOptimizeJob = new OptimizeJobV2( - optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); + optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); replayOptimizeJob.replay(optimizeJob); Assert.assertEquals(JobState.PENDING, replayOptimizeJob.getJobState()); @@ -342,8 +347,9 @@ public void testOptimizeReplay() throws Exception { @Test public void testOptimizeReplayPartialSuccess() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable("testTable2"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "testTable2"); String stmt = "alter table testTable2 distributed by hash(v1)"; AlterTableStmt alterStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); @@ -352,9 +358,8 @@ public void testOptimizeReplayPartialSuccess() throws Exception { Assert.assertEquals(1, alterJobsV2.size()); OptimizeJobV2 optimizeJob = (OptimizeJobV2) alterJobsV2.values().stream().findAny().get(); - OptimizeJobV2 replayOptimizeJob = new OptimizeJobV2( - optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); + optimizeJob.getJobId(), db.getId(), olapTable.getId(), olapTable.getName(), 1000); replayOptimizeJob.replay(optimizeJob); Assert.assertEquals(JobState.PENDING, replayOptimizeJob.getJobState()); @@ -387,8 +392,9 @@ public void testOptimizeReplayPartialSuccess() throws Exception { @Test public void testOptimizeFailedByVersion() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable("testTable2"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "testTable2"); String stmt = "alter table testTable2 distributed by hash(v1)"; AlterTableStmt alterStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java index 021852f5512b1..fc8e951b894be 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/RollupJobV2Test.java @@ -92,12 +92,12 @@ public class RollupJobV2Test extends DDLTestBase { public void setUp() throws Exception { super.setUp(); clause = new AddRollupClause(GlobalStateMgrTestUtil.testRollupIndex2, Lists.newArrayList("v1"), null, - GlobalStateMgrTestUtil.testTable1, null); + GlobalStateMgrTestUtil.testTable1, null); AlterTableClauseAnalyzer analyzer = new AlterTableClauseAnalyzer(null); analyzer.analyze(null, clause); clause2 = new AddRollupClause(GlobalStateMgrTestUtil.testRollupIndex3, Lists.newArrayList("v1", "v2"), null, - GlobalStateMgrTestUtil.testTable1, null); + GlobalStateMgrTestUtil.testTable1, null); analyzer.analyze(null, clause2); AgentTaskQueue.clearAllTasks(); @@ -114,15 +114,16 @@ public void testRunRollupJobConcurrentLimit() throws UserException { ArrayList alterClauses = new ArrayList<>(); alterClauses.add(clause); alterClauses.add(clause2); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); materializedViewHandler.process(alterClauses, db, olapTable); Map alterJobsV2 = materializedViewHandler.getAlterJobsV2(); materializedViewHandler.runAfterCatalogReady(); assertEquals(Config.max_running_rollup_job_num_per_table, - materializedViewHandler.getTableRunningJobMap().get(olapTable.getId()).size()); + materializedViewHandler.getTableRunningJobMap().get(olapTable.getId()).size()); assertEquals(2, alterJobsV2.size()); assertEquals(OlapTableState.ROLLUP, olapTable.getState()); } @@ -132,8 +133,9 @@ public void testAddSchemaChange() throws UserException { MaterializedViewHandler materializedViewHandler = GlobalStateMgr.getCurrentState().getRollupHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(clause); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); materializedViewHandler.process(alterClauses, db, olapTable); Map alterJobsV2 = materializedViewHandler.getAlterJobsV2(); assertEquals(1, alterJobsV2.size()); @@ -148,8 +150,9 @@ public void testSchemaChange1() throws Exception { // add a rollup job ArrayList alterClauses = new ArrayList<>(); alterClauses.add(clause); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable1); materializedViewHandler.process(alterClauses, db, olapTable); Map alterJobsV2 = materializedViewHandler.getAlterJobsV2(); @@ -191,8 +194,9 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { ArrayList alterClauses = new ArrayList<>(); alterClauses.add(clause); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable1); materializedViewHandler.process(alterClauses, db, olapTable); @@ -245,7 +249,7 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { @Test public void testSerializeOfRollupJob() throws IOException, - AnalysisException { + AnalysisException { Config.enable_materialized_view = true; // prepare file String fileName = "./RollupJobV2Test"; @@ -257,13 +261,13 @@ public void testSerializeOfRollupJob() throws IOException, List columns = Lists.newArrayList(); String mvColumnName = MVUtils.MATERIALIZED_VIEW_NAME_PREFIX + "bitmap_union_" + "c1"; Column column = new Column(mvColumnName, Type.BITMAP, false, AggregateType.BITMAP_UNION, false, - new ColumnDef.DefaultValueDef(true, new StringLiteral("1")), ""); + new ColumnDef.DefaultValueDef(true, new StringLiteral("1")), ""); columns.add(column); RollupJobV2 rollupJobV2 = new RollupJobV2(1, 1, 1, "test", 1, 1, - 1, "test", "rollup", 0, columns, null, 1, 1, - KeysType.AGG_KEYS, keysCount, - new OriginStatement("create materialized view rollup as select bitmap_union(to_bitmap(c1)) from test", - 0), "", false); + 1, "test", "rollup", 0, columns, null, 1, 1, + KeysType.AGG_KEYS, keysCount, + new OriginStatement("create materialized view rollup as select bitmap_union(to_bitmap(c1)) from test", + 0), "", false); // write rollup job rollupJobV2.write(out); @@ -278,7 +282,7 @@ public void testSerializeOfRollupJob() throws IOException, assertEquals(1, resultColumns.size()); Column resultColumn1 = resultColumns.get(0); assertEquals(mvColumnName, - resultColumn1.getName()); + resultColumn1.getName()); Assert.assertTrue(resultColumn1.getDefineExpr() instanceof FunctionCallExpr); FunctionCallExpr resultFunctionCall = (FunctionCallExpr) resultColumn1.getDefineExpr(); assertEquals("to_bitmap", resultFunctionCall.getFnName().getFunction()); @@ -289,8 +293,9 @@ public void testReplayPendingRollupJob() throws Exception { MaterializedViewHandler materializedViewHandler = GlobalStateMgr.getCurrentState().getRollupHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(clause); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); materializedViewHandler.process(alterClauses, db, olapTable); Map alterJobsV2 = materializedViewHandler.getAlterJobsV2(); assertEquals(1, alterJobsV2.size()); @@ -306,8 +311,9 @@ public void testCancelPendingJobWithFlag() throws Exception { // add a rollup job ArrayList alterClauses = new ArrayList<>(); alterClauses.add(clause); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable1); materializedViewHandler.process(alterClauses, db, olapTable); Map alterJobsV2 = materializedViewHandler.getAlterJobsV2(); @@ -318,7 +324,7 @@ public void testCancelPendingJobWithFlag() throws Exception { rollupJob.setIsCancelling(true); rollupJob.runPendingJob(); rollupJob.setIsCancelling(false); - + rollupJob.setWaitingCreatingReplica(true); rollupJob.cancel(""); rollupJob.setWaitingCreatingReplica(false); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java index 3f9907d28a314..d849abd9c1f6c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java @@ -81,38 +81,38 @@ protected void runBeforeAll() throws Exception { //create tables String createAggTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_agg (\n" + "user_id LARGEINT NOT NULL,\n" - + "date DATE NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" - + "last_visit_date DATETIME REPLACE DEFAULT '1970-01-01 00:00:00',\n" + "cost BIGINT SUM DEFAULT '0',\n" - + "max_dwell_time INT MAX DEFAULT '0',\n" + "min_dwell_time INT MIN DEFAULT '99999')\n" - + "AGGREGATE KEY(user_id, date, city, age, sex)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" - + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; + + "date DATE NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" + + "last_visit_date DATETIME REPLACE DEFAULT '1970-01-01 00:00:00',\n" + "cost BIGINT SUM DEFAULT '0',\n" + + "max_dwell_time INT MAX DEFAULT '0',\n" + "min_dwell_time INT MIN DEFAULT '99999')\n" + + "AGGREGATE KEY(user_id, date, city, age, sex)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; createTable(createAggTblStmtStr); String createUniqTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_uniq (\n" + "user_id LARGEINT NOT NULL,\n" - + "username VARCHAR(50) NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" - + "phone LARGEINT,\n" + "address VARCHAR(500),\n" + "register_time DATETIME)\n" - + "UNIQUE KEY(user_id, username)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" - + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; + + "username VARCHAR(50) NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" + + "phone LARGEINT,\n" + "address VARCHAR(500),\n" + "register_time DATETIME)\n" + + "UNIQUE KEY(user_id, username)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; createTable(createUniqTblStmtStr); String createDupTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_dup (\n" + "timestamp DATETIME,\n" - + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" - + "op_time DATETIME)\n" + "DUPLICATE KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" - + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; + + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" + + "op_time DATETIME)\n" + "DUPLICATE KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; createTable(createDupTblStmtStr); String createDupTbl2StmtStr = "CREATE TABLE IF NOT EXISTS test.sc_dup2 (\n" + "timestamp DATETIME,\n" - + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" - + "op_time DATETIME)\n" + "DUPLICATE KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" - + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; + + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" + + "op_time DATETIME)\n" + "DUPLICATE KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; createTable(createDupTbl2StmtStr); String createPKTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_pk (\n" + "timestamp DATETIME,\n" - + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" - + "op_time DATETIME)\n" + "PRIMARY KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" - + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; + + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" + + "op_time DATETIME)\n" + "PRIMARY KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'fast_schema_evolution' = 'true');"; createTable(createPKTblStmtStr); @@ -127,8 +127,9 @@ private void waitAlterJobDone(Map alterJobs) throws Exception LOG.info("alter job {} is done. state: {}", alterJobV2.getJobId(), alterJobV2.getJobState()); Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); - Database db = GlobalStateMgr.getCurrentState().getDb(alterJobV2.getDbId()); - OlapTable tbl = (OlapTable) db.getTable(alterJobV2.getTableId()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(alterJobV2.getDbId()); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), alterJobV2.getTableId()); while (tbl.getState() != OlapTable.OlapTableState.NORMAL) { Thread.sleep(1000); } @@ -139,8 +140,8 @@ private void waitAlterJobDone(Map alterJobs) throws Exception public void testAggAddOrDropColumn() throws Exception { LOG.info("dbName: {}", GlobalStateMgr.getCurrentState().getLocalMetastore().listDbNames()); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("sc_agg"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "sc_agg"); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(tbl.getId()), LockType.READ); try { @@ -246,8 +247,8 @@ public void testUniqAddOrDropColumn() throws Exception { LOG.info("dbName: {}", GlobalStateMgr.getCurrentState().getLocalMetastore().listDbNames()); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("sc_uniq"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "sc_uniq"); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(tbl.getId()), LockType.READ); try { @@ -304,8 +305,8 @@ public void testDupAddOrDropColumn() throws Exception { LOG.info("dbName: {}", GlobalStateMgr.getCurrentState().getLocalMetastore().listDbNames()); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("sc_dup"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "sc_dup"); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(tbl.getId()), LockType.READ); try { @@ -360,8 +361,8 @@ public void testDupAddOrDropColumn() throws Exception { @Test public void testModifyTableAddOrDropColumns() { GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("sc_dup2"); + Database db = globalStateMgr.getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "sc_dup2"); Map alterJobs = globalStateMgr.getSchemaChangeHandler().getAlterJobsV2(); // origin columns @@ -374,25 +375,25 @@ public void testModifyTableAddOrDropColumns() { List newIndexes = tbl.getCopiedIndexes(); Assertions.assertDoesNotThrow( - () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) - .modifyTableAddOrDrop(db, tbl, indexSchemaMap, newIndexes, 100, 100, - indexToNewSchemaId, false)); + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDrop(db, tbl, indexSchemaMap, newIndexes, 100, 100, + indexToNewSchemaId, false)); jobSize++; Assertions.assertEquals(jobSize, alterJobs.size()); Assertions.assertDoesNotThrow( - () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) - .modifyTableAddOrDrop(db, tbl, indexSchemaMap, newIndexes, 101, 101, - indexToNewSchemaId, true)); + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDrop(db, tbl, indexSchemaMap, newIndexes, 101, 101, + indexToNewSchemaId, true)); jobSize++; Assertions.assertEquals(jobSize, alterJobs.size()); OlapTableState beforeState = tbl.getState(); tbl.setState(OlapTableState.ROLLUP); Assertions.assertThrows(DdlException.class, - () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) - .modifyTableAddOrDrop(db, tbl, indexSchemaMap, newIndexes, 102, 102, indexToNewSchemaId, - false)); + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDrop(db, tbl, indexSchemaMap, newIndexes, 102, 102, indexToNewSchemaId, + false)); tbl.setState(beforeState); } @@ -401,8 +402,8 @@ public void testSetPrimaryIndexCacheExpireSec() throws Exception { LOG.info("dbName: {}", GlobalStateMgr.getCurrentState().getLocalMetastore().listDbNames()); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("sc_pk"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "sc_pk"); Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(tbl.getId()), LockType.READ); try { diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java index ab282213f61ac..091b22899e3d7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java @@ -58,7 +58,7 @@ import com.starrocks.common.UserException; import com.starrocks.common.jmockit.Deencapsulation; import com.starrocks.server.GlobalStateMgr; -import com.starrocks.server.MetadataMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.server.RunMode; import com.starrocks.server.WarehouseManager; import com.starrocks.sql.analyzer.DDLTestBase; @@ -118,8 +118,9 @@ public void clear() { @Test public void testAddSchemaChange() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); @@ -131,8 +132,9 @@ public void testAddSchemaChange() throws Exception { @Test public void testSchemaChange1() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); olapTable.setUseFastSchemaEvolution(false); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable1); @@ -185,8 +187,9 @@ public void testSchemaChange1() throws Exception { @Test public void testSchemaChangeWhileTabletNotStable() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); olapTable.setUseFastSchemaEvolution(false); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable1); @@ -254,14 +257,14 @@ public void testModifyDynamicPartitionNormal() throws Exception { OlapTable olapTable = (OlapTable) db.getTable(CatalogMocker.TEST_TBL2_ID); olapTable.setUseFastSchemaEvolution(false); - new MockUp() { + new MockUp() { @Mock - public Database getDb(String catalogName, String dbName) { + public Database getDb(String dbName) { return db; } @Mock - public Table getTable(String catalogName, String dbName, String tblName) { + public Table getTable(String dbName, String tblName) { return olapTable; } }; @@ -322,7 +325,7 @@ public void testModifyDynamicPropertyTrim() throws Exception { } public void modifyDynamicPartitionWithoutTableProperty(String propertyKey, String propertyValue, String expectErrMsg) - throws UserException { + throws UserException { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); ArrayList alterClauses = new ArrayList<>(); Map properties = new HashMap<>(); @@ -332,14 +335,14 @@ public void modifyDynamicPartitionWithoutTableProperty(String propertyKey, Strin Database db = CatalogMocker.mockDb(); OlapTable olapTable = (OlapTable) db.getTable(CatalogMocker.TEST_TBL2_ID); - new MockUp() { + new MockUp() { @Mock - public Database getDb(String catalogName, String dbName) { + public Database getDb(String dbName) { return db; } @Mock - public Table getTable(String catalogName, String dbName, String tblName) { + public Table getTable(String dbName, String tblName) { return olapTable; } }; @@ -353,14 +356,14 @@ public Table getTable(String catalogName, String dbName, String tblName) { @Test public void testModifyDynamicPartitionWithoutTableProperty() throws AlterJobException, UserException { modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.ENABLE, "false", - "not a dynamic partition table"); + "not a dynamic partition table"); modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.TIME_UNIT, "day", - DynamicPartitionProperty.ENABLE); + DynamicPartitionProperty.ENABLE); modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.END, "3", DynamicPartitionProperty.ENABLE); modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.PREFIX, "p", - DynamicPartitionProperty.ENABLE); + DynamicPartitionProperty.ENABLE); modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.BUCKETS, "30", - DynamicPartitionProperty.ENABLE); + DynamicPartitionProperty.ENABLE); } @Test @@ -411,7 +414,7 @@ public RunMode getCurrentRunMode() { @Mock public Warehouse getWarehouse(long warehouseId) { return new DefaultWarehouse(WarehouseManager.DEFAULT_WAREHOUSE_ID, - WarehouseManager.DEFAULT_WAREHOUSE_NAME); + WarehouseManager.DEFAULT_WAREHOUSE_NAME); } @Mock @@ -424,19 +427,19 @@ public List getAllComputeNodeIds(long warehouseId) { AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); ReorderColumnsClause clause = (ReorderColumnsClause) alterTableStmt.getAlterClauseList().get(0); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); AlterJobV2 alterJobV2 = schemaChangeHandler.analyzeAndCreateJob(Lists.newArrayList(clause), db, olapTable); Assert.assertEquals(0L, alterJobV2.warehouseId); - new MockUp() { @Mock public Warehouse getWarehouse(long warehouseId) { return new DefaultWarehouse(WarehouseManager.DEFAULT_WAREHOUSE_ID, - WarehouseManager.DEFAULT_WAREHOUSE_NAME); + WarehouseManager.DEFAULT_WAREHOUSE_NAME); } @Mock @@ -456,8 +459,9 @@ public List getAllComputeNodeIds(long warehouseId) { @Test public void testCancelPendingJobWithFlag() throws Exception { SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); - Database db = GlobalStateMgr.getCurrentState().getDb(GlobalStateMgrTestUtil.testDb1); - OlapTable olapTable = (OlapTable) db.getTable(GlobalStateMgrTestUtil.testTable1); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDb1); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), GlobalStateMgrTestUtil.testTable1); olapTable.setUseFastSchemaEvolution(false); Partition testPartition = olapTable.getPartition(GlobalStateMgrTestUtil.testTable1); diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java b/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java index 384c4d7978db5..0c33c0f430681 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/AccessTestUtil.java @@ -166,25 +166,28 @@ public static GlobalStateMgr fetchBlockCatalog() { Database db = mockDb("testDb"); + /* new Expectations(globalStateMgr) { { - globalStateMgr.getDb("testDb"); + globalStateMgr.getLocalMetastore().getDb("testDb"); minTimes = 0; result = db; - globalStateMgr.getDb("emptyDb"); + globalStateMgr.getLocalMetastore().getDb("emptyDb"); minTimes = 0; result = null; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = new Database(); - globalStateMgr.getDb("emptyCluster"); + globalStateMgr.getLocalMetastore().getDb("emptyCluster"); minTimes = 0; result = null; } }; + + */ return globalStateMgr; } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CTASAutoTabletTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CTASAutoTabletTest.java index 05d405e015318..bb449d8fa2222 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CTASAutoTabletTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CTASAutoTabletTest.java @@ -49,7 +49,7 @@ public void testCTASAutoTablet() throws Exception { PseudoCluster cluster = PseudoCluster.getInstance(); cluster.runSql("db_for_auto_tablets", "create table test_table1 (k1 bigint, k2 bigint, v0 string) DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1);"); - Database db = GlobalStateMgr.getCurrentState().getDb("db_for_auto_tablets"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_for_auto_tablets"); if (db == null) { return; } @@ -63,7 +63,7 @@ public void testCTASAutoTablet() throws Exception { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - OlapTable table = (OlapTable) db.getTable("test_table1"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_table1"); if (table == null) { return; } @@ -72,7 +72,7 @@ public void testCTASAutoTablet() throws Exception { } // ctas1 - table = (OlapTable) db.getTable("ctas1"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "ctas1"); if (table == null) { return; } @@ -81,7 +81,7 @@ public void testCTASAutoTablet() throws Exception { } // ctas2 - table = (OlapTable) db.getTable("ctas2"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "ctas2"); if (table == null) { return; } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java index fb4a1083a5ada..b990fdace4b53 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java @@ -325,7 +325,7 @@ public static void beforeClass() throws Exception { .useDatabase("test"); starRocksAssert.withView("create view test.view_to_tbl1 as select * from test.tbl1;"); currentState = GlobalStateMgr.getCurrentState(); - testDb = currentState.getDb("test"); + testDb = currentState.getLocalMetastore().getDb("test"); UtFrameUtils.setUpForPersistTest(); } @@ -411,7 +411,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { SlotRef slotRef = tableSlotRefs.get(0); TableName baseTableName = slotRef.getTblNameWithoutAnalyzed(); Assert.assertEquals(baseTableName.getDb(), testDb.getFullName()); - Table baseTable = testDb.getTable(baseTableName.getTbl()); + Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), baseTableName.getTbl()); Assert.assertNotNull(baseTable); Assert.assertEquals(baseTableInfos.get(0).getTableId(), baseTable.getId()); Assert.assertEquals(1, baseTable.getRelatedMaterializedViews().size()); @@ -620,7 +620,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { try { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - Table mv1 = testDb.getTable("mv1"); + Table mv1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "mv1"); Assert.assertTrue(mv1 instanceof MaterializedView); // test partition MaterializedView materializedView = (MaterializedView) mv1; @@ -640,7 +640,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { SlotRef slotRef = slotRefs.get(0); TableName baseTableName = slotRef.getTblNameWithoutAnalyzed(); Assert.assertEquals(baseTableName.getDb(), testDb.getFullName()); - Table baseTable = testDb.getTable(baseTableName.getTbl()); + Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), baseTableName.getTbl()); Assert.assertNotNull(baseTable); Assert.assertTrue(baseTableInfos.stream().anyMatch(baseTableInfo -> baseTableInfo.getTableId() == baseTable.getId())); @@ -687,7 +687,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { try { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - Table mv1 = testDb.getTable("mv1"); + Table mv1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "mv1"); Assert.assertTrue(mv1 instanceof MaterializedView); // test partition MaterializedView materializedView = (MaterializedView) mv1; @@ -1694,7 +1694,7 @@ public void testCreateMvFromInactiveMv() { Assert.fail(e.getMessage()); } - MaterializedView baseInactiveMv = ((MaterializedView) testDb.getTable("base_inactive_mv")); + MaterializedView baseInactiveMv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "base_inactive_mv")); baseInactiveMv.setInactiveAndReason(""); String sql2 = "create materialized view mv_from_base_inactive_mv " + @@ -1728,11 +1728,11 @@ public void testAsHasStar() throws Exception { try { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - MaterializedView mv = ((MaterializedView) testDb.getTable("testAsHasStar")); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "testAsHasStar")); mv.setInactiveAndReason(""); List mvColumns = mv.getFullSchema(); - Table baseTable = testDb.getTable("tbl1"); + Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1"); List baseColumns = baseTable.getFullSchema(); Assert.assertEquals(mvColumns.size(), baseColumns.size() + 1); @@ -1821,7 +1821,7 @@ public void testAsSelectItemAlias1() throws Exception { try { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - MaterializedView mv = ((MaterializedView) testDb.getTable("testAsSelectItemAlias1")); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "testAsSelectItemAlias1")); mv.setInactiveAndReason(""); List mvColumns = mv.getFullSchema(); @@ -1852,7 +1852,7 @@ public void testAsSelectItemAlias2() throws Exception { try { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - MaterializedView mv = ((MaterializedView) testDb.getTable("testAsSelectItemAlias2")); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "testAsSelectItemAlias2")); mv.setInactiveAndReason(""); List mvColumns = mv.getFullSchema(); @@ -1981,7 +1981,7 @@ public void testCreateColocateMvToExitGroup() throws Exception { new MockUp() { @Mock public boolean isEnableColocateMVIndex() throws Exception { - OlapTable table = (OlapTable) testDb.getTable("colocateTable"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "colocateTable"); // If the table's colocate group is empty, return false if (Strings.isNullOrEmpty(table.getColocateGroup())) { @@ -2010,7 +2010,7 @@ public boolean isEnableColocateMVIndex() throws Exception { ColocateTableIndex.GroupId groupId = colocateTableIndex.getGroup(tableId); Assert.assertEquals(1, colocateTableIndex.getAllTableIds(groupId).size()); - OlapTable table = (OlapTable) testDb.getTable("colocateTable"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "colocateTable"); Assert.assertTrue(table.isEnableColocateMVIndex()); dropMv("colocateMv"); @@ -2072,7 +2072,7 @@ public void testColocateMvAlterGroup() throws Exception { new MockUp() { @Mock public boolean isEnableColocateMVIndex() throws Exception { - OlapTable table = (OlapTable) testDb.getTable("colocateTable3"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "colocateTable3"); // If the table's colocate group is empty, return false if (Strings.isNullOrEmpty(table.getColocateGroup())) { @@ -2109,7 +2109,7 @@ public boolean isEnableColocateMVIndex() throws Exception { long tableId = colocateTableIndex.getTableIdByGroup(fullGroupName); Assert.assertNotEquals(-1, tableId); - OlapTable table = (OlapTable) testDb.getTable("colocateTable3"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "colocateTable3"); Assert.assertTrue(table.isEnableColocateMVIndex()); ColocateTableIndex.GroupId groupId = colocateTableIndex.getGroup(tableId); @@ -2288,7 +2288,7 @@ public void testCreateMVWithSessionProperties1() { (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, starRocksAssert.getCtx()); currentState.getLocalMetastore().createMaterializedView(stmt); - Table mv1 = testDb.getTable("mv_with_property1"); + Table mv1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "mv_with_property1"); Assert.assertTrue(mv1 instanceof MaterializedView); } catch (Exception e) { Assert.fail(); @@ -2704,8 +2704,8 @@ public void testPartitionByNotFirstColumn() throws Exception { " partition by k1" + " distributed by hash(k3) buckets 10" + " as select k3, k1, sum(v1) as total from tbl5 group by k3, k1"); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table = db.getTable("mv_with_partition_by_not_first_column"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "mv_with_partition_by_not_first_column"); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; PartitionInfo partitionInfo = mv.getPartitionInfo(); @@ -2726,8 +2726,8 @@ public void testHiveMVWithoutPartition() throws Exception { "DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10 REFRESH MANUAL AS select s_suppkey, s_nationkey," + "sum(s_acctbal) as total_s_acctbal, count(s_phone) as s_phone_count from hive0.tpch.supplier as supp " + "group by s_suppkey, s_nationkey order by s_suppkey;"); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table = db.getTable("supplier_hive_mv"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "supplier_hive_mv"); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; PartitionInfo partitionInfo = mv.getPartitionInfo(); @@ -2742,8 +2742,8 @@ public void testHiveMVJoinWithoutPartition() throws Exception { "HASH(`s_suppkey`) BUCKETS 10 REFRESH MANUAL AS select s_suppkey, n_name, sum(s_acctbal) " + "as total_s_acctbal, count(s_phone) as s_phone_count from " + "hive0.tpch.supplier as supp join hive0.tpch.nation group by s_suppkey, n_name order by s_suppkey;"); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table = db.getTable("supplier_nation_hive_mv"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "supplier_nation_hive_mv"); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; PartitionInfo partitionInfo = mv.getPartitionInfo(); @@ -2761,8 +2761,8 @@ public void testHiveMVWithPartition() throws Exception { "AS \n" + "select l_shipdate, l_orderkey, l_quantity, l_linestatus, s_name from " + "hive0.partitioned_db.lineitem_par join hive0.tpch.supplier where l_suppkey = s_suppkey\n"); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table = db.getTable("lineitem_supplier_hive_mv"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "lineitem_supplier_hive_mv"); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; PartitionInfo partitionInfo = mv.getPartitionInfo(); @@ -2782,8 +2782,8 @@ public void testHiveMVAsyncRefresh() throws Exception { "AS select s_suppkey, s_nationkey, sum(s_acctbal) as total_s_acctbal, " + "count(s_phone) as s_phone_count from hive0.tpch.supplier as supp " + "group by s_suppkey, s_nationkey order by s_suppkey;"); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table = db.getTable("supplier_hive_mv"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "supplier_hive_mv"); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; PartitionInfo partitionInfo = mv.getPartitionInfo(); @@ -2912,7 +2912,7 @@ public void testCreateAsyncMv() { " select c_1_9, c_1_4 from t1"; try { starRocksAssert.withMaterializedView(sql); - MaterializedView mv = (MaterializedView) testDb.getTable("async_mv_1"); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "async_mv_1"); Assert.assertTrue(mv.getFullSchema().get(0).isKey()); Assert.assertFalse(mv.getFullSchema().get(1).isKey()); } catch (Exception e) { @@ -2957,7 +2957,7 @@ public void testCreateMVWithDifferentDB() { currentState.getLocalMetastore().createMaterializedView(stmt); waitingRollupJobV2Finish(); - Table table = testDb.getTable("tbl5"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl5"); Assert.assertNotNull(table); OlapTable olapTable = (OlapTable) table; Assert.assertTrue(olapTable.getIndexIdToMeta().size() >= 2); @@ -3010,7 +3010,7 @@ public void testCreateAsyncMVWithDifferentDB() { currentState.getLocalMetastore().createMaterializedView(stmt); newStarRocksAssert.dropDatabase("test_mv_different_db"); - Table mv1 = testDb.getTable("test_mv_use_different_tbl"); + Table mv1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "test_mv_use_different_tbl"); Assert.assertTrue(mv1 instanceof MaterializedView); starRocksAssert.dropMaterializedView("test_mv_use_different_tbl"); } catch (Exception e) { @@ -3036,8 +3036,8 @@ public void testCreateAsyncMVWithDifferentDB2() { currentState.getLocalMetastore().createMaterializedView(stmt); - Database differentDb = currentState.getDb("test_mv_different_db"); - Table mv1 = differentDb.getTable("test_mv_use_different_tbl"); + Database differentDb = currentState.getLocalMetastore().getDb("test_mv_different_db"); + Table mv1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(differentDb.getFullName(), "test_mv_use_different_tbl"); Assert.assertTrue(mv1 instanceof MaterializedView); newStarRocksAssert.dropDatabase("test_mv_different_db"); @@ -3063,7 +3063,7 @@ public void testCreateSyncMVWithCaseWhenComplexExpression1() { starRocksAssert.withMaterializedView(mv1); waitingRollupJobV2Finish(); - Table table = testDb.getTable("case_when_t1"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "case_when_t1"); Assert.assertNotNull(table); OlapTable olapTable = (OlapTable) table; Assert.assertTrue(olapTable.getIndexIdToMeta().size() >= 2); @@ -3155,8 +3155,8 @@ private void testMVColumnAlias(String expr) throws Exception { "refresh deferred manual distributed by hash(c_1_9) as" + " select c_1_9, %s from t1", mvName, expr); starRocksAssert.withMaterializedView(createMvExpr); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table = db.getTable(mvName); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), mvName); List columnNames = table.getBaseSchema().stream().map(Column::getName).collect(Collectors.toList()); Assert.assertTrue(columnNames.toString(), columnNames.contains(expr)); } finally { @@ -3174,8 +3174,8 @@ public void testExprAlias() throws Exception { } private Table getTable(String dbName, String mvName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(mvName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), mvName); Assert.assertNotNull(table); return table; } @@ -3494,9 +3494,9 @@ public void testMvOnUnion() throws Exception { "\t c_mktsegment segment\n" + "from customer_nullable_3;"); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); - MaterializedView mv = (MaterializedView) db.getTable("customer_mv"); + MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "customer_mv"); Assert.assertTrue(mv.getColumn("total").getType().isDecimalOfAnyVersion()); Assert.assertFalse(mv.getColumn("segment").isAllowNull()); } @@ -3884,7 +3884,7 @@ MaterializedView getMaterializedViewChecked(String sql) { ThreadUtil.sleepAtLeastIgnoreInterrupts(4000L); TableName mvName = createMaterializedViewStatement.getTableName(); - Table table = testDb.getTable(mvName.getTbl()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), mvName.getTbl()); Assert.assertNotNull(table); Assert.assertTrue(table instanceof MaterializedView); return (MaterializedView) table; @@ -3908,7 +3908,7 @@ List getMaterializedViewKeysChecked(String sql) { TableName mvTableName = createMaterializedViewStatement.getTableName(); mvName = mvTableName.getTbl(); - Table table = testDb.getTable(mvName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), mvName); Assert.assertNotNull(table); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateSyncMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateSyncMaterializedViewTest.java index 9ee8dc9e34e1f..3a7cd51fb8d57 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateSyncMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateSyncMaterializedViewTest.java @@ -198,12 +198,12 @@ public static void beforeClass() throws Exception { .useDatabase("test"); starRocksAssert.withView("create view test.view_to_tbl1 as select * from test.tbl1;"); currentState = GlobalStateMgr.getCurrentState(); - testDb = currentState.getDb("test"); + testDb = currentState.getLocalMetastore().getDb("test"); } private Table getTable(String dbName, String mvName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(mvName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName); Assert.assertNotNull(table); return table; } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableAutoTabletTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableAutoTabletTest.java index 6bb7c8d094741..f46a80f89f70c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableAutoTabletTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableAutoTabletTest.java @@ -54,11 +54,11 @@ public void testAutoTabletWithoutPartition() throws Exception { PseudoCluster cluster = PseudoCluster.getInstance(); cluster.runSql("db_for_auto_tablets", "create table test_table1 (pk bigint NOT NULL, v0 string not null) primary KEY (pk) DISTRIBUTED BY HASH(pk) PROPERTIES(\"replication_num\" = \"3\", \"storage_medium\" = \"SSD\");"); - Database db = GlobalStateMgr.getCurrentState().getDb("db_for_auto_tablets"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_for_auto_tablets"); if (db == null) { return; } - OlapTable table = (OlapTable) db.getTable("test_table1"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_table1"); if (table == null) { return; } @@ -100,12 +100,12 @@ public void test1AutoTabletWithPartition() throws Exception { " PARTITION BY RANGE(pk2) (START (\"2022-08-01\") END (\"2022-08-10\") EVERY (INTERVAL 1 day))" + " DISTRIBUTED BY HASH(pk1)" + " PROPERTIES (\"replication_num\" = \"3\", \"storage_medium\" = \"SSD\");"); - Database db = GlobalStateMgr.getCurrentState().getDb("db_for_auto_tablets"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_for_auto_tablets"); if (db == null) { return; } - OlapTable table = (OlapTable) db.getTable("test_table2"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_table2"); if (table == null) { return; } @@ -150,12 +150,12 @@ public void test1AutoTabletWithDynamicPartition() throws Exception { " 'dynamic_partition.end' = '3'," + " 'dynamic_partition.prefix' = 'p');"); Thread.sleep(1000); // wait for the dynamic partition created - Database db = GlobalStateMgr.getCurrentState().getDb("db_for_auto_tablets"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_for_auto_tablets"); if (db == null) { return; } - OlapTable table = (OlapTable) db.getTable("test_auto_tablets_of_dynamic_partition"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_auto_tablets_of_dynamic_partition"); if (table == null) { return; } @@ -197,12 +197,12 @@ public void test1AutoTabletWithModifyDynamicPartitionProperty() throws Exception " 'dynamic_partition.buckets' = '3'," + " 'dynamic_partition.prefix' = 'p');"); Thread.sleep(1000); // wait for the dynamic partition created - Database db = GlobalStateMgr.getCurrentState().getDb("db_for_auto_tablets"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_for_auto_tablets"); if (db == null) { return; } - OlapTable table = (OlapTable) db.getTable("test_modify_dynamic_partition_property"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_modify_dynamic_partition_property"); if (table == null) { return; } @@ -244,12 +244,12 @@ public void test1AutoTabletWithColocate() throws Exception { " PROPERTIES (" + " 'replication_num' = '1'," + " 'colocate_with' = 'g1');"); - Database db = GlobalStateMgr.getCurrentState().getDb("db_for_auto_tablets"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_for_auto_tablets"); if (db == null) { return; } - OlapTable table = (OlapTable) db.getTable("colocate_partition"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "colocate_partition"); if (table == null) { return; } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/MaterializedViewAutoTabletTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/MaterializedViewAutoTabletTest.java index 139cdf35ed681..5b7fdac57176a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/MaterializedViewAutoTabletTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/MaterializedViewAutoTabletTest.java @@ -48,7 +48,7 @@ public void testMaterializedViewAutoTablet() throws Exception { PseudoCluster cluster = PseudoCluster.getInstance(); cluster.runSql("db_for_auto_tablets", "create table test_table1 (k1 bigint, v0 int) DUPLICATE KEY (k1) DISTRIBUTED BY HASH(k1);"); - Database db = GlobalStateMgr.getCurrentState().getDb("db_for_auto_tablets"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_for_auto_tablets"); if (db == null) { return; } @@ -59,7 +59,7 @@ public void testMaterializedViewAutoTablet() throws Exception { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - OlapTable table = (OlapTable) db.getTable("test_table1"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_table1"); if (table == null) { return; } @@ -67,7 +67,7 @@ public void testMaterializedViewAutoTablet() throws Exception { bucketNum1 += partition.getDistributionInfo().getBucketNum(); } - table = (OlapTable) db.getTable("mv1"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv1"); if (table == null) { return; } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java index 69ba56b7e6ac9..4751f3d69f0a8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewStatementTest.java @@ -86,14 +86,14 @@ public void testRereshNotMaterializedView() { @Test public void testRefreshMaterializedView() throws Exception { - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); starRocksAssert.withMaterializedView("create materialized view mv1 distributed by hash(`c1`) " + " refresh manual" + " as select c1, sum(c3) as total from table_name_tmp_1 group by c1"); cluster.runSql("test", "insert into table_name_tmp_1 values(1, \"str1\", 100)"); - Table table = db.getTable("table_name_tmp_1"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "table_name_tmp_1"); Assert.assertNotNull(table); - Table t2 = db.getTable("mv1"); + Table t2 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv1"); Assert.assertNotNull(t2); MaterializedView mv1 = (MaterializedView) t2; cluster.runSql("test", "refresh materialized view mv1 with sync mode"); diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java index 606320873e8f6..a8e9a75593a5b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/RefreshMaterializedViewTest.java @@ -33,7 +33,6 @@ import com.starrocks.sql.ast.DmlStmt; import com.starrocks.sql.ast.InsertStmt; import com.starrocks.sql.ast.RefreshMaterializedViewStatement; -import com.starrocks.sql.optimizer.QueryMaterializationContext; import com.starrocks.sql.optimizer.rule.transformation.materialization.MvRewriteTestBase; import com.starrocks.sql.plan.ExecPlan; import com.starrocks.utframe.UtFrameUtils; @@ -53,7 +52,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -public class RefreshMaterializedViewTest extends MvRewriteTestBase { +public class RefreshMaterializedViewTest extends MvRewriteTestBase { // 1hour: set it to 1 hour to avoid FE's async update too late. private static final long MV_STALENESS = 60 * 60; @@ -79,7 +78,7 @@ public static void beforeClass() throws Exception { "refresh manual\n" + "as select k2, sum(v1) as total from tbl_with_mv group by k2;") .withMaterializedView("create materialized view mv2_to_refresh\n" + - "PARTITION BY k1\n"+ + "PARTITION BY k1\n" + "distributed by hash(k2) buckets 3\n" + "refresh manual\n" + "as select k1, k2, v1 from tbl_with_mv;"); @@ -96,7 +95,8 @@ public void testNormal() throws Exception { Assert.assertEquals("mv_to_refresh", mvName); String sql = "REFRESH MATERIALIZED VIEW test.mv2_to_refresh PARTITION START('2022-02-03') END ('2022-02-25') FORCE;"; - RefreshMaterializedViewStatement statement = (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + RefreshMaterializedViewStatement statement = + (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); Assert.assertTrue(statement.isForceRefresh()); Assert.assertEquals("2022-02-03", statement.getPartitionRangeDesc().getPartitionStart()); Assert.assertEquals("2022-02-25", statement.getPartitionRangeDesc().getPartitionEnd()); @@ -122,7 +122,7 @@ public void testNormal() throws Exception { statement = (RefreshMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); } catch (Exception e) { Assert.assertEquals("Getting analyzing error from line 1, column 56 to line 1, column 90. " + - "Detail message: Batch build partition EVERY is date type but START or END does not type match.", + "Detail message: Batch build partition EVERY is date type but START or END does not type match.", e.getMessage()); } } @@ -179,7 +179,7 @@ public void testMaxMVRewriteStaleness1() { ), () -> { starRocksAssert.withMaterializedView("create materialized view mv_with_mv_rewrite_staleness\n" + - "PARTITION BY k1\n"+ + "PARTITION BY k1\n" + "distributed by hash(k2) buckets 3\n" + "PROPERTIES (\n" + "\"replication_num\" = \"1\"" + @@ -258,7 +258,8 @@ public void testMaxMVRewriteStaleness2() { // refresh partitions are not empty if base table is updated. { - executeInsertSql(connectContext, "insert into tbl_staleness2 partition(p2) values(\"2022-02-20\", 1, 10)"); + executeInsertSql(connectContext, + "insert into tbl_staleness2 partition(p2) values(\"2022-02-20\", 1, 10)"); refreshMaterializedView("test", "mv_with_mv_rewrite_staleness2"); Table tbl1 = getTable("test", "tbl_staleness2"); @@ -339,7 +340,8 @@ public void testMaxMVRewriteStaleness3() { "as select k1, k2, count(1) from mv_with_mv_rewrite_staleness21 group by k1, k2;"); { - executeInsertSql(connectContext, "insert into tbl_staleness3 partition(p2) values(\"2022-02-20\", 1, 10)"); + executeInsertSql(connectContext, + "insert into tbl_staleness3 partition(p2) values(\"2022-02-20\", 1, 10)"); refreshMaterializedView("test", "mv_with_mv_rewrite_staleness21"); refreshMaterializedView("test", "mv_with_mv_rewrite_staleness22"); { @@ -427,7 +429,8 @@ public void testMaxMVRewriteStaleness3() { String alterMvSql = "alter materialized view mv_with_mv_rewrite_staleness21 " + "set (\"mv_rewrite_staleness_second\" = \"0\")"; AlterMaterializedViewStmt stmt = - (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(alterMvSql, connectContext); + (AlterMaterializedViewStmt) UtFrameUtils.parseStmtWithNewParser(alterMvSql, + connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().alterMaterializedView(stmt); } @@ -457,8 +460,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { if (stmt instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) stmt; TableName tableName = insertStmt.getTableName(); - Database testDb = GlobalStateMgr.getCurrentState().getDb(stmt.getTableName().getDb()); - OlapTable tbl = ((OlapTable) testDb.getTable(tableName.getTbl())); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(stmt.getTableName().getDb()); + OlapTable tbl = ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), tableName.getTbl())); for (Partition partition : tbl.getPartitions()) { if (insertStmt.getTargetPartitionIds().contains(partition.getId())) { long version = partition.getVisibleVersion() + 1; @@ -560,8 +564,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { if (stmt instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) stmt; TableName tableName = insertStmt.getTableName(); - Database testDb = GlobalStateMgr.getCurrentState().getDb(stmt.getTableName().getDb()); - OlapTable tbl = ((OlapTable) testDb.getTable(tableName.getTbl())); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(stmt.getTableName().getDb()); + OlapTable tbl = ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), tableName.getTbl())); for (Partition partition : tbl.getPartitions()) { if (insertStmt.getTargetPartitionIds().contains(partition.getId())) { long version = partition.getVisibleVersion() + 1; @@ -581,49 +586,50 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { }; starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE `test`.`tbl_with_partition` (\n" + - " `k1` date,\n" + - " `k2` int,\n" + - " `v1` string\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`)\n" + - "PARTITION BY RANGE(`k1`)\n" + - "(\n" + - "PARTITION p20230410 VALUES [(\"2023-04-10\"), (\"2023-04-11\")),\n" + - "PARTITION p20230411 VALUES [(\"2023-04-11\"), (\"2023-04-12\")),\n" + - "PARTITION p20230412 VALUES [(\"2023-04-12\"), (\"2023-04-13\")),\n" + - "PARTITION p20230413 VALUES [(\"2023-04-13\"), (\"2023-04-14\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(`v1`) BUCKETS 3\n" + - "PROPERTIES (\"replication_num\" = \"1\");") - .withMaterializedView("CREATE MATERIALIZED VIEW `mv_with_partition`\n" + - "PARTITION BY (date_trunc('day', k1))\n" + - "REFRESH DEFERRED MANUAL \n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 3\n" + - "PROPERTIES (\"replication_num\" = \"1\", \"partition_refresh_number\"=\"1\")\n" + - "AS\n" + - "SELECT \n" + - "k1,\n" + - "count(DISTINCT `v1`) AS `v` \n" + - "FROM `test`.`tbl_with_partition`\n" + - "group by k1;"); + .withTable("CREATE TABLE `test`.`tbl_with_partition` (\n" + + " `k1` date,\n" + + " `k2` int,\n" + + " `v1` string\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`)\n" + + "PARTITION BY RANGE(`k1`)\n" + + "(\n" + + "PARTITION p20230410 VALUES [(\"2023-04-10\"), (\"2023-04-11\")),\n" + + "PARTITION p20230411 VALUES [(\"2023-04-11\"), (\"2023-04-12\")),\n" + + "PARTITION p20230412 VALUES [(\"2023-04-12\"), (\"2023-04-13\")),\n" + + "PARTITION p20230413 VALUES [(\"2023-04-13\"), (\"2023-04-14\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(`v1`) BUCKETS 3\n" + + "PROPERTIES (\"replication_num\" = \"1\");") + .withMaterializedView("CREATE MATERIALIZED VIEW `mv_with_partition`\n" + + "PARTITION BY (date_trunc('day', k1))\n" + + "REFRESH DEFERRED MANUAL \n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 3\n" + + "PROPERTIES (\"replication_num\" = \"1\", \"partition_refresh_number\"=\"1\")\n" + + "AS\n" + + "SELECT \n" + + "k1,\n" + + "count(DISTINCT `v1`) AS `v` \n" + + "FROM `test`.`tbl_with_partition`\n" + + "group by k1;"); starRocksAssert.updateTablePartitionVersion("test", "tbl_with_partition", 2); starRocksAssert.refreshMvPartition("REFRESH MATERIALIZED VIEW test.mv_with_partition \n" + "PARTITION START (\"2023-04-10\") END (\"2023-04-14\")"); MaterializedView mv = getMv("test", "mv_with_partition"); Map> versionMap = - mv.getRefreshScheme().getAsyncRefreshContext().getBaseTableVisibleVersionMap(); + mv.getRefreshScheme().getAsyncRefreshContext().getBaseTableVisibleVersionMap(); Assert.assertEquals(1, versionMap.size()); Set partitions = versionMap.values().iterator().next().keySet(); // Assert.assertEquals(4, partitions.size()); starRocksAssert.alterMvProperties( - "alter materialized view test.mv_with_partition set (\"partition_ttl_number\" = \"1\")"); + "alter materialized view test.mv_with_partition set (\"partition_ttl_number\" = \"1\")"); DynamicPartitionScheduler dynamicPartitionScheduler = GlobalStateMgr.getCurrentState() - .getDynamicPartitionScheduler(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("mv_with_partition"); + .getDynamicPartitionScheduler(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv_with_partition"); dynamicPartitionScheduler.registerTtlPartitionTable(db.getId(), tbl.getId()); dynamicPartitionScheduler.runOnceForTest(); starRocksAssert.refreshMvPartition("REFRESH MATERIALIZED VIEW test.mv_with_partition \n" + @@ -631,9 +637,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { Assert.assertEquals(Sets.newHashSet("p20230413"), versionMap.values().iterator().next().keySet()); starRocksAssert - .useDatabase("test") - .dropMaterializedView("mv_with_partition") - .dropTable("tbl_with_partition"); + .useDatabase("test") + .dropMaterializedView("mv_with_partition") + .dropTable("tbl_with_partition"); } private Set buildTimePartitions(String tableName, OlapTable tbl, int partitionCount) throws Exception { @@ -656,22 +662,22 @@ public void testMaterializedViewPartitionTTL() throws Exception { String dbName = "test"; String tableName = "test.tbl1"; starRocksAssert.withTable("CREATE TABLE " + tableName + - "(\n" + - " k1 date,\n" + - " v1 int \n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2022-06-01'),\n" + - " PARTITION p2 values less than('2022-07-01'),\n" + - " PARTITION p3 values less than('2022-08-01'),\n" + - " PARTITION p4 values less than('2022-09-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + - "PROPERTIES\n" + - "(\n" + - " 'replication_num' = '1'\n" + - ");"); + "(\n" + + " k1 date,\n" + + " v1 int \n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2022-06-01'),\n" + + " PARTITION p2 values less than('2022-07-01'),\n" + + " PARTITION p3 values less than('2022-08-01'),\n" + + " PARTITION p4 values less than('2022-09-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + + "PROPERTIES\n" + + "(\n" + + " 'replication_num' = '1'\n" + + ");"); starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW test.mv_ttl_mv1\n" + " REFRESH ASYNC " + " PARTITION BY k1\n" + @@ -680,8 +686,8 @@ public void testMaterializedViewPartitionTTL() throws Exception { DynamicPartitionScheduler dynamicPartitionScheduler = GlobalStateMgr.getCurrentState() .getDynamicPartitionScheduler(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("mv_ttl_mv1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv_ttl_mv1"); Set addedPartitions = buildTimePartitions(tableName, tbl, 10); // Build expectations @@ -849,7 +855,8 @@ public void testMvOnUnion_IntersectedPartition1() throws Exception { "AS \n" + "SELECT * FROM mv_union_t1 t1\n" + "UNION ALL\n" + - "SELECT * FROM mv_union_t2 t2\n", () -> {}); + "SELECT * FROM mv_union_t2 t2\n", () -> { + }); } { @@ -860,7 +867,8 @@ public void testMvOnUnion_IntersectedPartition1() throws Exception { "AS \n" + "SELECT * FROM mv_union_t1 t1\n" + "UNION ALL\n" + - "SELECT * FROM mv_union_t3 t2\n", () -> {}); + "SELECT * FROM mv_union_t3 t2\n", () -> { + }); } { @@ -871,7 +879,8 @@ public void testMvOnUnion_IntersectedPartition1() throws Exception { "AS \n" + "SELECT * FROM mv_union_t1 t1\n" + "UNION ALL\n" + - "SELECT * FROM mv_union_t4 t2\n", () -> {}); + "SELECT * FROM mv_union_t4 t2\n", () -> { + }); // add partition to child table starRocksAssert.ddl("alter table mv_union_t4 add partition p20240325 values less than ('2024-03-25')"); starRocksAssert.ddl("alter table mv_union_t1 add partition p20240326 values less than ('2024-03-26')"); @@ -968,8 +977,9 @@ public void testMvOnUnion_IntersectedPartition3() throws Exception { "UNION ALL\n" + "SELECT * FROM t2\n", () -> { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = (MaterializedView) db.getTable("mv1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv1"); System.out.println(mv.getPartitionNames()); }); } @@ -1018,8 +1028,9 @@ public void testMvOnUnion_IntersectedPartition4() throws Exception { "SELECT * FROM t1\n" + "UNION ALL\n" + "SELECT * FROM t2\n", () -> { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = (MaterializedView) db.getTable("mv1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv1"); Assert.assertEquals(2, mv.getPartitionExprMaps().size()); System.out.println(mv.getPartitionNames()); }); @@ -1067,8 +1078,9 @@ public void testMvOnUnion_IntersectedPartition5() throws Exception { "REFRESH ASYNC\n" + "AS \n" + "select k1 from (SELECT * FROM t1 UNION ALL SELECT * FROM t2) t group by k1\n", () -> { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = (MaterializedView) db.getTable("mv1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv1"); System.out.println(mv.getPartitionExprMaps()); Assert.assertEquals(2, mv.getPartitionExprMaps().size()); System.out.println(mv.getPartitionNames()); diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/ResourceGroupStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/ResourceGroupStmtTest.java index 587d053953e2d..0175497b4214d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/ResourceGroupStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/ResourceGroupStmtTest.java @@ -531,7 +531,7 @@ public void testChooseResourceGroupWithDb() throws Exception { starRocksAssert.getCtx().setCurrentUserIdentity(new UserIdentity(qualifiedUser, "%")); starRocksAssert.getCtx().setRemoteIP(remoteIp); { - long dbId = GlobalStateMgr.getCurrentState().getDb("db1").getId(); + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1").getId(); Set dbIds = ImmutableSet.of(dbId); TWorkGroup wg = GlobalStateMgr.getCurrentState().getResourceGroupMgr().chooseResourceGroup( starRocksAssert.getCtx(), @@ -540,7 +540,7 @@ public void testChooseResourceGroupWithDb() throws Exception { Assert.assertEquals("rg5", wg.getName()); } { - long dbId = GlobalStateMgr.getCurrentState().getDb("db2").getId(); + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db2").getId(); Set dbIds = ImmutableSet.of(dbId); TWorkGroup wg = GlobalStateMgr.getCurrentState().getResourceGroupMgr().chooseResourceGroup( starRocksAssert.getCtx(), @@ -551,8 +551,8 @@ public void testChooseResourceGroupWithDb() throws Exception { } { Set dbIds = ImmutableSet.of( - GlobalStateMgr.getCurrentState().getDb("db1").getId(), - GlobalStateMgr.getCurrentState().getDb("db2").getId()); + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1").getId(), + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db2").getId()); TWorkGroup wg = GlobalStateMgr.getCurrentState().getResourceGroupMgr().chooseResourceGroup( starRocksAssert.getCtx(), ResourceGroupClassifier.QueryType.SELECT, @@ -773,7 +773,7 @@ public void testChooseShortQueryResourceGroup() throws Exception { // Prefer the short query group regardless its classifier weight is the lowest. String qualifiedUser = "rt_rg_user"; - long dbId = GlobalStateMgr.getCurrentState().getDb("db1").getId(); + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1").getId(); Set dbs = ImmutableSet.of(dbId); starRocksAssert.getCtx().setQualifiedUser(qualifiedUser); starRocksAssert.getCtx().setCurrentUserIdentity(new UserIdentity(qualifiedUser, "%")); diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateMaterializedViewStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateMaterializedViewStmtTest.java index ffa12d054d7b8..20f9e51633af9 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateMaterializedViewStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateMaterializedViewStmtTest.java @@ -97,7 +97,7 @@ public void testShowInternalCatalogConstraints() throws Exception { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(createMvSql, ctx); GlobalStateMgr currentState = GlobalStateMgr.getCurrentState(); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - Table table = currentState.getDb("test").getTable("mv9"); + Table table = currentState.getLocalMetastore().getDb("test").getTable("mv9"); List createTableStmt = Lists.newArrayList(); AstToStringBuilder.getDdlStmt(table, createTableStmt, null, null, false, true); Assert.assertEquals("CREATE MATERIALIZED VIEW `mv9` (`k1`, `k2`)\n" + @@ -125,7 +125,7 @@ public void testShowExternalCatalogConstraints() throws Exception { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(createMvSql, ctx); GlobalStateMgr currentState = GlobalStateMgr.getCurrentState(); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - Table table = currentState.getDb("test").getTable("mv10"); + Table table = currentState.getLocalMetastore().getDb("test").getTable("mv10"); List createTableStmt = Lists.newArrayList(); AstToStringBuilder.getDdlStmt(table, createTableStmt, null, null, false, true); Assert.assertEquals("CREATE MATERIALIZED VIEW `mv10` (`c1`, `c2`)\n" + @@ -154,7 +154,7 @@ public void testShowExternalTableCreateMvSql() throws Exception { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(createMvSql, ctx); GlobalStateMgr currentState = GlobalStateMgr.getCurrentState(); currentState.getLocalMetastore().createMaterializedView((CreateMaterializedViewStatement) statementBase); - Table table = currentState.getDb("test").getTable("mv8"); + Table table = currentState.getLocalMetastore().getDb("test").getTable("mv8"); List createTableStmt = Lists.newArrayList(); AstToStringBuilder.getDdlStmt(table, createTableStmt, null, null, false, true); Assert.assertEquals(createTableStmt.get(0), diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateViewStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateViewStmtTest.java index aac41a7ecca84..dde731d6f8ac8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateViewStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCreateViewStmtTest.java @@ -189,14 +189,14 @@ public void testCreateView() throws Exception { CreateViewStmt createViewStmt = (CreateViewStmt) UtFrameUtils.parseStmtWithNewParser(testcase[1], ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().createView(createViewStmt); - List
views = GlobalStateMgr.getCurrentState().getDb(createViewStmt.getDbName()).getViews(); + List
views = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createViewStmt.getDbName()).getViews(); List res = Lists.newArrayList(); AstToStringBuilder.getDdlStmt(createViewStmt.getDbName(), views.get(0), res, null, null, false, false, false); Assert.assertEquals(testcase[2], res.get(0)); - GlobalStateMgr.getCurrentState().getDb(createViewStmt.getDbName()).dropTable(createViewStmt.getTable()); + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createViewStmt.getDbName()).dropTable(createViewStmt.getTable()); } } @@ -208,7 +208,7 @@ public void testShowCreateView() throws Exception { CreateViewStmt createViewStmt = (CreateViewStmt) UtFrameUtils.parseStmtWithNewParser(createViewSql, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().createView(createViewStmt); - List
views = GlobalStateMgr.getCurrentState().getDb(createViewStmt.getDbName()).getViews(); + List
views = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createViewStmt.getDbName()).getViews(); List res = Lists.newArrayList(); AstToStringBuilder.getDdlStmt(createViewStmt.getDbName(), views.get(0), res, null, null, false, false, false); @@ -276,7 +276,7 @@ public void testViewOfThreeUnionAllWithConstNullOutput() throws Exception { @Test public void testDdlComment() { - List
tables = GlobalStateMgr.getCurrentState().getDb("test").getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTables(); Table commentTest = tables.stream().filter(table -> table.getName().equals("comment_test")).findFirst().get(); List res = Lists.newArrayList(); AstToStringBuilder.getDdlStmt("test", commentTest, res, @@ -287,7 +287,7 @@ public void testDdlComment() { @Test public void testDdlStorageType() { - List
tables = GlobalStateMgr.getCurrentState().getDb("test").getTables(); + List
tables = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTables(); Table storageTest = tables.stream().filter(table -> table.getName().equals("storage_test")).findFirst().get(); List res = Lists.newArrayList(); AstToStringBuilder.getDdlStmt("storage_test", storageTest, res, diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDbStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDbStmtTest.java index f5c1044a79c2a..1a3656f419c59 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDbStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDbStmtTest.java @@ -19,7 +19,6 @@ import com.google.common.collect.Lists; import com.starrocks.catalog.Database; -import com.starrocks.common.jmockit.Deencapsulation; import com.starrocks.common.util.UUIDUtil; import com.starrocks.mysql.MysqlCommand; import com.starrocks.qe.ConnectContext; @@ -51,33 +50,11 @@ public void setUp() throws Exception { Database db = new Database(); new Expectations(db) { { - db.getTable(anyString); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), anyString); minTimes = 0; } }; - // mock globalStateMgr. - globalStateMgr = Deencapsulation.newInstance(GlobalStateMgr.class); - new Expectations(globalStateMgr) { - { - globalStateMgr.getDb("testCluster:testDb"); - minTimes = 0; - result = db; - - globalStateMgr.getDb("testCluster:emptyDb"); - minTimes = 0; - result = null; - - GlobalStateMgr.getCurrentState(); - minTimes = 0; - result = globalStateMgr; - - GlobalStateMgr.getCurrentState(); - minTimes = 0; - result = globalStateMgr; - } - }; - // mock scheduler ConnectScheduler scheduler = new ConnectScheduler(10); new Expectations(scheduler) { diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDeleteTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDeleteTest.java index 072fb2c827dac..f77123e639b40 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDeleteTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowDeleteTest.java @@ -16,7 +16,6 @@ import com.google.common.collect.Lists; import com.starrocks.catalog.Database; -import com.starrocks.common.jmockit.Deencapsulation; import com.starrocks.common.util.UUIDUtil; import com.starrocks.mysql.MysqlCommand; import com.starrocks.qe.ConnectContext; @@ -45,33 +44,11 @@ public void setUp() throws Exception { Database db = new Database(); new Expectations(db) { { - db.getTable(anyString); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), anyString); minTimes = 0; } }; - // mock globalStateMgr. - globalStateMgr = Deencapsulation.newInstance(GlobalStateMgr.class); - new Expectations(globalStateMgr) { - { - globalStateMgr.getDb("testCluster:testDb"); - minTimes = 0; - result = db; - - globalStateMgr.getDb("testCluster:emptyDb"); - minTimes = 0; - result = null; - - GlobalStateMgr.getCurrentState(); - minTimes = 0; - result = globalStateMgr; - - GlobalStateMgr.getCurrentState(); - minTimes = 0; - result = globalStateMgr; - } - }; - // mock scheduler ConnectScheduler scheduler = new ConnectScheduler(10); new Expectations(scheduler) { diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/UseMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/UseMaterializedViewTest.java index cf088eb1acb35..b41ec6fe4cb8a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/UseMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/UseMaterializedViewTest.java @@ -20,10 +20,9 @@ import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; -import com.starrocks.common.Config; -import com.starrocks.common.FeConstants; import com.starrocks.qe.ConnectContext; import com.starrocks.qe.StmtExecutor; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.Field; import com.starrocks.sql.ast.QueryRelation; import com.starrocks.sql.ast.QueryStatement; @@ -122,18 +121,18 @@ public void testSelect() { public void testDropMaterializedView() { String sql = "drop materialized view mv_to_drop"; try { - Database database = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); + Database database = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); Assert.assertTrue(database != null); - Table table = database.getTable("mv_to_drop"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "mv_to_drop"); Assert.assertTrue(table != null); MaterializedView materializedView = (MaterializedView) table; long baseTableId = materializedView.getBaseTableInfos().iterator().next().getTableId(); - OlapTable baseTable = ((OlapTable) database.getTable(baseTableId)); + OlapTable baseTable = ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), baseTableId)); Assert.assertEquals(2, baseTable.getRelatedMaterializedViews().size()); StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statementBase); stmtExecutor.execute(); - table = database.getTable("mv_to_drop"); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "mv_to_drop"); Assert.assertTrue(table == null); Assert.assertEquals(1, baseTable.getRelatedMaterializedViews().size()); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java index 2be4d47dd3d07..8806b92658e2f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupHandlerTest.java @@ -58,6 +58,7 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.qe.DDLStmtExecutor; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.sql.analyzer.BackupRestoreAnalyzer; import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.sql.ast.BackupStmt; @@ -119,8 +120,19 @@ public void setUpMocker(GlobalStateMgr globalStateMgr, BrokerMgr brokerMgr, Edit MetricRepo.init(); + try { + db = CatalogMocker.mockDb(); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + new Expectations() { { + GlobalStateMgr.getCurrentState(); + minTimes = 0; + result = globalStateMgr; + globalStateMgr.getBrokerMgr(); minTimes = 0; result = brokerMgr; @@ -133,30 +145,11 @@ public void setUpMocker(GlobalStateMgr globalStateMgr, BrokerMgr brokerMgr, Edit minTimes = 0; result = editLog; - GlobalStateMgr.getCurrentState(); - minTimes = 0; - result = globalStateMgr; - - GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); + globalStateMgr.getTabletInvertedIndex(); minTimes = 0; result = invertedIndex; } }; - - try { - db = CatalogMocker.mockDb(); - } catch (AnalysisException e) { - e.printStackTrace(); - Assert.fail(); - } - - new Expectations() { - { - globalStateMgr.getDb(anyString); - minTimes = 0; - result = db; - } - }; } @After @@ -164,8 +157,8 @@ public void done() { if (rootDir != null) { try { Files.walk(Paths.get(Config.tmp_dir), - FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) - .forEach(File::delete); + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) + .forEach(File::delete); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -176,7 +169,7 @@ public void done() { @Test public void testInit(@Mocked GlobalStateMgr globalStateMgr, @Mocked BrokerMgr brokerMgr, @Mocked EditLog editLog) { setUpMocker(globalStateMgr, brokerMgr, editLog); - handler = new BackupHandler(globalStateMgr); + BackupHandler handler = new BackupHandler(globalStateMgr); handler.runAfterCatalogReady(); File backupDir = new File(BackupHandler.BACKUP_ROOT_DIR.toString()); @@ -185,7 +178,7 @@ public void testInit(@Mocked GlobalStateMgr globalStateMgr, @Mocked BrokerMgr br @Test public void testCreateAndDropRepository( - @Mocked GlobalStateMgr globalStateMgr, @Mocked BrokerMgr brokerMgr, @Mocked EditLog editLog) throws Exception { + @Mocked GlobalStateMgr globalStateMgr, @Mocked BrokerMgr brokerMgr, @Mocked EditLog editLog) throws Exception { setUpMocker(globalStateMgr, brokerMgr, editLog); new Expectations() { { @@ -204,6 +197,10 @@ public void logDropRepository(String repoName) { } }; + + globalStateMgr.getLocalMetastore().getDb(anyLong); + minTimes = 0; + result = db; } }; @@ -221,7 +218,8 @@ public Status listSnapshots(List snapshotNames) { @Mock public Status getSnapshotInfoFile(String label, String backupTimestamp, List infos) { - OlapTable tbl = (OlapTable) db.getTable(CatalogMocker.TEST_TBL_NAME); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), CatalogMocker.TEST_TBL_NAME); List
tbls = Lists.newArrayList(); tbls.add(tbl); Map snapshotInfos = Maps.newHashMap(); @@ -230,15 +228,15 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List files = Lists.newArrayList(); SnapshotInfo sinfo = new SnapshotInfo(db.getId(), tbl.getId(), part.getId(), idx.getId(), - tablet.getId(), -1, 0, "./path", files); + tablet.getId(), -1, 0, "./path", files); snapshotInfos.put(tablet.getId(), sinfo); } } } BackupJobInfo info = BackupJobInfo.fromCatalog(System.currentTimeMillis(), - "ss2", CatalogMocker.TEST_DB_NAME, - CatalogMocker.TEST_DB_ID, tbls, snapshotInfos); + "ss2", CatalogMocker.TEST_DB_NAME, + CatalogMocker.TEST_DB_ID, tbls, snapshotInfos); infos.add(info); return Status.OK; } @@ -252,10 +250,48 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List() { + Database database = CatalogMocker.mockDb(); + + @Mock + public Database getDb(String dbName) { + return database; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return database.getTable(tblName); + } + }; + // add repo - handler = new BackupHandler(globalStateMgr); + BackupHandler handler = new BackupHandler(globalStateMgr); CreateRepositoryStmt stmt = new CreateRepositoryStmt(false, "repo", "broker", "bos://location", - Maps.newHashMap()); + Maps.newHashMap()); try { handler.createRepository(stmt); } catch (DdlException e) { @@ -267,7 +303,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List tblRefs = Lists.newArrayList(); tblRefs.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME), null)); BackupStmt backupStmt = new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label1"), "repo", tblRefs, - null); + null); try { handler.process(backupStmt); } catch (DdlException e1) { @@ -278,7 +314,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List snapshotFiles = Lists.newArrayList(); request.setSnapshot_files(snapshotFiles); @@ -289,7 +325,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List srcToDestPath = Maps.newHashMap(); UploadTask uploadTask = new UploadTask(null, 0, 0, backupJob.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath, null, null); + srcToDestPath, null, null); request = new TFinishTaskRequest(); Map> tabletFiles = Maps.newHashMap(); request.setTablet_files(tabletFiles); @@ -325,8 +361,8 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List tblRefs1 = Lists.newArrayList(); tblRefs1.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL3_NAME), null)); BackupStmt backupStmt1 = - new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs1, - null); + new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs1, + null); try { handler.process(backupStmt1); } catch (DdlException e1) { @@ -337,7 +373,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List snapshotFiles1 = Lists.newArrayList(); request1.setSnapshot_files(snapshotFiles1); @@ -348,7 +384,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List srcToDestPath1 = Maps.newHashMap(); UploadTask uploadTask1 = new UploadTask(null, 0, 0, backupJob1.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath1, null, null); + srcToDestPath1, null, null); request1 = new TFinishTaskRequest(); Map> tabletFiles1 = Maps.newHashMap(); request1.setTablet_files(tabletFiles1); @@ -386,7 +422,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List properties = Maps.newHashMap(); properties.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "ss2"), "repo", tblRefs2, - properties); + properties); try { BackupRestoreAnalyzer.analyze(restoreStmt, new ConnectContext()); } catch (SemanticException e2) { @@ -404,7 +440,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List downloadedTabletIds = Lists.newArrayList(); request.setDownloaded_tablet_ids(downloadedTabletIds); @@ -421,7 +457,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List properties1 = Maps.newHashMap(); properties1.put("backup_timestamp", "2018-08-08-08-08-08"); RestoreStmt restoreStmt1 = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label2"), "repo", tblRefs3, - properties1); + properties1); try { BackupRestoreAnalyzer.analyze(restoreStmt1, new ConnectContext()); } catch (SemanticException e2) { @@ -482,7 +518,7 @@ BackupHandler getBackupHandler() { // handleFinishedSnapshotTask RestoreJob restoreJob1 = (RestoreJob) handler.getJob(CatalogMocker.TEST_DB_ID); snapshotTask1 = new SnapshotTask(null, 0, 0, restoreJob1.getJobId(), CatalogMocker.TEST_DB_ID, - 0, 0, 0, 0, 0, 0, 1, true); + 0, 0, 0, 0, 0, 0, 1, true); request1 = new TFinishTaskRequest(); request1.setSnapshot_path("./snapshot/path1"); request1.setTask_status(new TStatus(TStatusCode.OK)); @@ -490,7 +526,7 @@ BackupHandler getBackupHandler() { // handleDownloadSnapshotTask DownloadTask downloadTask1 = new DownloadTask(null, 0, 0, restoreJob1.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath1, null, null); + srcToDestPath1, null, null); request1 = new TFinishTaskRequest(); List downloadedTabletIds1 = Lists.newArrayList(); request1.setDownloaded_tablet_ids(downloadedTabletIds1); @@ -499,7 +535,7 @@ BackupHandler getBackupHandler() { // handleDirMoveTask DirMoveTask dirMoveTask1 = new DirMoveTask(null, 0, 0, restoreJob1.getJobId(), CatalogMocker.TEST_DB_ID, 0, 0, 0, - 0, "", 0, true); + 0, "", 0, true); request1 = new TFinishTaskRequest(); request1.setTask_status(new TStatus(TStatusCode.OK)); handler.handleDirMoveTask(dirMoveTask1, request1); @@ -595,7 +631,6 @@ public void testSaveLoadJsonFormatImage() throws Exception { BackupJob runningJob = new BackupJob("running_job", 1, "test_db", new ArrayList<>(), 10000, globalStateMgr, 1); handler.dbIdToBackupOrRestoreJob.put(runningJob.getDbId(), runningJob); - UtFrameUtils.PseudoImage pseudoImage = new UtFrameUtils.PseudoImage(); handler.saveBackupHandlerV2(pseudoImage.getImageWriter()); BackupHandler followerHandler = new BackupHandler(globalStateMgr); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobMaterializedViewTest.java index dde16ba561502..70c48dc031ca7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobMaterializedViewTest.java @@ -122,7 +122,7 @@ public Repository getRepo(long repoId) { private EditLog editLog; private Repository repo = new Repository(repoId, "repo", false, "my_repo", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); @BeforeAll public static void start() { @@ -141,7 +141,7 @@ public static void end() throws IOException { File backupDir = new File(path.toString()); if (backupDir.exists()) { Files.walk(path, FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) - .forEach(File::delete); + .forEach(File::delete); } } } @@ -156,11 +156,11 @@ public void setUp() { Deencapsulation.setField(globalStateMgr, "backupHandler", backupHandler); db = UnitTestUtil.createDbWithMaterializedView(dbId, tblId, partId, idxId, tabletId, - backendId, version, KeysType.DUP_KEYS); + backendId, version, KeysType.DUP_KEYS); new Expectations(globalStateMgr) { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; @@ -171,6 +171,14 @@ public void setUp() { globalStateMgr.getEditLog(); minTimes = 0; result = editLog; + + globalStateMgr.getLocalMetastore().getTable("testDb", "unknown_mv"); + minTimes = 0; + result = null; + + globalStateMgr.getLocalMetastore().getTable("testDb", "unknown_tbl"); + minTimes = 0; + result = null; } }; @@ -239,14 +247,18 @@ public void testRunNormal() { List partNames = Lists.newArrayList(backupTbl.getPartitionNames()); Assert.assertNotNull(backupTbl); Assert.assertEquals(backupTbl.getSignature(BackupHandler.SIGNATURE_VERSION, partNames, true), - ((OlapTable) db.getTable(tblId)).getSignature(BackupHandler.SIGNATURE_VERSION, partNames, true)); + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tblId)).getSignature(BackupHandler.SIGNATURE_VERSION, partNames, + true)); } { OlapTable backupTbl = (OlapTable) backupMeta.getTable(UnitTestUtil.MATERIALIZED_VIEW_NAME); List partNames = Lists.newArrayList(backupTbl.getPartitionNames()); Assert.assertNotNull(backupTbl); Assert.assertEquals(backupTbl.getSignature(BackupHandler.SIGNATURE_VERSION, partNames, true), - ((OlapTable) db.getTable(tblId + 1)).getSignature(BackupHandler.SIGNATURE_VERSION, partNames, true)); + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tblId + 1)).getSignature(BackupHandler.SIGNATURE_VERSION, partNames, + true)); } } @@ -271,7 +283,7 @@ public void testRunNormal() { Assert.assertTrue(task instanceof SnapshotTask); SnapshotTask snapshotTask = (SnapshotTask) task; TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - snapshotTask.getSignature(), taskStatus); + snapshotTask.getSignature(), taskStatus); request.setSnapshot_files(snapshotFiles); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(snapshotTask, request)); @@ -284,7 +296,7 @@ public void testRunNormal() { Assert.assertTrue(task instanceof SnapshotTask); SnapshotTask snapshotTask = (SnapshotTask) task; TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - snapshotTask.getSignature(), taskStatus); + snapshotTask.getSignature(), taskStatus); request.setSnapshot_files(snapshotFiles); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(snapshotTask, request)); @@ -307,7 +319,7 @@ public void testRunNormal() { Assert.assertEquals(job.getJobId(), upTask.getJobId()); Map srcToDest = upTask.getSrcToDestPath(); Assert.assertEquals(1, srcToDest.size()); - String dest = srcToDest.get(snapshotPath + "/" + tabletId + "/" + 0); + String dest = srcToDest.get(snapshotPath + "/" + tabletId + "/" + 0); Assert.assertNotNull(dest); // 5. uploading @@ -316,7 +328,7 @@ public void testRunNormal() { Assert.assertEquals(BackupJobState.UPLOADING, job.getState()); Map> tabletFileMap = Maps.newHashMap(); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.UPLOAD, - upTask.getSignature(), taskStatus); + upTask.getSignature(), taskStatus); request.setTablet_files(tabletFileMap); Assert.assertFalse(job.finishSnapshotUploadTask(upTask, request)); @@ -354,16 +366,18 @@ public void testRunNormal() { Assert.assertNotNull(olapTable); Assert.assertNotNull(restoreMetaInfo.getTable(UnitTestUtil.TABLE_NAME)); List names = Lists.newArrayList(olapTable.getPartitionNames()); - Assert.assertEquals(((OlapTable) db.getTable(tblId)).getSignature(BackupHandler.SIGNATURE_VERSION, names, true), - olapTable.getSignature(BackupHandler.SIGNATURE_VERSION, names, true)); + Assert.assertEquals(((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tblId)).getSignature(BackupHandler.SIGNATURE_VERSION, names, true), + olapTable.getSignature(BackupHandler.SIGNATURE_VERSION, names, true)); } { MaterializedView mv = (MaterializedView) restoreMetaInfo.getTable(tblId + 1); Assert.assertNotNull(mv); Assert.assertNotNull(restoreMetaInfo.getTable(UnitTestUtil.MATERIALIZED_VIEW_NAME)); List names = Lists.newArrayList(mv.getPartitionNames()); - Assert.assertEquals(((OlapTable) db.getTable(tblId + 1)).getSignature(BackupHandler.SIGNATURE_VERSION, names, - true), mv.getSignature(BackupHandler.SIGNATURE_VERSION, names, true)); + Assert.assertEquals(((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tblId + 1)).getSignature(BackupHandler.SIGNATURE_VERSION, names, + true), mv.getSignature(BackupHandler.SIGNATURE_VERSION, names, true)); } restoreJobInfo = BackupJobInfo.fromFile(job.getLocalJobInfoFilePath()); @@ -387,7 +401,7 @@ public void testRunNormal() { Assert.assertTrue(mv != null); Assert.assertTrue(!mv.isActive()); Assert.assertTrue(mv.getInactiveReason().contains(String.format("Set the materialized view %s inactive in backup", - UnitTestUtil.MATERIALIZED_VIEW_NAME))); + UnitTestUtil.MATERIALIZED_VIEW_NAME))); } catch (IOException e) { e.printStackTrace(); Assert.fail(); @@ -417,7 +431,7 @@ public void testRunAbnormal() { tableRefs.add(new TableRef(new TableName(UnitTestUtil.DB_NAME, "unknown_mv"), null)); job = new BackupJob("mv_label_abnormal", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, - globalStateMgr, repo.getId()); + globalStateMgr, repo.getId()); job.run(); Assert.assertEquals(Status.ErrCode.NOT_FOUND, job.getStatus().getErrCode()); Assert.assertEquals(BackupJobState.CANCELLED, job.getState()); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobPrimaryKeyTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobPrimaryKeyTest.java index 5797d5551a2c9..72cc02e534fd9 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobPrimaryKeyTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobPrimaryKeyTest.java @@ -138,7 +138,7 @@ public Repository getRepo(long repoId) { private EditLog editLog; private Repository repo = new Repository(repoId, "repo_pk", false, "my_repo_pk", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); @BeforeClass public static void start() { @@ -157,7 +157,7 @@ public static void end() throws IOException { File backupDir = new File(path.toString()); if (backupDir.exists()) { Files.walk(path, FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) - .forEach(File::delete); + .forEach(File::delete); } } } @@ -172,11 +172,11 @@ public void setUp() { Deencapsulation.setField(globalStateMgr, "backupHandler", backupHandler); db = UnitTestUtil.createDbByName(dbId, tblId, partId, idxId, tabletId, backendId, version, KeysType.PRIMARY_KEYS, - testDbName, testTableName); + testDbName, testTableName); new Expectations(globalStateMgr) { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; @@ -187,6 +187,14 @@ public void setUp() { globalStateMgr.getEditLog(); minTimes = 0; result = editLog; + + globalStateMgr.getLocalMetastore().getTable(testDbName, testTableName); + minTimes = 0; + result = db.getTable(tblId); + + globalStateMgr.getLocalMetastore().getTable(testDbName, "unknown_tbl"); + minTimes = 0; + result = null; } }; @@ -261,7 +269,7 @@ public void testRunNormal() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - snapshotTask.getSignature(), taskStatus); + snapshotTask.getSignature(), taskStatus); request.setSnapshot_files(snapshotFiles); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(snapshotTask, request)); @@ -292,7 +300,7 @@ public void testRunNormal() { Assert.assertEquals(BackupJobState.UPLOADING, job.getState()); Map> tabletFileMap = Maps.newHashMap(); request = new TFinishTaskRequest(tBackend, TTaskType.UPLOAD, - upTask.getSignature(), taskStatus); + upTask.getSignature(), taskStatus); request.setTablet_files(tabletFileMap); Assert.assertFalse(job.finishSnapshotUploadTask(upTask, request)); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java index e13b9f86143bf..81e24d4739df9 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java @@ -133,7 +133,7 @@ public Repository getRepo(long repoId) { private EditLog editLog; private Repository repo = new Repository(repoId, "repo", false, "my_repo", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); @BeforeClass public static void start() { @@ -150,8 +150,8 @@ public static void end() throws IOException { File backupDir = new File(BackupHandler.BACKUP_ROOT_DIR.toString()); if (backupDir.exists()) { Files.walk(BackupHandler.BACKUP_ROOT_DIR, - FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) - .forEach(File::delete); + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile) + .forEach(File::delete); } } @@ -168,10 +168,12 @@ public void setUp() { new Expectations(globalStateMgr) { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; + + globalStateMgr.getNextId(); minTimes = 0; result = id.getAndIncrement(); @@ -179,6 +181,14 @@ public void setUp() { globalStateMgr.getEditLog(); minTimes = 0; result = editLog; + + globalStateMgr.getLocalMetastore().getTable("testDb", "testTable"); + minTimes = 0; + result = db.getTable(tblId); + + globalStateMgr.getLocalMetastore().getTable("testDb", "unknown_tbl"); + minTimes = 0; + result = null; } }; @@ -253,7 +263,7 @@ public void testRunNormal() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - snapshotTask.getSignature(), taskStatus); + snapshotTask.getSignature(), taskStatus); request.setSnapshot_files(snapshotFiles); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(snapshotTask, request)); @@ -284,7 +294,7 @@ public void testRunNormal() { Assert.assertEquals(BackupJobState.UPLOADING, job.getState()); Map> tabletFileMap = Maps.newHashMap(); request = new TFinishTaskRequest(tBackend, TTaskType.UPLOAD, - upTask.getSignature(), taskStatus); + upTask.getSignature(), taskStatus); request.setTablet_files(tabletFileMap); Assert.assertFalse(job.finishSnapshotUploadTask(upTask, request)); @@ -345,7 +355,8 @@ public void testRunNormal() { try { // test get backup info job.getInfo(); - } catch (Exception ignore) { } + } catch (Exception ignore) { + } } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java index 349c39815edfe..997d26e21f9ef 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java @@ -531,19 +531,19 @@ public static GlobalStateMgr fetchAdminCatalog() { new Expectations(globalStateMgr) { { - globalStateMgr.getDb(TEST_DB_NAME); + globalStateMgr.getLocalMetastore().getDb(TEST_DB_NAME); minTimes = 0; result = db; - globalStateMgr.getDb(WRONG_DB); + globalStateMgr.getLocalMetastore().getDb(WRONG_DB); minTimes = 0; result = null; - globalStateMgr.getDb(TEST_DB_ID); + globalStateMgr.getLocalMetastore().getDb(TEST_DB_ID); minTimes = 0; result = db; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = new Database(); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java index 462503dfa5727..d0aab1e9fa861 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java @@ -44,6 +44,7 @@ import com.starrocks.persist.EditLog; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.server.MetadataMgr; import com.starrocks.system.NodeSelector; import com.starrocks.system.SystemInfoService; @@ -85,7 +86,6 @@ import static com.starrocks.common.util.UnitTestUtil.MATERIALIZED_VIEW_NAME; import static com.starrocks.common.util.UnitTestUtil.TABLE_NAME; - @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RestoreJobMaterializedViewTest { @@ -113,7 +113,6 @@ public class RestoreJobMaterializedViewTest { private MvRestoreContext mvRestoreContext; - // Thread is not mockable in Jmockit, use subclass instead private final class MockBackupHandler extends BackupHandler { public MockBackupHandler(GlobalStateMgr globalStateMgr) { @@ -147,11 +146,12 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); private BackupMeta backupMeta; private Object[] arrayIds; + private void setUpMocker() { MetricRepo.init(); @@ -164,19 +164,27 @@ private void setUpMocker() { new Expectations() { { - globalStateMgr.getDb(anyLong); + GlobalStateMgr.getCurrentState(); + minTimes = 0; + result = globalStateMgr; + + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; + globalStateMgr.getLocalMetastore().getTable(UnitTestUtil.DB_NAME, MATERIALIZED_VIEW_NAME); + minTimes = 0; + result = db.getTable(MATERIALIZED_VIEW_NAME); + globalStateMgr.getEditLog(); minTimes = 0; result = editLog; - GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); - minTimes = 0; - result = systemInfoService; + //GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); + //minTimes = 0; + //result = systemInfoService; - globalStateMgr.mayGetDb(anyLong); + globalStateMgr.getLocalMetastore().mayGetDb(anyLong); minTimes = 0; result = Optional.of(db); @@ -225,24 +233,24 @@ public Status getSnapshotMetaFile(String label, List backupMetas) { } }; new MockUp() { - @Mock - boolean await(long timeout, TimeUnit unit) { - return true; - } - }; + @Mock + boolean await(long timeout, TimeUnit unit) { + return true; + } + }; new MockUp() { - @Mock - GlobalStateMgr getGlobalStateMgr() { - return globalStateMgr; - } - }; + @Mock + GlobalStateMgr getGlobalStateMgr() { + return globalStateMgr; + } + }; new MockUp() { - @Mock - BackupHandler getBackupHandler() { - return backupHandler; - } - }; + @Mock + BackupHandler getBackupHandler() { + return backupHandler; + } + }; new MockUp() { @Mock @@ -259,7 +267,7 @@ public void setUp() throws AnalysisException { Deencapsulation.setField(globalStateMgr, "backupHandler", backupHandler); systemInfoService = new SystemInfoService(); db = UnitTestUtil.createDbWithMaterializedView(dbId, tblId, partId, idxId, tabletId, - backendId, version, KeysType.DUP_KEYS); + backendId, version, KeysType.DUP_KEYS); mvRestoreContext = new MvRestoreContext(); setUpMocker(); } @@ -315,8 +323,8 @@ public RestoreJob createRestoreJob(List restoreTbls) { } RestoreJob job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), - jobInfo, false, 3, 100000, - globalStateMgr, repo.getId(), backupMeta, mvRestoreContext); + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, mvRestoreContext); job.setRepo(repo); // add job into mvRestoreContext @@ -358,7 +366,7 @@ private void checkJobRun(RestoreJob job) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - task.getSignature(), taskStatus); + task.getSignature(), taskStatus); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(task, request)); } @@ -391,7 +399,7 @@ private void checkJobRun(RestoreJob job) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); request.setDownloaded_tablet_ids(downloadedTabletIds); Assert.assertTrue(job.finishTabletDownloadTask((DownloadTask) agentTask, request)); } @@ -423,7 +431,7 @@ private void checkJobRun(RestoreJob job) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); job.finishDirMoveTask((DirMoveTask) agentTask, request); } @@ -439,6 +447,24 @@ private void checkJobRun(RestoreJob job) { @Order(1) public void testMVRestore_TestOneTable1() { RestoreJob job = createRestoreJob(ImmutableList.of(UnitTestUtil.MATERIALIZED_VIEW_NAME)); + + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + }; + checkJobRun(job); assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true); } @@ -455,6 +481,22 @@ public void testMVRestore_TestOneTable2() { public void testMVRestore_TestMVWithBaseTable1() { // gen BackupJobInfo RestoreJob job = createRestoreJob(ImmutableList.of(TABLE_NAME, MATERIALIZED_VIEW_NAME)); + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + }; // backup & restore checkJobRun(job); assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true); @@ -465,6 +507,22 @@ public void testMVRestore_TestMVWithBaseTable1() { public void testMVRestore_TestMVWithBaseTable2() { // gen BackupJobInfo RestoreJob job = createRestoreJob(ImmutableList.of(MATERIALIZED_VIEW_NAME, TABLE_NAME)); + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + }; // backup & restore checkJobRun(job); assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true); @@ -489,11 +547,27 @@ public void testMVRestore_TestMVWithBaseTable4() { new MockUp() { @Mock public Table getTable(String catalogName, String dbName, String tblName) { - return db.getTable(tblName); + return GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName); } }; // gen BackupJobInfo RestoreJob job1 = createRestoreJob(ImmutableList.of(MATERIALIZED_VIEW_NAME)); + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + }; // backup & restore checkJobRun(job1); assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true); @@ -504,7 +578,7 @@ public Table getTable(String catalogName, String dbName, String tblName) { } private void assertMVActiveEquals(String mvName, boolean expect) { - Table mvTable = db.getTable(mvName); + Table mvTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName); Assert.assertTrue(mvTable != null); Assert.assertTrue(mvTable.isMaterializedView()); MaterializedView mv = (MaterializedView) mvTable; diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java index 37f167eb02844..947e2f8231bbe 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobPrimaryKeyTest.java @@ -137,7 +137,7 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); private BackupMeta backupMeta; @@ -151,10 +151,6 @@ public void setUp() throws AnalysisException { new Expectations() { { - globalStateMgr.getDb(anyLong); - minTimes = 0; - result = db; - globalStateMgr.getNextId(); minTimes = 0; result = id.getAndIncrement(); @@ -264,8 +260,8 @@ boolean await(long timeout, TimeUnit unit) { tbls.add(expectedRestoreTbl); backupMeta = new BackupMeta(tbls); job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), - jobInfo, false, 3, 100000, - globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); } @Ignore @@ -301,7 +297,7 @@ public void testRun() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - task.getSignature(), taskStatus); + task.getSignature(), taskStatus); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(task, request)); } @@ -334,7 +330,7 @@ public void testRun() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); request.setDownloaded_tablet_ids(downloadedTabletIds); Assert.assertTrue(job.finishTabletDownloadTask((DownloadTask) agentTask, request)); } @@ -366,7 +362,7 @@ public void testRun() { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - agentTask.getSignature(), taskStatus); + agentTask.getSignature(), taskStatus); job.finishDirMoveTask((DirMoveTask) agentTask, request); } diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java index 323c93e3dee46..1ed06f0ce9df8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java @@ -61,6 +61,7 @@ import com.starrocks.metric.MetricRepo; import com.starrocks.persist.EditLog; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.system.SystemInfoService; import com.starrocks.task.AgentTask; import com.starrocks.task.AgentTaskQueue; @@ -138,7 +139,7 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - new BlobStorage("broker", Maps.newHashMap())); + new BlobStorage("broker", Maps.newHashMap())); private BackupMeta backupMeta; @@ -155,12 +156,12 @@ public void testResetPartitionForRestore() { expectedRestoreTbl = (OlapTable) db.getTable(CatalogMocker.TEST_TBL4_ID); OlapTable localTbl = new OlapTable(expectedRestoreTbl.getId(), expectedRestoreTbl.getName(), - expectedRestoreTbl.getBaseSchema(), KeysType.DUP_KEYS, expectedRestoreTbl.getPartitionInfo(), - expectedRestoreTbl.getDefaultDistributionInfo()); + expectedRestoreTbl.getBaseSchema(), KeysType.DUP_KEYS, expectedRestoreTbl.getPartitionInfo(), + expectedRestoreTbl.getDefaultDistributionInfo()); job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), - jobInfo, false, 3, 100000, - globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); job.resetPartitionForRestore(localTbl, expectedRestoreTbl, CatalogMocker.TEST_PARTITION1_NAME, 3); } @@ -169,7 +170,7 @@ public void testResetPartitionForRestore() { public void testRunBackupMultiSubPartitionTable() { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; @@ -287,12 +288,29 @@ boolean await(long timeout, TimeUnit unit) { // drop this table, cause we want to try restoring this table db.dropTable(expectedRestoreTbl.getName()); + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + }; + List
tbls = Lists.newArrayList(); tbls.add(expectedRestoreTbl); backupMeta = new BackupMeta(tbls); job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), - jobInfo, false, 3, 100000, - globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); job.setRepo(repo); // pending job.run(); @@ -324,7 +342,7 @@ boolean await(long timeout, TimeUnit unit) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - task.getSignature(), taskStatus); + task.getSignature(), taskStatus); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(task, request)); } @@ -344,7 +362,7 @@ boolean await(long timeout, TimeUnit unit) { public void testRunBackupRangeTable() { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; @@ -454,12 +472,29 @@ boolean await(long timeout, TimeUnit unit) { // drop this table, cause we want to try restoring this table db.dropTable(expectedRestoreTbl.getName()); + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + }; + List
tbls = Lists.newArrayList(); tbls.add(expectedRestoreTbl); backupMeta = new BackupMeta(tbls); job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), - jobInfo, false, 3, 100000, - globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); job.setRepo(repo); // pending job.run(); @@ -491,7 +526,7 @@ boolean await(long timeout, TimeUnit unit) { TStatus taskStatus = new TStatus(TStatusCode.OK); TBackend tBackend = new TBackend("", 0, 1); TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, - task.getSignature(), taskStatus); + task.getSignature(), taskStatus); request.setSnapshot_path(snapshotPath); Assert.assertTrue(job.finishTabletSnapshotTask(task, request)); } @@ -536,7 +571,7 @@ public void testColocateRestore() { { try { GlobalStateMgr.getCurrentState().getColocateTableIndex() - .addTableToGroup((Database) any, (OlapTable) any, (String) any, false); + .addTableToGroup((Database) any, (OlapTable) any, (String) any, false); } catch (Exception e) { } result = true; diff --git a/fe/fe-core/src/test/java/com/starrocks/benchmark/MvRefreshConcurrencyTest.java b/fe/fe-core/src/test/java/com/starrocks/benchmark/MvRefreshConcurrencyTest.java index 4e7413a95828d..09591cc5a9cdb 100644 --- a/fe/fe-core/src/test/java/com/starrocks/benchmark/MvRefreshConcurrencyTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/benchmark/MvRefreshConcurrencyTest.java @@ -123,9 +123,9 @@ private void testRefreshWithConcurrency(int tableNum, int mvNum) { System.out.println(sql); starRocksAssert.withMaterializedView(sql); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String mvName = buildMVName(i); - Table table = db.getTable(mvName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName); Assert.assertTrue(table != null); mvs.add((MaterializedView) table); starRocksAssert.useDatabase("test"); diff --git a/fe/fe-core/src/test/java/com/starrocks/binlog/BinlogManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/binlog/BinlogManagerTest.java index 2b1c482f760ae..5dbf5cd46da9d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/binlog/BinlogManagerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/binlog/BinlogManagerTest.java @@ -55,8 +55,9 @@ public static void beforeClass() throws Exception { @Test public void testTryEnableBinlog() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("binlog_test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "binlog_test"); boolean result = binlogManager.tryEnableBinlog(db, table.getId(), 200L, -1L); Assert.assertTrue(result); Assert.assertEquals(1, table.getBinlogVersion()); @@ -68,16 +69,18 @@ public void testTryEnableBinlog() { @Test public void testTryDisableBinlog() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("binlog_test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "binlog_test"); boolean result = binlogManager.tryDisableBinlog(db, table.getId()); Assert.assertFalse(table.isBinlogEnabled()); } @Test public void testCheckAndSetBinlogAvailableVersion() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("binlog_test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "binlog_test"); table.setBinlogTxnId(2); long totalNum = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletIdsByBackendId(10001).size(); binlogManager.checkAndSetBinlogAvailableVersion(db, table, 1, 10002); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java index 2b58f35ccc7ed..7618278e8876e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/AdminStmtTest.java @@ -86,9 +86,9 @@ public static void beforeClass() throws Exception { @Test public void testAdminSetReplicaStatus() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Assert.assertNotNull(db); - OlapTable tbl = (OlapTable) db.getTable("tbl1"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Assert.assertNotNull(tbl); // tablet id, backend id List> tabletToBackendList = Lists.newArrayList(); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java index a7a4f0e06ca26..3f63a7dd47abe 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinLakeTableTest.java @@ -57,8 +57,8 @@ public static void beforeClass() { private static Table createTable(ConnectContext connectContext, String sql) throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().createTable(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb(createTableStmt.getDbName()); - return db.getTable(createTableStmt.getTableName()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(createTableStmt.getDbName()); + return GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), createTableStmt.getTableName()); } private static void dropTable(ConnectContext connectContext, String sql) throws Exception { @@ -215,7 +215,7 @@ public LakeService getLakeService(TNetworkAddress address) throws RpcException { // Recover table2 Assert.assertTrue(recycleBin.recoverTable(db, "t0")); - Assert.assertSame(table2, db.getTable("t0")); + Assert.assertSame(table2, GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0")); Assert.assertNull(recycleBin.getTable(db.getId(), table2.getId())); checkTableTablet(table2, true); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableIndexTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableIndexTest.java index 23a6387ee0219..f7b9913933e0d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableIndexTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableIndexTest.java @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.catalog; import com.starrocks.lake.LakeTable; @@ -58,13 +57,13 @@ public void teardown() { /** * [ - * [10002.10006, 10002_group1, 10004, 10016, 4, 1, int(11), true], - * [10026.10030, 10026_group2, 10028, 4, 1, int(11), true] - * ] + * [10002.10006, 10002_group1, 10004, 10016, 4, 1, int(11), true], + * [10026.10030, 10026_group2, 10028, 4, 1, int(11), true] + * ] * -> * { - * 'group1': [10002.10006, 10002_group1, 10004, 10016, 4, 1, int(11), true], - * 'group2': [10026.10030, 10026_group2, 10028, 4, 1, int(11), true] + * 'group1': [10002.10006, 10002_group1, 10004, 10016, 4, 1, int(11), true], + * 'group2': [10026.10030, 10026_group2, 10028, 4, 1, int(11), true] * } */ private Map> groupByName(List> lists) { @@ -86,33 +85,33 @@ public void testDropTable() throws Exception { // create table1_1->group1 String sql = "CREATE TABLE db1.table1_1 (k1 int, k2 int, k3 varchar(32))\n" + - "PRIMARY KEY(k1)\n" + - "DISTRIBUTED BY HASH(k1)\n" + - "BUCKETS 4\n" + - "PROPERTIES(\"colocate_with\"=\"group1\", \"replication_num\" = \"1\");\n"; + "PRIMARY KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "BUCKETS 4\n" + + "PROPERTIES(\"colocate_with\"=\"group1\", \"replication_num\" = \"1\");\n"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StarRocksAssert.utCreateTableWithRetry(createTableStmt); List> infos = GlobalStateMgr.getCurrentState().getColocateTableIndex().getInfos(); // group1->table1To1 Assert.assertEquals(1, infos.size()); Map> map = groupByName(infos); - Table table1To1 = GlobalStateMgr.getCurrentState().getDb("db1").getTable("table1_1"); + Table table1To1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1").getTable("table1_1"); Assert.assertEquals(String.format("%d", table1To1.getId()), map.get("group1").get(2)); LOG.info("after create db1.table1_1: {}", infos); // create table1_2->group1 sql = "CREATE TABLE db1.table1_2 (k1 int, k2 int, k3 varchar(32))\n" + - "PRIMARY KEY(k1)\n" + - "DISTRIBUTED BY HASH(k1)\n" + - "BUCKETS 4\n" + - "PROPERTIES(\"colocate_with\"=\"group1\", \"replication_num\" = \"1\");\n"; + "PRIMARY KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "BUCKETS 4\n" + + "PROPERTIES(\"colocate_with\"=\"group1\", \"replication_num\" = \"1\");\n"; createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StarRocksAssert.utCreateTableWithRetry(createTableStmt); // group1 -> table1To1, table1To2 infos = GlobalStateMgr.getCurrentState().getColocateTableIndex().getInfos(); Assert.assertEquals(1, infos.size()); map = groupByName(infos); - Table table1To2 = GlobalStateMgr.getCurrentState().getDb("db1").getTable("table1_2"); + Table table1To2 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1").getTable("table1_2"); Assert.assertEquals(String.format("%d, %d", table1To1.getId(), table1To2.getId()), map.get("group1").get(2)); LOG.info("after create db1.table1_2: {}", infos); @@ -122,10 +121,10 @@ public void testDropTable() throws Exception { GlobalStateMgr.getCurrentState().getMetadata().createDb(createDbStmt.getFullDbName()); // create table2_1 -> group2 sql = "CREATE TABLE db2.table2_1 (k1 int, k2 int, k3 varchar(32))\n" + - "PRIMARY KEY(k1)\n" + - "DISTRIBUTED BY HASH(k1)\n" + - "BUCKETS 4\n" + - "PROPERTIES(\"colocate_with\"=\"group2\", \"replication_num\" = \"1\");\n"; + "PRIMARY KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "BUCKETS 4\n" + + "PROPERTIES(\"colocate_with\"=\"group2\", \"replication_num\" = \"1\");\n"; createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StarRocksAssert.utCreateTableWithRetry(createTableStmt); // group1 -> table1_1, table1_2 @@ -134,7 +133,7 @@ public void testDropTable() throws Exception { Assert.assertEquals(2, infos.size()); map = groupByName(infos); Assert.assertEquals(String.format("%d, %d", table1To1.getId(), table1To2.getId()), map.get("group1").get(2)); - Table table2To1 = GlobalStateMgr.getCurrentState().getDb("db2").getTable("table2_1"); + Table table2To1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db2").getTable("table2_1"); Assert.assertEquals(String.format("%d", table2To1.getId()), map.get("group2").get(2)); LOG.info("after create db2.table2_1: {}", infos); @@ -162,12 +161,11 @@ public void testDropTable() throws Exception { Assert.assertEquals(2, infos.size()); Assert.assertEquals(String.format("%d*, %d*", table1To1.getId(), table1To2.getId()), map.get("group1").get(2)); Assert.assertEquals(String.format("[deleted], [deleted]", table1To1.getId(), table1To2.getId()), - map.get("group1").get(3)); + map.get("group1").get(3)); Assert.assertEquals(String.format("%d", table2To1.getId()), map.get("group2").get(2)); Assert.assertEquals(String.format("table2_1", table2To1.getId()), map.get("group2").get(3)); - LOG.info("after drop db1.table1_2: {}", infos); // drop db2 @@ -189,13 +187,13 @@ public void testDropTable() throws Exception { GlobalStateMgr.getCurrentState().getMetadata().createDb(createDbStmt.getFullDbName()); // create table2_1 -> group2 sql = "CREATE TABLE db2.table2_3 (k1 int, k2 int, k3 varchar(32))\n" + - "PRIMARY KEY(k1)\n" + - "DISTRIBUTED BY HASH(k1)\n" + - "BUCKETS 4\n" + - "PROPERTIES(\"colocate_with\"=\"group3\", \"replication_num\" = \"1\");\n"; + "PRIMARY KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "BUCKETS 4\n" + + "PROPERTIES(\"colocate_with\"=\"group3\", \"replication_num\" = \"1\");\n"; createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Table table2To3 = GlobalStateMgr.getCurrentState().getDb("db2").getTable("table2_3"); + Table table2To3 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db2").getTable("table2_3"); sql = "DROP DATABASE db2;"; dropDbStmt = (DropDbStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getMetadata().dropDb(dropDbStmt.getDbName(), dropDbStmt.isForceDrop()); @@ -207,10 +205,10 @@ public void testDropTable() throws Exception { Assert.assertEquals("[deleted], [deleted]", map.get("group1").get(3)); Assert.assertEquals(String.format("%d*", table2To1.getId()), map.get("group2").get(2)); Assert.assertEquals(String.format("[deleted], [deleted]", table1To1.getId(), table1To2.getId()), - map.get("group1").get(3)); + map.get("group1").get(3)); Assert.assertEquals(String.format("%d*", table2To3.getId()), map.get("group3").get(2)); Assert.assertEquals(String.format("[deleted], [deleted]", table1To1.getId(), table1To2.getId()), - map.get("group1").get(3)); + map.get("group1").get(3)); } @Test @@ -221,31 +219,31 @@ public void testCleanUp() throws Exception { // create goodDb CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseStmtWithNewParser("create database goodDb;", connectContext); GlobalStateMgr.getCurrentState().getMetadata().createDb(createDbStmt.getFullDbName()); - Database goodDb = GlobalStateMgr.getCurrentState().getDb("goodDb"); + Database goodDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("goodDb"); // create goodtable String sql = "CREATE TABLE " + - "goodDb.goodTable (k1 int, k2 int, k3 varchar(32))\n" + - "PRIMARY KEY(k1)\n" + - "DISTRIBUTED BY HASH(k1)\n" + - "BUCKETS 4\n" + - "PROPERTIES(\"colocate_with\"=\"goodGroup\", \"replication_num\" = \"1\");\n"; + "goodDb.goodTable (k1 int, k2 int, k3 varchar(32))\n" + + "PRIMARY KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "BUCKETS 4\n" + + "PROPERTIES(\"colocate_with\"=\"goodGroup\", \"replication_num\" = \"1\");\n"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - OlapTable table = (OlapTable) goodDb.getTable("goodTable"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(goodDb.getFullName(), "goodTable"); ColocateTableIndex.GroupId goodGroup = GlobalStateMgr.getCurrentState().getColocateTableIndex().getGroup(table.getId()); - // create a bad db long badDbId = 4000; table.id = 4001; table.name = "goodTableOfBadDb"; colocateTableIndex.addTableToGroup( - badDbId, table, "badGroupOfBadDb", new ColocateTableIndex.GroupId(badDbId, 4002), false); + badDbId, table, "badGroupOfBadDb", new ColocateTableIndex.GroupId(badDbId, 4002), false); // create a bad table in good db table.id = 4003; table.name = "badTable"; colocateTableIndex.addTableToGroup( - goodDb.getId(), table, "badGroupOfBadTable", new ColocateTableIndex.GroupId(goodDb.getId(), 4004), false); + goodDb.getId(), table, "badGroupOfBadTable", new ColocateTableIndex.GroupId(goodDb.getId(), 4004), false); Map> map = groupByName(GlobalStateMgr.getCurrentState().getColocateTableIndex().getInfos()); Assert.assertTrue(map.containsKey("goodGroup")); @@ -299,10 +297,11 @@ public List getShardGroupIds() { }; colocateTableIndex.addTableToGroup( - dbId, (OlapTable) olapTable, "lakeGroup", new ColocateTableIndex.GroupId(dbId, 10000), false /* isReplay */); + dbId, (OlapTable) olapTable, "lakeGroup", new ColocateTableIndex.GroupId(dbId, 10000), false /* isReplay */); Assert.assertTrue(colocateTableIndex.isLakeColocateTable(tableId)); colocateTableIndex.addTableToGroup( - dbId2, (OlapTable) olapTable, "lakeGroup", new ColocateTableIndex.GroupId(dbId2, 10000), false /* isReplay */); + dbId2, (OlapTable) olapTable, "lakeGroup", new ColocateTableIndex.GroupId(dbId2, 10000), + false /* isReplay */); Assert.assertTrue(colocateTableIndex.isLakeColocateTable(tableId)); Assert.assertTrue(colocateTableIndex.isGroupUnstable(new ColocateTableIndex.GroupId(dbId, 10000))); @@ -319,19 +318,19 @@ public void testSaveLoadJsonFormatImage() throws Exception { // create goodDb CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils - .parseStmtWithNewParser("create database db_image;", connectContext); + .parseStmtWithNewParser("create database db_image;", connectContext); GlobalStateMgr.getCurrentState().getMetadata().createDb(createDbStmt.getFullDbName()); - Database db = GlobalStateMgr.getCurrentState().getDb("db_image"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db_image"); // create goodtable String sql = "CREATE TABLE " + - "db_image.tbl1 (k1 int, k2 int, k3 varchar(32))\n" + - "PRIMARY KEY(k1)\n" + - "DISTRIBUTED BY HASH(k1)\n" + - "BUCKETS 4\n" + - "PROPERTIES(\"colocate_with\"=\"goodGroup\", \"replication_num\" = \"1\");\n"; + "db_image.tbl1 (k1 int, k2 int, k3 varchar(32))\n" + + "PRIMARY KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1)\n" + + "BUCKETS 4\n" + + "PROPERTIES(\"colocate_with\"=\"goodGroup\", \"replication_num\" = \"1\");\n"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - OlapTable table = (OlapTable) db.getTable("tbl1"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); UtFrameUtils.PseudoImage image = new UtFrameUtils.PseudoImage(); colocateTableIndex.saveColocateTableIndexV2(image.getImageWriter()); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableTest.java index 028496b8f1771..8e9b33f90d3cd 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableTest.java @@ -110,8 +110,8 @@ public void testCreateOneTable() throws Exception { ");"); ColocateTableIndex index = GlobalStateMgr.getCurrentState().getColocateTableIndex(); - Database db = GlobalStateMgr.getCurrentState().getDb(fullDbName); - long tableId = db.getTable(tableName1).getId(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(fullDbName); + long tableId = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName1).getId(); Assert.assertEquals(1, Deencapsulation.>getField(index, "group2Tables").size()); Assert.assertEquals(1, index.getAllGroupIds().size()); @@ -168,9 +168,9 @@ public void testCreateTwoTableWithSameGroup() throws Exception { ");"); ColocateTableIndex index = GlobalStateMgr.getCurrentState().getColocateTableIndex(); - Database db = GlobalStateMgr.getCurrentState().getDb(fullDbName); - long firstTblId = db.getTable(tableName1).getId(); - long secondTblId = db.getTable(tableName2).getId(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(fullDbName); + long firstTblId = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName1).getId(); + long secondTblId = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName2).getId(); Assert.assertEquals(2, Deencapsulation.>getField(index, "group2Tables").size()); Assert.assertEquals(1, index.getAllGroupIds().size()); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableLikeTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableLikeTest.java index db0d562b71077..108d215e364ff 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableLikeTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableLikeTest.java @@ -76,7 +76,7 @@ private static void createTable(String sql) throws Exception { private static void createTableLike(String sql) throws Exception { CreateTableLikeStmt createTableLikeStmt = - (CreateTableLikeStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + (CreateTableLikeStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); GlobalStateMgr.getCurrentState().getLocalMetastore().createTable(createTableLikeStmt.getCreateTableStmt()); } @@ -86,7 +86,7 @@ private static void checkTableEqual(Table newTable, Table existedTable) { List existedTableStmt = Lists.newArrayList(); AstToStringBuilder.getDdlStmt(existedTable, existedTableStmt, null, null, false, true /* hide password */); Assert.assertEquals(newCreateTableStmt.get(0).replace(newTable.getName(), existedTable.getName()), - existedTableStmt.get(0)); + existedTableStmt.get(0)); } private static void checkCreateOlapTableLike(String createTableSql, String createTableLikeSql, @@ -94,10 +94,12 @@ private static void checkCreateOlapTableLike(String createTableSql, String creat String newTblName, String existedTblName) throws Exception { createTable(createTableSql); createTableLike(createTableLikeSql); - Database newDb = GlobalStateMgr.getCurrentState().getDb(newDbName); - Database existedDb = GlobalStateMgr.getCurrentState().getDb(existedDbName); - OlapTable newTbl = (OlapTable) newDb.getTable(newTblName); - OlapTable existedTbl = (OlapTable) existedDb.getTable(existedTblName); + Database newDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(newDbName); + Database existedDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(existedDbName); + OlapTable newTbl = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(newDb.getFullName(), newTblName); + OlapTable existedTbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(existedDb.getFullName(), existedTblName); checkTableEqual(newTbl, existedTbl); } @@ -107,10 +109,12 @@ private static void checkCreateMysqlTableLike(String createTableSql, String crea createTable(createTableSql); createTableLike(createTableLikeSql); - Database newDb = GlobalStateMgr.getCurrentState().getDb(newDbName); - Database existedDb = GlobalStateMgr.getCurrentState().getDb(existedDbName); - MysqlTable newTbl = (MysqlTable) newDb.getTable(newTblName); - MysqlTable existedTbl = (MysqlTable) existedDb.getTable(existedTblName); + Database newDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(newDbName); + Database existedDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(existedDbName); + MysqlTable newTbl = + (MysqlTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(newDb.getFullName(), newTblName); + MysqlTable existedTbl = (MysqlTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(existedDb.getFullName(), existedTblName); checkTableEqual(newTbl, existedTbl); } @@ -118,7 +122,7 @@ private static void checkCreateMysqlTableLike(String createTableSql, String crea public void testNormal() throws Exception { // 1. creat table with single partition String createTableSql = "create table test.testTbl1\n" + "(k1 int, k2 int)\n" + "duplicate key(k1)\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; String createTableLikeSql = "create table test.testTbl1_like like test.testTbl1"; String newDbName = "test"; String newTblName = "testTbl1_like"; @@ -126,157 +130,158 @@ public void testNormal() throws Exception { checkCreateOlapTableLike(createTableSql, createTableLikeSql, newDbName, newDbName, newTblName, existedTblName); // 2. create table with hash partition String createTableWithHashPartitionSql = "create table test.testTbl2\n" + "(k1 int, k2 int)\n" - + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; + + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; String createTableLikeSql2 = "create table test.testTbl2_like like test.testTbl2"; String newDbName2 = "test"; String newTblName2 = "testTbl2_like"; String existedTblName2 = "testTbl2"; checkCreateOlapTableLike(createTableWithHashPartitionSql, createTableLikeSql2, newDbName2, newDbName2, - newTblName2, existedTblName2); + newTblName2, existedTblName2); // 3. create aggregate table String createAggTableSql3 = "create table test.testTbl3\n" + "(k1 varchar(40), k2 int, v1 int sum)\n" - + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" - + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1');"; + + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1');"; String createTableLikeSql3 = "create table test.testTbl3_like like test.testTbl3"; String newDbName3 = "test"; String newTblName3 = "testTbl3_like"; String existedTblName3 = "testTbl3"; checkCreateOlapTableLike(createAggTableSql3, createTableLikeSql3, newDbName3, newDbName3, newTblName3, - existedTblName3); + existedTblName3); // 4. create aggregate table without partition String createAggTableWithoutPartitionSql4 = - "create table test.testTbl4\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" - + "partition by range(k2)\n" + "()\n" - + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1');"; + "create table test.testTbl4\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" + + "partition by range(k2)\n" + "()\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1');"; String createTableLikeSql4 = "create table test.testTbl4_like like test.testTbl4"; String newDbName4 = "test"; String newTblName4 = "testTbl4_like"; String existedTblName4 = "testTbl4"; checkCreateOlapTableLike(createAggTableWithoutPartitionSql4, createTableLikeSql4, newDbName4, newDbName4, - newTblName4, existedTblName4); + newTblName4, existedTblName4); // 5. create table from different db String createTableFromDiffDb5 = - "create table test.testTbl5\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" - + "partition by range(k2)\n" + "()\n" - + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1');"; + "create table test.testTbl5\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" + + "partition by range(k2)\n" + "()\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1');"; String createTableLikeSql5 = "create table test2.testTbl5_like like test.testTbl5"; String newDbName5 = "test2"; String existedDbName5 = "test"; String newTblName5 = "testTbl5_like"; String existedTblName5 = "testTbl5"; checkCreateOlapTableLike(createTableFromDiffDb5, createTableLikeSql5, newDbName5, existedDbName5, newTblName5, - existedTblName5); + existedTblName5); // 6. create table from dynamic partition table String createDynamicTblSql = "CREATE TABLE test.`dynamic_partition_normal` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"false\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"false\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; String createTableLikeSql6 = - "create table test.dynamic_partition_normal_like like test.dynamic_partition_normal"; + "create table test.dynamic_partition_normal_like like test.dynamic_partition_normal"; String newDbName6 = "test"; String newTblName6 = "dynamic_partition_normal_like"; String existedTblName6 = "dynamic_partition_normal"; checkCreateOlapTableLike(createDynamicTblSql, createTableLikeSql6, newDbName6, newDbName6, newTblName6, - existedTblName6); + existedTblName6); // 7. create table from colocate table String createColocateTblSql = "create table test.colocateTbl (\n" + - " `k1` int NULL COMMENT \"\",\n" + - " `k2` varchar(10) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\",\n" + - " \"colocate_with\" = \"test_group\"\n" + - ");"; + " `k1` int NULL COMMENT \"\",\n" + + " `k2` varchar(10) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"colocate_with\" = \"test_group\"\n" + + ");"; String createTableLikeSql7 = "create table test.colocateTbl_like like test.colocateTbl"; String newDbName7 = "test"; String newTblName7 = "colocateTbl_like"; String existedTblName7 = "colocateTbl"; checkCreateOlapTableLike(createColocateTblSql, createTableLikeSql7, newDbName7, newDbName7, newTblName7, - existedTblName7); + existedTblName7); // 8. creat non-OLAP table String createNonOlapTableSql = "create table test.testMysqlTbl\n" + - "(k1 DATE, k2 INT, k3 SMALLINT, k4 VARCHAR(2048), k5 DATETIME)\n" + - "ENGINE=mysql\nPROPERTIES(\n" + - "\"host\" = \"127.0.0.1\",\n" + - "\"port\" = \"8239\",\n" + - "\"user\" = \"mysql_passwd\",\n" + - "\"password\" = \"mysql_passwd\",\n" + - "\"database\" = \"mysql_db_test\",\n" + - "\"table\" = \"mysql_table_test\");"; + "(k1 DATE, k2 INT, k3 SMALLINT, k4 VARCHAR(2048), k5 DATETIME)\n" + + "ENGINE=mysql\nPROPERTIES(\n" + + "\"host\" = \"127.0.0.1\",\n" + + "\"port\" = \"8239\",\n" + + "\"user\" = \"mysql_passwd\",\n" + + "\"password\" = \"mysql_passwd\",\n" + + "\"database\" = \"mysql_db_test\",\n" + + "\"table\" = \"mysql_table_test\");"; String createTableLikeSql8 = "create table test.testMysqlTbl_like like test.testMysqlTbl"; String newDbName8 = "test"; String existedDbName8 = "test"; String newTblName8 = "testMysqlTbl_like"; String existedTblName8 = "testMysqlTbl"; checkCreateMysqlTableLike(createNonOlapTableSql, createTableLikeSql8, newDbName8, existedDbName8, newTblName8, - existedTblName8); + existedTblName8); // 9. create automatic table String automaticTableSql = "CREATE TABLE test.`duplicate_table_with_null` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` datetime NULL COMMENT \"\",\n" + - " `k3` char(20) NULL COMMENT \"\",\n" + - " `k4` varchar(20) NULL COMMENT \"\",\n" + - " `k5` boolean NULL COMMENT \"\",\n" + - " `k6` tinyint(4) NULL COMMENT \"\",\n" + - " `k7` smallint(6) NULL COMMENT \"\",\n" + - " `k8` int(11) NULL COMMENT \"\",\n" + - " `k9` bigint(20) NULL COMMENT \"\",\n" + - " `k10` largeint(40) NULL COMMENT \"\",\n" + - " `k11` float NULL COMMENT \"\",\n" + - " `k12` double NULL COMMENT \"\",\n" + - " `k13` decimal128(27, 9) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY date_trunc('day', k1)\n" + - "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 2 \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"LZ4\"\n" + - ");"; + " `k1` date NULL COMMENT \"\",\n" + + " `k2` datetime NULL COMMENT \"\",\n" + + " `k3` char(20) NULL COMMENT \"\",\n" + + " `k4` varchar(20) NULL COMMENT \"\",\n" + + " `k5` boolean NULL COMMENT \"\",\n" + + " `k6` tinyint(4) NULL COMMENT \"\",\n" + + " `k7` smallint(6) NULL COMMENT \"\",\n" + + " `k8` int(11) NULL COMMENT \"\",\n" + + " `k9` bigint(20) NULL COMMENT \"\",\n" + + " `k10` largeint(40) NULL COMMENT \"\",\n" + + " `k11` float NULL COMMENT \"\",\n" + + " `k12` double NULL COMMENT \"\",\n" + + " `k13` decimal128(27, 9) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY date_trunc('day', k1)\n" + + "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 2 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"LZ4\"\n" + + ");"; String createTableLikeSql9 = "create table test.table_like_02 like test.duplicate_table_with_null;"; String newDbName9 = "test"; String existedDbName9 = "test"; String newTblName9 = "duplicate_table_with_null"; String existedTblName9 = "table_like_02"; checkCreateOlapTableLike(automaticTableSql, createTableLikeSql9, newDbName9, existedDbName9, newTblName9, - existedTblName9); + existedTblName9); // 10. create table like with properties String sql = "create table test.table_like_10 " + - "distributed by random buckets 7 " + - "properties('replication_num'='1') " + - "like test.duplicate_table_with_null"; + "distributed by random buckets 7 " + + "properties('replication_num'='1') " + + "like test.duplicate_table_with_null"; createTableLike(sql); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb(newDbName).getTable("table_like_10"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(newDbName).getTable("table_like_10"); Assert.assertEquals(new RandomDistributionInfo(7), table.getDefaultDistributionInfo()); Assert.assertEquals("1", table.getProperties().get("replication_num")); } @@ -285,23 +290,23 @@ public void testNormal() throws Exception { public void testAbnormal() { // 1. create table with same name String createTableSql = "create table test.testAbTbl1\n" + "(k1 int, k2 int)\n" + "duplicate key(k1)\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; String createTableLikeSql = "create table test.testAbTbl1 like test.testAbTbl1"; String newDbName = "test"; String newTblName = "testAbTbl1"; ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Table 'testAbTbl1' already exists", - () -> checkCreateOlapTableLike(createTableSql, createTableLikeSql, newDbName, newDbName, newTblName, - newTblName)); + () -> checkCreateOlapTableLike(createTableSql, createTableLikeSql, newDbName, newDbName, newTblName, + newTblName)); // 2. create table with not existed DB String createTableSql2 = "create table test.testAbTbl2\n" + "(k1 int, k2 int)\n" + "duplicate key(k1)\n" - + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; + + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; String createTableLikeSql2 = "create table fake_test.testAbTbl2_like like test.testAbTbl1"; String newDbName2 = "fake_test"; String existedDbName2 = "test"; String newTblName2 = "testAbTbl2_like"; String existedTblName2 = "testAbTbl1"; ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Unknown database 'fake_test'", - () -> checkCreateOlapTableLike(createTableSql2, createTableLikeSql2, newDbName2, existedDbName2, - newTblName2, existedTblName2)); + () -> checkCreateOlapTableLike(createTableSql2, createTableLikeSql2, newDbName2, existedDbName2, + newTblName2, existedTblName2)); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableTest.java index b11b6d9f7bf65..0a2128db1abd3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableTest.java @@ -297,13 +297,13 @@ public void testNormal() throws DdlException { " \"dynamic_partition.history_partition_num\" = \"0\"\n" + ");")); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl6 = (OlapTable) db.getTable("tbl6"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl6 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl6"); Assert.assertTrue(tbl6.getColumn("k1").isKey()); Assert.assertTrue(tbl6.getColumn("k2").isKey()); Assert.assertTrue(tbl6.getColumn("k3").isKey()); - OlapTable tbl7 = (OlapTable) db.getTable("tbl7"); + OlapTable tbl7 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl7"); Assert.assertTrue(tbl7.getColumn("k1").isKey()); Assert.assertFalse(tbl7.getColumn("k2").isKey()); Assert.assertTrue(tbl7.getColumn("k2").getAggregationType() == AggregateType.NONE); @@ -331,7 +331,7 @@ public void testPartitionByExprDynamicPartition() { " \"dynamic_partition.buckets\" = \"32\",\n" + " \"dynamic_partition.history_partition_num\" = \"0\"\n" + ");")); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Table str2dateTable = db.getTable("partition_str2date"); Assert.assertTrue(DynamicPartitionUtil.isDynamicPartitionTable(str2dateTable)); @@ -682,7 +682,8 @@ public void testCreateTableWithoutDistribution() { ExceptionChecker.expectThrowsNoException( () -> createTable("create table test.tmp1\n" + "(k1 int, k2 int)\n")); ExceptionChecker.expectThrowsNoException( - () -> createTable("create table test.tmp2\n" + "(k1 int, k2 float) PROPERTIES(\"replication_num\" = \"1\");\n")); + () -> createTable( + "create table test.tmp2\n" + "(k1 int, k2 float) PROPERTIES(\"replication_num\" = \"1\");\n")); ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Data type of first column cannot be HLL", () -> createTable("create table test.tmp3\n" + "(k1 hll, k2 float)\n")); } @@ -700,7 +701,7 @@ public void testCreateSumAgg() throws Exception { "AGGREGATE KEY(id_int)\n" + "DISTRIBUTED BY HASH(id_int) BUCKETS 10\n" + "PROPERTIES(\"replication_num\" = \"1\");"); - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb(connectContext.getDatabase()) .getTable("aggregate_table_sum"); String columns = table.getColumns().toString(); System.out.println("columns = " + columns); @@ -748,7 +749,7 @@ public void testLongColumnName() throws Exception { "than_64_chars VARCHAR(100)) DISTRIBUTED BY HASH(oh_my_gosh_this_is_a_long_column_name_look_at_it_it_" + "has_more_than_64_chars) BUCKETS 8 PROPERTIES(\"replication_num\" = \"1\");"; starRocksAssert.withTable(sql); - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb(connectContext.getDatabase()) .getTable("long_column_table"); Assert.assertEquals(1, table.getColumns().size()); Assert.assertNotNull( @@ -771,7 +772,7 @@ public void testCreateTableDefaultCurrentTimestamp() throws Exception { " \"in_memory\" = \"false\"\n" + ");"; starRocksAssert.withTable(sql); - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb(connectContext.getDatabase()) .getTable("test_create_default_current_timestamp"); Assert.assertEquals(2, table.getColumns().size()); } @@ -792,7 +793,7 @@ public void testCreateTableDefaultUUID() throws Exception { " \"in_memory\" = \"false\"\n" + ");"; starRocksAssert.withTable(sql); - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb(connectContext.getDatabase()) .getTable("test_create_default_uuid"); Assert.assertEquals(2, table.getColumns().size()); @@ -809,7 +810,7 @@ public void testCreateTableDefaultUUID() throws Exception { ");"; starRocksAssert.withTable(sql2); - final Table table2 = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table2 = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb(connectContext.getDatabase()) .getTable("test_create_default_uuid_numeric"); Assert.assertEquals(2, table2.getColumns().size()); } @@ -860,7 +861,7 @@ public void testCreateTableDefaultUUIDFailed() { @Test public void testCreateTableWithLocation() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // add label to backend SystemInfoService systemInfoService = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); @@ -886,7 +887,8 @@ public void testCreateTableWithLocation() throws Exception { " \"in_memory\" = \"false\"\n" + ");"); - OlapTable table = (OlapTable) testDb.getTable("test_location_no_prop"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_no_prop"); Assert.assertNotNull(table.getLocation()); System.out.println(table.getLocation()); Assert.assertTrue(table.getLocation().containsKey("*")); @@ -895,7 +897,7 @@ public void testCreateTableWithLocation() throws Exception { String showSql = "show create table test.`test_location_no_prop`"; ShowCreateTableStmt showCreateTableStmt = (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showSql, connectContext); - + ShowResultSet showResultSet = ShowExecutor.execute(showCreateTableStmt, connectContext); System.out.println(showResultSet.getResultRows()); Assert.assertTrue(showResultSet.getResultRows().get(0).toString().contains("\"" + @@ -920,7 +922,8 @@ public void testCreateTableWithLocation() throws Exception { " \"replication_num\" = \"1\",\n" + " \"in_memory\" = \"false\"\n" + ");"); - table = (OlapTable) testDb.getTable("test_location_no_backend_prop"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_no_backend_prop"); Assert.assertNull(table.getLocation()); // verify the location property in show create table result, shouldn't exist @@ -985,7 +988,8 @@ public void testCreateTableWithLocation() throws Exception { } } else { createTable(createTableSql); - table = (OlapTable) testDb.getTable("test_location_prop_" + i); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_location_prop_" + i); if (tableLocationProp.isEmpty()) { Assert.assertNull(table.getLocation()); continue; @@ -1339,8 +1343,9 @@ public void testCreateTableWithBinlogProperties() { ");" )); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("binlog_table"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "binlog_table"); Assert.assertNotNull(table.getCurBinlogConfig()); Assert.assertTrue(table.isBinlogEnabled()); @@ -1375,8 +1380,9 @@ public void testCreateTableWithoutBinlogProperties() { ");" )); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("not_binlog_table"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "not_binlog_table"); Assert.assertFalse(table.containsBinlogConfig()); Assert.assertFalse(table.isBinlogEnabled()); @@ -1405,8 +1411,9 @@ public void testCreateTableWithConstraint() { ");" )); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("parent_table1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "parent_table1"); Assert.assertTrue(table.hasUniqueConstraints()); List uniqueConstraint = table.getUniqueConstraints(); @@ -1429,7 +1436,8 @@ public void testCreateTableWithConstraint() { ");" )); - OlapTable table2 = (OlapTable) db.getTable("parent_table2"); + OlapTable table2 = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "parent_table2"); Assert.assertTrue(table2.hasUniqueConstraints()); List uniqueConstraint2 = table2.getUniqueConstraints(); @@ -2049,8 +2057,8 @@ public void testReservedColumnName() { String sql1 = "create table tbl_simple_pk(key0 string, __op boolean) primary key(key0)" + " distributed by hash(key0) properties(\"replication_num\"=\"1\");"; ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Getting analyzing error." + - " Detail message: Column name [__op] is a system reserved name." + - " If you are sure you want to use it, please set FE configuration allow_system_reserved_names", + " Detail message: Column name [__op] is a system reserved name." + + " If you are sure you want to use it, please set FE configuration allow_system_reserved_names", () -> starRocksAssert.withTable(sql1)); } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithAggStateTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithAggStateTest.java index f3213a376e189..541e7011d5a42 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithAggStateTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithAggStateTest.java @@ -65,7 +65,8 @@ public void testCreateTableWithAggStateAvg() { "PARTITION BY (k1) \n" + "DISTRIBUTED BY HASH(k1) BUCKETS 3;", () -> { - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); String columns = table.getColumns().toString(); String expect = "[`k1` varchar(10) NULL COMMENT \"\", " + @@ -100,7 +101,8 @@ public void testCreateTableWithAggStateSum() { "PARTITION BY (k1) \n" + "DISTRIBUTED BY HASH(k1) BUCKETS 3;", () -> { - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); String columns = table.getColumns().toString(); String expect = "[`k1` varchar(10) NULL COMMENT \"\", " + @@ -131,7 +133,8 @@ public void testCreateTableWithAggStateHllSketch() { "PARTITION BY (dt) \n" + "DISTRIBUTED BY HASH(dt) BUCKETS 4;", () -> { - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); String columns = table.getColumns().toString(); String expect = "[`dt` varchar(10) NULL COMMENT \"\", " + @@ -162,7 +165,8 @@ public void testCreateTableWithAggStateMinBy() { "PARTITION BY (k1) \n" + "DISTRIBUTED BY HASH(k1) BUCKETS 3;", () -> { - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); String columns = table.getColumns().toString(); String expect = "[`k1` varchar(10) NULL COMMENT \"\", " + @@ -198,7 +202,8 @@ public void testCreateTableWithAggStateArrayAgg() { "PARTITION BY (k1) \n" + "DISTRIBUTED BY HASH(k1) BUCKETS 3;", () -> { - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); String columns = table.getColumns().toString(); String expect = "[`k1` varchar(10) NULL COMMENT \"\", " + @@ -234,7 +239,8 @@ public void testCreateTableWithAggStateGroupConcat() { "PARTITION BY (k1) \n" + "DISTRIBUTED BY HASH(k1) BUCKETS 3;", () -> { - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); String columns = table.getColumns().toString(); String expect = "[`k1` varchar(10) NULL COMMENT \"\", " + @@ -263,7 +269,8 @@ public void testCreateTableWithAggStateBadCase1() { "AGGREGATE KEY(k1)\n" + "PARTITION BY (k1) \n" + "DISTRIBUTED BY HASH(k1) BUCKETS 3;"); - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); Assert.assertEquals(null, table); } catch (Exception e) { @@ -284,7 +291,8 @@ public void testCreateTableWithAggStateBadCase2() { "AGGREGATE KEY(k1)\n" + "PARTITION BY (k1) \n" + "DISTRIBUTED BY HASH(k1) BUCKETS 3;"); - final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getDb(connectContext.getDatabase()) + final Table table = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() + .getDb(connectContext.getDatabase()) .getTable("test_agg_tbl1"); Assert.assertEquals(null, table); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java index edc7791a91c9e..d1809b2a25fca 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateTableWithLocationTest.java @@ -75,7 +75,7 @@ private void clearBackendLocationProp() throws SQLException { PseudoCluster cluster = PseudoCluster.getInstance(); for (PseudoBackend backend : cluster.getBackends()) { String stmtStr = "alter system modify backend '" + backend.getHostHeartbeatPort() + "' set ('" + - AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = '')"; + AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = '')"; System.out.println(stmtStr); cluster.runSql(null, stmtStr); } @@ -86,32 +86,32 @@ private void setBackendLocationProp(List locations) throws SQLException int i = 0; for (PseudoBackend backend : cluster.getBackends()) { String stmtStr = "alter system modify backend '" + backend.getHostHeartbeatPort() + "' set ('" + - AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = '" + locations.get(i++) + "')"; + AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = '" + locations.get(i++) + "')"; System.out.println(stmtStr); cluster.runSql(null, stmtStr); } } - @Test public void testCreateTableAndBackendNoLocationProp() throws SQLException { clearBackendLocationProp(); PseudoCluster cluster = PseudoCluster.getInstance(); String sql = "CREATE TABLE test.`test_table_backend_no_loc` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 2\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_table_backend_no_loc"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_backend_no_loc"); Assert.assertNull(table.getLocation()); } @@ -120,7 +120,7 @@ private Set getBackendIdsWithLocProp() { Set backendIds = Sets.newHashSet(); for (PseudoBackend pseudoBackend : cluster.getBackends()) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(pseudoBackend.getId()); + .getClusterInfo().getBackend(pseudoBackend.getId()); if (!backend.getLocation().isEmpty()) { backendIds.add(backend.getId()); } @@ -133,7 +133,7 @@ private Set getBackendIdsWithLocProp(String locationKey) { Set backendIds = Sets.newHashSet(); for (PseudoBackend pseudoBackend : cluster.getBackends()) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(pseudoBackend.getId()); + .getClusterInfo().getBackend(pseudoBackend.getId()); if (backend.getLocation().containsKey(locationKey)) { backendIds.add(backend.getId()); } @@ -146,9 +146,9 @@ private Set getBackendIdsWithLocProp(String locationKey, String locationVa Set backendIds = Sets.newHashSet(); for (PseudoBackend pseudoBackend : cluster.getBackends()) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(pseudoBackend.getId()); + .getClusterInfo().getBackend(pseudoBackend.getId()); if (backend.getLocation().containsKey(locationKey) && - Objects.equals(backend.getLocation().get(locationKey), locationVal)) { + Objects.equals(backend.getLocation().get(locationKey), locationVal)) { backendIds.add(backend.getId()); } } @@ -160,7 +160,7 @@ private Set getBackendIdsWithoutLocProp() { Set backendIds = Sets.newHashSet(); for (PseudoBackend pseudoBackend : cluster.getBackends()) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(pseudoBackend.getId()); + .getClusterInfo().getBackend(pseudoBackend.getId()); if (backend.getLocation().isEmpty()) { backendIds.add(backend.getId()); } @@ -172,24 +172,25 @@ private Set getBackendIdsWithoutLocProp() { public void testCreateTableNoLocPropBackendWithLocProp() throws SQLException { // last backend doesn't contain location property List locations = Lists.newArrayList("rack:r1", "rack:rack2", "rack:rack3", - "region:r1", "region:r2", "region:r3", ""); + "region:r1", "region:r2", "region:r3", ""); setBackendLocationProp(locations); PseudoCluster cluster = PseudoCluster.getInstance(); String sql = "CREATE TABLE test.`test_table_no_loc_backend_with_loc` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_table_no_loc_backend_with_loc"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_no_loc_backend_with_loc"); Assert.assertTrue(table.getLocation().keySet().contains("*")); List partitions = new ArrayList<>(table.getAllPartitions()); @@ -215,27 +216,28 @@ public void testCreateTableNoLocPropBackendWithLocProp() throws SQLException { public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLException { // last backend doesn't contain location property List locations = Lists.newArrayList("rack:r1", "rack:rack2", "rack:rack3", - "region:r1", "region:r2", "region:r3", ""); + "region:r1", "region:r2", "region:r3", ""); setBackendLocationProp(locations); PseudoCluster cluster = PseudoCluster.getInstance(); // Test: rack:* String sql = "CREATE TABLE test.`test_table_explicit_loc_backend_with_loc1` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:*\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:*\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; System.out.println(sql); cluster.runSql("test", sql); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_table_explicit_loc_backend_with_loc1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc1"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); List partitions = new ArrayList<>(table.getAllPartitions()); List tablets = partitions.get(0).getBaseIndex().getTablets(); @@ -259,21 +261,22 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc // Test: rack:*, region:* sql = "CREATE TABLE test.`test_table_explicit_loc_backend_with_loc2` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:*, region:*\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:*, region:*\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); boolean hasReplicaOnRegionBackend = false; boolean hasReplicaOnRackBackend = false; - table = (OlapTable) db.getTable("test_table_explicit_loc_backend_with_loc2"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc2"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); Assert.assertTrue(table.getLocation().keySet().contains("region")); partitions = new ArrayList<>(table.getAllPartitions()); @@ -299,19 +302,20 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc // Test: rack:r1, rack:rack2, rack:rack3 sql = "CREATE TABLE test.`test_table_explicit_loc_backend_with_loc3` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2, rack:rack3\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2, rack:rack3\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); - table = (OlapTable) db.getTable("test_table_explicit_loc_backend_with_loc3"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc3"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); Assert.assertEquals(1, table.getLocation().keySet().size()); partitions = new ArrayList<>(table.getAllPartitions()); @@ -333,19 +337,20 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc // Test: rack:r1, region:* sql = "CREATE TABLE test.`test_table_explicit_loc_backend_with_loc4` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, region:*\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, region:*\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); - table = (OlapTable) db.getTable("test_table_explicit_loc_backend_with_loc4"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc4"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); Assert.assertTrue(table.getLocation().keySet().contains("region")); partitions = new ArrayList<>(table.getAllPartitions()); @@ -372,19 +377,20 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc // Test: rack:r1, rack:rack2, not enough hosts, fallback to ignore location prop sql = "CREATE TABLE test.`test_table_explicit_loc_backend_with_loc5` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); - table = (OlapTable) db.getTable("test_table_explicit_loc_backend_with_loc5"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc5"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); partitions = new ArrayList<>(table.getAllPartitions()); tablets = partitions.get(0).getBaseIndex().getTablets(); @@ -403,27 +409,27 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc backendIdsWithLocProp = getBackendIdsWithLocProp(); backendIdsWithLocProp.addAll(getBackendIdsWithoutLocProp()); Assert.assertEquals(backendIdsWithLocProp.size(), - Sets.intersection(allReplicasBackendIds, backendIdsWithLocProp).size()); + Sets.intersection(allReplicasBackendIds, backendIdsWithLocProp).size()); // Test: rack:r1, rack:rack2, not enough hosts, fallback to ignore location prop sql = "CREATE TABLE test.`test_table_explicit_loc_backend_with_loc6` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"10\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"10\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; try { cluster.runSql("test", sql); } catch (Exception e) { Assert.assertTrue(e.getMessage().contains( - "Table replication num should be less than of equal to the number of available BE nodes")); + "Table replication num should be less than of equal to the number of available BE nodes")); } // clean up @@ -434,27 +440,28 @@ public void testCreateTableWithExplicitLocPropBackendWithLocProp() throws SQLExc public void testBestEffortMatchLocationProp() throws SQLException { // last backend has the same location prop with the first one List locations = Lists.newArrayList("rack:r1", "rack:rack2", "rack:rack3", - "region:r1", "region:r2", "region:r3", "rack:r1"); + "region:r1", "region:r2", "region:r3", "rack:r1"); setBackendLocationProp(locations); PseudoCluster cluster = PseudoCluster.getInstance(); // Test: rack:r1, rack:rack2, 3 hosts, but only 2 racks String sql = "CREATE TABLE test.`test_table_explicit_loc_backend_with_loc_best_effort` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r1, rack:rack2\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; System.out.println(sql); cluster.runSql("test", sql); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_table_explicit_loc_backend_with_loc_best_effort"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_explicit_loc_backend_with_loc_best_effort"); Assert.assertTrue(table.getLocation().keySet().contains("rack")); List partitions = new ArrayList<>(table.getAllPartitions()); List tablets = partitions.get(0).getBaseIndex().getTablets(); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java index 19d14ccb17db7..23b2495b99eb1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java @@ -74,7 +74,7 @@ public void testCreateViewNullable() throws Exception { .withView("create view test_null_view as select * from site_access;"); Table view = starRocksAssert.getCtx().getGlobalStateMgr() - .getDb("test").getTable("test_null_view"); + .getLocalMetastore().getDb("test").getTable("test_null_view"); Assert.assertTrue(view instanceof View); List columns = view.getColumns(); for (Column column : columns) { @@ -135,7 +135,7 @@ public void testCreateViewWithWindowFunctionIgnoreNulls() throws Exception { ", lead(price ignore nulls,1,0) over (partition by username) as leadValue\n" + "from sample_data;"); - Table view = starRocksAssert.getCtx().getGlobalStateMgr() + Table view = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore() .getDb("test").getTable("test_ignore_nulls"); Assert.assertTrue(view instanceof View); String str = ((View) view).getInlineViewDef(); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java index 8508457321767..ab5109b561a54 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/DropPartitionTest.java @@ -97,8 +97,8 @@ private void dropPartition(String sql) throws Exception { @Test public void testNormalDropPartition() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("tbl1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Partition partition = table.getPartition("p20210201"); long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); String dropPartitionSql = " alter table test.tbl1 drop partition p20210201;"; @@ -119,8 +119,8 @@ public void testNormalDropPartition() throws Exception { @Test public void testForceDropPartition() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("tbl1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Partition partition = table.getPartition("p20210202"); long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); String dropPartitionSql = " alter table test.tbl1 drop partition p20210202 force;"; @@ -144,8 +144,8 @@ public void testForceDropPartition() throws Exception { @Test public void testDropPartitionAndReserveTablets() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("tbl1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Partition partition = table.getPartition("p20210203"); long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); table.dropPartitionAndReserveTablet("p20210203"); @@ -249,8 +249,8 @@ private void checkForceDropPartitions(String dropPartitionSql) throws Exception } private void checkBeforeDrop(String dbName, String tableName, String partitionName, long tabletId) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - OlapTable table = (OlapTable) db.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); List replicaList = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getReplicasByTabletId(tabletId); Partition partition = table.getPartition(partitionName); @@ -273,7 +273,7 @@ private Partition recoverPartition(String db, String table, String partitionName } private Table getTable(String dbName, String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - return (OlapTable) db.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + return (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/DynamicPartitionTableTest.java index 3fc83355accdd..bc4ef7e0074c5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/DynamicPartitionTableTest.java @@ -78,34 +78,35 @@ private static void createTable(String sql) throws Exception { @Test public void testNormal() throws Exception { starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_normal` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("dynamic_partition_normal"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "dynamic_partition_normal"); Assert.assertEquals(table.getTableProperty().getDynamicPartitionProperty().getReplicationNum(), - DynamicPartitionProperty.NOT_SET_REPLICATION_NUM); + DynamicPartitionProperty.NOT_SET_REPLICATION_NUM); } @Test @@ -113,57 +114,57 @@ public void testMissTimeUnit() throws Exception { expectedException.expect(DdlException.class); expectedException.expectMessage("Must assign dynamic_partition.time_unit properties"); starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_time_unit` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"); } @Test public void testMissStart() throws Exception { starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_start` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"); } @Test @@ -171,59 +172,60 @@ public void testMissEnd() throws Exception { expectedException.expect(DdlException.class); expectedException.expectMessage("Must assign dynamic_partition.end properties"); starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_end` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"); } @Test public void testMissBuckets() throws Exception { starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_buckets` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\"\n" + - ");"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("dynamic_partition_buckets"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\"\n" + + ");"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "dynamic_partition_buckets"); Assert.assertEquals(table.getTableProperty().getDynamicPartitionProperty().getBuckets(), 0); } @@ -232,24 +234,24 @@ public void testNotAllowed() throws Exception { expectedException.expect(DdlException.class); expectedException.expectMessage("Only support dynamic partition properties on range partition table"); starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_non_range` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"); } @Test @@ -257,89 +259,89 @@ public void testNotAllowedInMultiPartitions() throws Exception { expectedException.expect(DdlException.class); expectedException.expectMessage("Dynamic partition only support single-column range partition"); starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_normal` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1, k2)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\", \"100\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\", \"200\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\", \"300\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1, k2)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\", \"100\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\", \"200\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\", \"300\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"); } @Test public void testMissTimeZone() throws Exception { starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_miss_time_zone` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.buckets\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\"\n" + + ");"); } @Test public void testNormalTimeZone() throws Exception { starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_time_zone` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.buckets\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.time_zone\" = \"Asia/Shanghai\",\n" + - "\"dynamic_partition.prefix\" = \"p\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.time_zone\" = \"Asia/Shanghai\",\n" + + "\"dynamic_partition.prefix\" = \"p\"\n" + + ");"); } @Test @@ -347,96 +349,96 @@ public void testInvalidTimeZone() throws Exception { expectedException.expect(DdlException.class); expectedException.expectMessage("Unknown or incorrect time zone: 'invalid'"); starRocksAssert.withTable("CREATE TABLE test.`dynamic_partition_invalid_time_zone` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.buckets\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.time_zone\" = \"invalid\",\n" + - "\"dynamic_partition.prefix\" = \"p\"\n" + - ");"); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.time_zone\" = \"invalid\",\n" + + "\"dynamic_partition.prefix\" = \"p\"\n" + + ");"); } @Test public void testSetDynamicPartitionReplicationNum() throws Exception { String tableName = "dynamic_partition_replication_num"; starRocksAssert.withTable("CREATE TABLE test.`" + tableName + "` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE (k1)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\",\n" + - "\"dynamic_partition.replication_num\" = \"2\"\n" + - ");"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable(tableName); + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\",\n" + + "\"dynamic_partition.replication_num\" = \"2\"\n" + + ");"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); Assert.assertEquals(table.getTableProperty().getDynamicPartitionProperty().getReplicationNum(), 2); } @Test public void testEmptyDynamicPartition() throws Exception { String createOlapTblStmt = "CREATE TABLE test.`empty_dynamic_partition` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` int NULL COMMENT \"\",\n" + - " `k3` smallint NULL COMMENT \"\",\n" + - " `v1` varchar(2048) NULL COMMENT \"\",\n" + - " `v2` datetime NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE(`k1`)\n" + - "()\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"false\",\n" + - "\"dynamic_partition.start\" = \"-3\",\n" + - "\"dynamic_partition.end\" = \"3\",\n" + - "\"dynamic_partition.time_unit\" = \"day\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"1\"\n" + - ");"; + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE(`k1`)\n" + + "()\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"false\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; String insertStmt = - "insert into test.`empty_dynamic_partition` values ('2020-09-10', 1000, 100, 'test', '2020-09-10 23:59:59');"; + "insert into test.`empty_dynamic_partition` values ('2020-09-10', 1000, 100, 'test', '2020-09-10 23:59:59');"; createTable(createOlapTblStmt); expectedException.expect(AnalysisException.class); expectedException.expectMessage("data cannot be inserted into table with empty partition." + - "Use `SHOW PARTITIONS FROM empty_dynamic_partition` to see the currently partitions of this table. "); + "Use `SHOW PARTITIONS FROM empty_dynamic_partition` to see the currently partitions of this table. "); UtFrameUtils.parseStmtWithNewParser("explain " + insertStmt, connectContext); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/ExpressionRangePartitionInfoTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/ExpressionRangePartitionInfoTest.java index d630940b6c9ac..3c6d77872f153 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/ExpressionRangePartitionInfoTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/ExpressionRangePartitionInfoTest.java @@ -58,7 +58,6 @@ public class ExpressionRangePartitionInfoTest { private static ConnectContext connectContext; private static StarRocksAssert starRocksAssert; - @Before public void setUp() throws Exception { partitionExprs = Lists.newArrayList(); @@ -71,7 +70,7 @@ public void setUp() throws Exception { fnChildren.add(slotRef2); functionCallExpr = new FunctionCallExpr("date_trunc", fnChildren); functionCallExpr.setFn(Expr.getBuiltinFunction( - "date_trunc", new Type[] {Type.VARCHAR, Type.DATETIME}, Function.CompareMode.IS_IDENTICAL)); + "date_trunc", new Type[] {Type.VARCHAR, Type.DATETIME}, Function.CompareMode.IS_IDENTICAL)); FeConstants.runningUnitTest = true; UtFrameUtils.createMinStarRocksCluster(); @@ -90,9 +89,9 @@ public void testInitUseSlotRef() { partitionExprs.add(ColumnIdExpr.create(slotRef)); List schema = Collections.singletonList(k1); ExpressionRangePartitionInfo expressionRangePartitionInfo = new ExpressionRangePartitionInfo(partitionExprs, - schema, PartitionType.RANGE); + schema, PartitionType.RANGE); List partitionColumns = expressionRangePartitionInfo.getPartitionColumns( - MetaUtils.buildIdToColumn(schema)); + MetaUtils.buildIdToColumn(schema)); Assert.assertEquals(partitionColumns.size(), 1); Assert.assertEquals(partitionColumns.get(0), k1); } @@ -102,9 +101,9 @@ public void testInitUseFunction() { partitionExprs.add(ColumnIdExpr.create(functionCallExpr)); List schema = Collections.singletonList(k2); ExpressionRangePartitionInfo expressionRangePartitionInfo = - new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); + new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); List partitionColumns = expressionRangePartitionInfo.getPartitionColumns( - MetaUtils.buildIdToColumn(schema)); + MetaUtils.buildIdToColumn(schema)); Assert.assertEquals(partitionColumns.size(), 1); Assert.assertEquals(partitionColumns.get(0), k2); } @@ -117,9 +116,9 @@ public void testInitHybrid() { partitionExprs.add(ColumnIdExpr.create(functionCallExpr)); List schema = Arrays.asList(k1, k2); ExpressionRangePartitionInfo expressionRangePartitionInfo = - new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); + new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); List partitionColumns = expressionRangePartitionInfo.getPartitionColumns( - MetaUtils.buildIdToColumn(schema)); + MetaUtils.buildIdToColumn(schema)); Assert.assertEquals(partitionColumns.size(), 2); Assert.assertEquals(partitionColumns.get(0), k1); Assert.assertEquals(partitionColumns.get(1), k2); @@ -135,15 +134,15 @@ public void testTinyInt() throws DdlException, AnalysisException { partitionExprs.add(ColumnIdExpr.create(slotRef)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"))), - null)); + new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"))), + null)); List schema = Collections.singletonList(k1); partitionInfo = new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(1, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), - singleRangePartitionDesc, 20000L, false); + singleRangePartitionDesc, 20000L, false); } } @@ -154,15 +153,15 @@ public void testSmallInt() throws DdlException, AnalysisException { partitionExprs.add(ColumnIdExpr.create(slotRef)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-32768"))), - null)); + new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-32768"))), + null)); List schema = Collections.singletonList(k1); partitionInfo = new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(1, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), - singleRangePartitionDesc, 20000L, false); + singleRangePartitionDesc, 20000L, false); } } @@ -173,15 +172,15 @@ public void testInt() throws DdlException, AnalysisException { partitionExprs.add(ColumnIdExpr.create(slotRef)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-2147483648"))), - null)); + new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-2147483648"))), + null)); List schema = Collections.singletonList(k1); partitionInfo = new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(1, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -192,13 +191,13 @@ public void testBigInt() throws DdlException, AnalysisException { partitionExprs.add(ColumnIdExpr.create(slotRef)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("-9223372036854775808"))), null)); + .newArrayList(new PartitionValue("-9223372036854775808"))), null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("-9223372036854775806"))), null)); + .newArrayList(new PartitionValue("-9223372036854775806"))), null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p3", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("0"))), null)); + .newArrayList(new PartitionValue("0"))), null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p4", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("9223372036854775806"))), null)); + .newArrayList(new PartitionValue("9223372036854775806"))), null)); List schema = Collections.singletonList(k1); partitionInfo = new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); @@ -206,7 +205,7 @@ public void testBigInt() throws DdlException, AnalysisException { for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(1, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -217,13 +216,13 @@ public void testBigIntNormal() throws DdlException, AnalysisException { partitionExprs.add(ColumnIdExpr.create(slotRef)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("-9223372036854775806"))), null)); + .newArrayList(new PartitionValue("-9223372036854775806"))), null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("-9223372036854775805"))), null)); + .newArrayList(new PartitionValue("-9223372036854775805"))), null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p3", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("0"))), null)); + .newArrayList(new PartitionValue("0"))), null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p4", new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("9223372036854775806"))), null)); + .newArrayList(new PartitionValue("9223372036854775806"))), null)); List schema = Collections.singletonList(k1); partitionInfo = new ExpressionRangePartitionInfo(partitionExprs, schema, PartitionType.RANGE); @@ -231,7 +230,7 @@ public void testBigIntNormal() throws DdlException, AnalysisException { for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(1, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -256,17 +255,17 @@ public void testFixedRange() throws DdlException, AnalysisException { //add RangePartitionDescs PartitionKeyDesc p1 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200"))); + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200"))); PartitionKeyDesc p2 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("20190105"), new PartitionValue("10")), - Lists.newArrayList(new PartitionValue("20190107"), new PartitionValue("10"))); + Lists.newArrayList(new PartitionValue("20190105"), new PartitionValue("10")), + Lists.newArrayList(new PartitionValue("20190107"), new PartitionValue("10"))); PartitionKeyDesc p3 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("20181231"), new PartitionValue("10")), - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100"))); + Lists.newArrayList(new PartitionValue("20181231"), new PartitionValue("10")), + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100"))); PartitionKeyDesc p4 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("20190105"), new PartitionValue("100")), - Lists.newArrayList(new PartitionValue("20190120"), new PartitionValue("10000000000"))); + Lists.newArrayList(new PartitionValue("20190105"), new PartitionValue("100")), + Lists.newArrayList(new PartitionValue("20190120"), new PartitionValue("10000000000"))); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", p2, null)); @@ -279,7 +278,7 @@ public void testFixedRange() throws DdlException, AnalysisException { for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(columns, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -306,14 +305,14 @@ public void testFixedRange1() throws DdlException, AnalysisException { //add RangePartitionDescs PartitionKeyDesc p1 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("2019-02-01"), new PartitionValue("100"), - new PartitionValue("200"))); + Lists.newArrayList(new PartitionValue("2019-02-01"), new PartitionValue("100"), + new PartitionValue("200"))); PartitionKeyDesc p2 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("2020-02-01"), new PartitionValue("100"), - new PartitionValue("200")), - Lists.newArrayList(new PartitionValue("10000000000"))); + Lists.newArrayList(new PartitionValue("2020-02-01"), new PartitionValue("100"), + new PartitionValue("200")), + Lists.newArrayList(new PartitionValue("10000000000"))); PartitionKeyDesc p3 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("2021-02-01"))); + Lists.newArrayList(new PartitionValue("2021-02-01"))); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", p2, null)); @@ -331,7 +330,7 @@ public void testFixedRange1() throws DdlException, AnalysisException { } singleRangePartitionDesc.analyze(partitionExprs.size(), null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -353,7 +352,7 @@ public void testFixedRange2() throws DdlException, AnalysisException { //add RangePartitionDescs PartitionKeyDesc p1 = new PartitionKeyDesc(new ArrayList<>(), - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200"))); + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200"))); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); @@ -363,7 +362,7 @@ public void testFixedRange2() throws DdlException, AnalysisException { for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(columns, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -386,8 +385,8 @@ public void testFixedRange3() throws DdlException, AnalysisException { //add RangePartitionDescs PartitionKeyDesc p1 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200")), - new ArrayList<>()); + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("200")), + new ArrayList<>()); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); @@ -397,7 +396,7 @@ public void testFixedRange3() throws DdlException, AnalysisException { for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(columns, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -419,8 +418,8 @@ public void testFixedRange4() throws DdlException, AnalysisException { //add RangePartitionDescs PartitionKeyDesc p1 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), - Lists.newArrayList(new PartitionValue("20190201"))); + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), + Lists.newArrayList(new PartitionValue("20190201"))); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); @@ -430,7 +429,7 @@ public void testFixedRange4() throws DdlException, AnalysisException { for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(columns, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -453,8 +452,8 @@ public void testFixedRange5() throws DdlException, AnalysisException { //add RangePartitionDescs PartitionKeyDesc p1 = new PartitionKeyDesc( - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), - Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100"))); + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100")), + Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100"))); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); @@ -464,7 +463,7 @@ public void testFixedRange5() throws DdlException, AnalysisException { for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { singleRangePartitionDesc.analyze(columns, null); partitionInfo.handleNewSinglePartitionDesc(MetaUtils.buildIdToColumn(schema), singleRangePartitionDesc, - 20000L, false); + 20000L, false); } } @@ -472,35 +471,35 @@ public void testFixedRange5() throws DdlException, AnalysisException { public void testExpressionRangePartitionInfoSerialized_FunctionExpr() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE table_hitcount (\n" + - "databaseName varchar(200) NULL COMMENT \"\",\n" + - "tableName varchar(200) NULL COMMENT \"\",\n" + - "queryTime varchar(50) NULL COMMENT \"\",\n" + - "queryId varchar(50) NULL COMMENT \"\",\n" + - "partitionHitSum int(11) NULL COMMENT \"\",\n" + - "partitionSum int(11) NULL COMMENT \"\",\n" + - "tabletHitNum int(11) NULL COMMENT \"\",\n" + - "tabletSum int(11) NULL COMMENT \"\",\n" + - "startHitPartition varchar(20) NULL COMMENT \"\",\n" + - "dt date NULL COMMENT \"\",\n" + - "clusterAddress varchar(50) NULL COMMENT \"\",\n" + - "costTime int(11) NULL COMMENT \"\",\n" + - "tableQueryCount int(11) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(databaseName, tableName)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY date_trunc('day', dt)\n" + - "DISTRIBUTED BY HASH(databaseName) BUCKETS 1\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"LZ4\"\n" + - ");"; + "databaseName varchar(200) NULL COMMENT \"\",\n" + + "tableName varchar(200) NULL COMMENT \"\",\n" + + "queryTime varchar(50) NULL COMMENT \"\",\n" + + "queryId varchar(50) NULL COMMENT \"\",\n" + + "partitionHitSum int(11) NULL COMMENT \"\",\n" + + "partitionSum int(11) NULL COMMENT \"\",\n" + + "tabletHitNum int(11) NULL COMMENT \"\",\n" + + "tabletSum int(11) NULL COMMENT \"\",\n" + + "startHitPartition varchar(20) NULL COMMENT \"\",\n" + + "dt date NULL COMMENT \"\",\n" + + "clusterAddress varchar(50) NULL COMMENT \"\",\n" + + "costTime int(11) NULL COMMENT \"\",\n" + + "tableQueryCount int(11) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(databaseName, tableName)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY date_trunc('day', dt)\n" + + "DISTRIBUTED BY HASH(databaseName) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"LZ4\"\n" + + ");"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("table_hitcount"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "table_hitcount"); // serialize String json = GsonUtils.GSON.toJson(table); // deserialize @@ -516,34 +515,34 @@ public void testExpressionRangePartitionInfoSerialized_FunctionExpr() throws Exc public void testExpressionRangePartitionInfoSerialized_SlotRef() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test_table (\n" + - "databaseName varchar(200) NULL COMMENT \"\",\n" + - "tableName varchar(200) NULL COMMENT \"\",\n" + - "queryTime varchar(50) NULL COMMENT \"\",\n" + - "queryId varchar(50) NULL COMMENT \"\",\n" + - "dt date NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(databaseName, tableName)\n" + - "PARTITION BY RANGE (`dt`)\n" + - "(\n" + - " PARTITION p1 values less than('2021-02-01'),\n" + - " PARTITION p2 values less than('2021-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(databaseName) BUCKETS 1\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");"; + "databaseName varchar(200) NULL COMMENT \"\",\n" + + "tableName varchar(200) NULL COMMENT \"\",\n" + + "queryTime varchar(50) NULL COMMENT \"\",\n" + + "queryId varchar(50) NULL COMMENT \"\",\n" + + "dt date NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(databaseName, tableName)\n" + + "PARTITION BY RANGE (`dt`)\n" + + "(\n" + + " PARTITION p1 values less than('2021-02-01'),\n" + + " PARTITION p2 values less than('2021-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(databaseName) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); starRocksAssert.withMaterializedView("create materialized view test_mv1 " + - " DISTRIBUTED BY HASH(dt, queryId) BUCKETS 4\n" + - " PARTITION BY dt\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") as select dt, queryId, count(1) from test_table group by dt, queryId" + " DISTRIBUTED BY HASH(dt, queryId) BUCKETS 4\n" + + " PARTITION BY dt\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") as select dt, queryId, count(1) from test_table group by dt, queryId" ); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("test_mv1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_mv1"); // serialize String json = GsonUtils.GSON.toJson(table); // deserialize @@ -551,7 +550,7 @@ public void testExpressionRangePartitionInfoSerialized_SlotRef() throws Exceptio ExpressionRangePartitionInfo expressionRangePartitionInfo = (ExpressionRangePartitionInfo) readTable.getPartitionInfo(); List readPartitionExprs = expressionRangePartitionInfo.getPartitionExprs(readTable.getIdToColumn()); Assert.assertTrue(readPartitionExprs.get(0) instanceof SlotRef); - SlotRef slotRef = (SlotRef) readPartitionExprs.get(0); + SlotRef slotRef = (SlotRef) readPartitionExprs.get(0); Assert.assertTrue(!slotRef.getType().isInvalid()); Assert.assertTrue(slotRef.getType().isDateType()); starRocksAssert.dropMaterializedView("test_mv1"); @@ -562,30 +561,31 @@ public void testExpressionRangePartitionInfoSerialized_SlotRef() throws Exceptio public void testExpressionRangePartitionInfoV2SerializedWrongNotFailed() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE `game_log2` (\n" + - " `cloud_id` varchar(65533) NULL COMMENT \"\",\n" + - " `user_id` varchar(65533) NULL COMMENT \"\",\n" + - " `day` date NULL COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`cloud_id`, `user_id`)\n" + - "PARTITION BY RANGE(cast(substr(cloud_id, 3, 11) as bigint))()\n" + - "DISTRIBUTED BY HASH(`cloud_id`, `user_id`) BUCKETS 1 \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"ZSTD\"\n" + - ");"; + " `cloud_id` varchar(65533) NULL COMMENT \"\",\n" + + " `user_id` varchar(65533) NULL COMMENT \"\",\n" + + " `day` date NULL COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`cloud_id`, `user_id`)\n" + + "PARTITION BY RANGE(cast(substr(cloud_id, 3, 11) as bigint))()\n" + + "DISTRIBUTED BY HASH(`cloud_id`, `user_id`) BUCKETS 1 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"ZSTD\"\n" + + ");"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable olapTable = (OlapTable) db.getTable("game_log2"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "game_log2"); ExpressionRangePartitionInfoV2 expressionRangePartitionInfo = - (ExpressionRangePartitionInfoV2) olapTable.getPartitionInfo(); + (ExpressionRangePartitionInfoV2) olapTable.getPartitionInfo(); expressionRangePartitionInfo.setPartitionExprs(Lists.newArrayList( - ColumnIdExpr.create(new FunctionCallExpr("abc", Lists.newArrayList(new SlotRef( - new TableName("test", "game_log2"), "cloud_id")))))); + ColumnIdExpr.create(new FunctionCallExpr("abc", Lists.newArrayList(new SlotRef( + new TableName("test", "game_log2"), "cloud_id")))))); // serialize String json = GsonUtils.GSON.toJson(olapTable); // deserialize diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/ForeignKeyConstraintTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/ForeignKeyConstraintTest.java index c928ab337aca8..a6239bb2a8c43 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/ForeignKeyConstraintTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/ForeignKeyConstraintTest.java @@ -37,6 +37,10 @@ public void beforeAll() { db.registerTableUnlocked(table1); db.registerTableUnlocked(table2); db.registerTableUnlocked(table3); + } + + @Test + public void testParseInternal() { new Expectations(globalStateMgr) { { @@ -44,15 +48,24 @@ public void beforeAll() { minTimes = 0; result = globalStateMgr; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; + + globalStateMgr.getLocalMetastore().getTable(anyLong, 1000L); + minTimes = 0; + result = db.getTable(1000L); + + globalStateMgr.getLocalMetastore().getTable(anyLong, 1001L); + minTimes = 0; + result = db.getTable(1001L); + + globalStateMgr.getLocalMetastore().getTable(anyLong, 1002L); + minTimes = 0; + result = db.getTable(1002L); } }; - } - @Test - public void testParseInternal() { // internal catalog String constraintDescs = "(column1) REFERENCES default_catalog.100.1000(newColumn1)"; List foreignKeyConstraints1 = ForeignKeyConstraint.parse(constraintDescs); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java b/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java index b9a4f740c6226..6cb7bbc568532 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java @@ -115,11 +115,11 @@ public static GlobalStateMgr createTestState() throws InstantiationException, Il } public static boolean compareState(GlobalStateMgr masterGlobalStateMgr, GlobalStateMgr slaveGlobalStateMgr) { - Database masterDb = masterGlobalStateMgr.getDb(testDb1); - Database slaveDb = slaveGlobalStateMgr.getDb(testDb1); + Database masterDb = masterGlobalStateMgr.getLocalMetastore().getDb(testDb1); + Database slaveDb = slaveGlobalStateMgr.getLocalMetastore().getDb(testDb1); List
tables = masterDb.getTables(); for (Table table : tables) { - Table slaveTable = slaveDb.getTable(table.getId()); + Table slaveTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(slaveDb.getId(), table.getId()); if (slaveTable == null) { return false; } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/InfoSchemaDbTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/InfoSchemaDbTest.java index 14a92a6777023..96ebdb24f105f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/InfoSchemaDbTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/InfoSchemaDbTest.java @@ -107,7 +107,7 @@ public void testNormal() throws IOException { Assert.assertFalse(db.registerTableUnlocked(null)); db.dropTable("authors"); db.write(null); - Assert.assertNull(db.getTable("authors")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "authors")); } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/MVPartitionExprResolverTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/MVPartitionExprResolverTest.java index 9b993621ca093..1aa5f1248c118 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/MVPartitionExprResolverTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/MVPartitionExprResolverTest.java @@ -240,7 +240,7 @@ private Map checkMVPartitionExprs(String sql, Expr slot, int expe @Test public void testFilterPartitionByUnion() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withMaterializedView( "create materialized view test.mv_union_filter\n" + @@ -252,7 +252,9 @@ public void testFilterPartitionByUnion() { "union " + "select k1, k2 from test.tbl16;", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_union_filter")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_union_filter")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); @@ -262,8 +264,12 @@ public void testFilterPartitionByUnion() { PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); Map> baseTables = getRefTableRefreshedPartitions(processor); Assert.assertEquals(2, baseTables.size()); - Assert.assertEquals(Sets.newHashSet("p20220101"), baseTables.get(testDb.getTable("tbl15"))); - Assert.assertEquals(Sets.newHashSet("p20220101"), baseTables.get(testDb.getTable("tbl16"))); + Assert.assertEquals(Sets.newHashSet("p20220101"), baseTables.get( + GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tbl15"))); + Assert.assertEquals(Sets.newHashSet("p20220101"), baseTables.get( + GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tbl16"))); // insert new data into tbl16's p20220202 partition String insertSql = "insert into tbl16 partition(p20220202) values('2022-02-02', 3, 10);"; @@ -287,7 +293,9 @@ public void testFilterPartitionByUnion() { "union " + "select k1, k2 from test.tbl15;", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_union_filter")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_union_filter")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); @@ -310,7 +318,7 @@ public void testFilterPartitionByUnion() { @Test public void testFilterPartitionByJoinPredicate1() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); //normal case, join predicate is partition column // a.k1 = date_trunc(month, b.k1) starRocksAssert.withMaterializedView( @@ -322,7 +330,9 @@ public void testFilterPartitionByJoinPredicate1() { "as select a.k1, b.k2 from test.tbl15 as a join test.tbl16 as b " + "on a.k1 = date_trunc('month', b.k1);", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); @@ -330,13 +340,16 @@ public void testFilterPartitionByJoinPredicate1() { { taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); Map> baseTables = getRefTableRefreshedPartitions(processor); Assert.assertEquals(2, baseTables.size()); Assert.assertEquals(Sets.newHashSet("p20220101"), - baseTables.get(testDb.getTable("tbl15"))); + baseTables.get(GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tbl15"))); Assert.assertEquals(Sets.newHashSet("p20220101", "p20220102", "p20220103"), - baseTables.get(testDb.getTable("tbl16"))); + baseTables.get(GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tbl16"))); Assert.assertTrue(processor.getNextTaskRun() == null); } @@ -346,7 +359,8 @@ public void testFilterPartitionByJoinPredicate1() { new StmtExecutor(connectContext, SqlParser.parseSingleStatement( insertSql, connectContext.getSessionVariable().getSqlMode())).execute(); taskRun.executeTaskRun(); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); // 1. updated partition of tbl16 is p20220202 // 2. date_trunc('month', p20220202) is '2022-02' // 3. tbl15's associated partitions are p20220201 and p20220202 @@ -360,7 +374,7 @@ public void testFilterPartitionByJoinPredicate1() { @Test public void testFilterPartitionByJoinPredicate3() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // join predicate has no equal condition starRocksAssert.withMaterializedView( "create materialized view test.mv_join_predicate\n" + @@ -368,9 +382,12 @@ public void testFilterPartitionByJoinPredicate3() { "distributed by hash(k2) buckets 10\n" + "PROPERTIES('partition_refresh_number' = '10000')" + "refresh deferred manual\n" + - "as select tbl1.k1, tbl2.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1 or tbl1.k2 = tbl2.k2;", () -> { + "as select tbl1.k1, tbl2.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1 or tbl1.k2 = tbl2.k2;", + () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -381,7 +398,8 @@ public void testFilterPartitionByJoinPredicate3() { executeInsertSql(connectContext, "insert into tbl1 partition(p2) values('2022-02-02', 3, 10);"); taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); Assert.assertEquals(Sets.newHashSet("p2"), processor.getMVTaskRunExtraMessage().getMvPartitionsToRefresh()); ExecPlan execPlan = processor.getMvContext().getExecPlan(); @@ -394,17 +412,19 @@ public void testFilterPartitionByJoinPredicate3() { executeInsertSql(connectContext, "insert into tbl2 partition(p2) values('2022-02-02', 3, 10);"); taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); ExecPlan execPlan = processor.getMvContext().getExecPlan(); Assert.assertTrue(execPlan != null); - assertPlanContains(execPlan, "partitions=5/5\n rollup: tbl1", "partitions=2/2\n rollup: tbl2"); + assertPlanContains(execPlan, "partitions=5/5\n rollup: tbl1", + "partitions=2/2\n rollup: tbl2"); } }); } @Test public void testFilterPartitionByJoinPredicate31() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // join predicate is not mv partition expr starRocksAssert.withMaterializedView( "create materialized view test.mv_join_predicate\n" + @@ -414,14 +434,16 @@ public void testFilterPartitionByJoinPredicate31() { "refresh deferred manual\n" + "as select tbl1.k1, tbl2.k2 from tbl1 join tbl2 using(k1);", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); }); } @Test public void testFilterPartitionByJoinPredicate4() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // nest table alias join starRocksAssert.withMaterializedView( "create materialized view test.mv_join_predicate\n" + @@ -431,7 +453,9 @@ public void testFilterPartitionByJoinPredicate4() { "as select a.k1, b.k2 from test.tbl15 as a join test.tbl16 as b " + "on date_trunc('month', a.k1) = b.k1;", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -452,7 +476,7 @@ public void testFilterPartitionByJoinPredicate4() { @Test public void testFilterPartitionByJoinPredicate5() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // nest table alias join starRocksAssert.withMaterializedView( "create materialized view test.mv_join_predicate\n" + @@ -468,7 +492,9 @@ public void testFilterPartitionByJoinPredicate5() { " (select date_trunc('DAY', k1) as ds, k2 from (select * from tbl2)t ) b " + "on date_trunc('DAY', a.k1) = b.ds and a.k2 = b.k2", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -498,7 +524,7 @@ public void testFilterPartitionByJoinPredicate5() { @Test public void testFilterPartitionByJoinPredicate6() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // nest function in join predicate starRocksAssert.useDatabase("test").withMaterializedView( "create materialized view test.mv_join_predicate\n" + @@ -512,14 +538,16 @@ public void testFilterPartitionByJoinPredicate6() { "(select k1 as ds, k2 from tbl2) b " + "on date_trunc('day', a.ds) = b.ds and a.k2 = b.k2 ;", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); }); } @Test public void testFilterPartitionByJoinPredicate7() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // duplicate table join starRocksAssert.withMaterializedView( "create materialized view test.mv_join_predicate\n" + @@ -534,14 +562,16 @@ public void testFilterPartitionByJoinPredicate7() { "(select k1 as ds, k2 from tbl2) c " + "on a.ds = c.ds and a.k2 = c.k2;", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); }); } @Test public void testFilterPartitionByJoinPredicate8() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withMaterializedView( "create materialized view test.mv_join_predicate\n" + "partition by k1\n" + @@ -557,14 +587,16 @@ public void testFilterPartitionByJoinPredicate8() { "(select date_trunc('day', k1) as ds, k2 from tbl1) c " + "on a.ds = c.ds;", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); }); } @Test public void testFilterPartitionByJoinPredicate9() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); // unsupported function in join predicate starRocksAssert.withMaterializedView( "create materialized view test.mv_join_predicate\n" + @@ -579,7 +611,9 @@ public void testFilterPartitionByJoinPredicate9() { "(select date_add(k1, INTERVAL 1 DAY) as ds, k2 from tbl15) c " + "on a.ds = c.ds and a.k2 = c.k2;", () -> { - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_join_predicate")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_join_predicate")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/MaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/MaterializedViewTest.java index b198f075bf934..8562e8ace445d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/MaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/MaterializedViewTest.java @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.catalog; import com.google.common.base.Strings; @@ -91,7 +90,7 @@ public void testInit() { Assert.assertEquals(null, mv.getTableProperty()); MaterializedView mv2 = new MaterializedView(1000, 100, "mv2", columns, KeysType.AGG_KEYS, - null, null, null); + null, null, null); Assert.assertEquals(100, mv2.getDbId()); Assert.assertEquals(Table.TableType.MATERIALIZED_VIEW, mv2.getType()); Assert.assertEquals(null, mv2.getTableProperty()); @@ -131,16 +130,16 @@ public void testInit() { @Test public void testSchema() { MaterializedView mv = new MaterializedView(1000, 100, "mv2", columns, KeysType.AGG_KEYS, - null, null, null); + null, null, null); mv.setBaseIndexId(1L); mv.setIndexMeta(1L, "mv_name", columns, 0, - 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); + 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); Assert.assertEquals(1, mv.getBaseIndexId()); mv.rebuildFullSchema(); Assert.assertEquals("mv_name", mv.getIndexNameById(1L)); List indexColumns = Lists.newArrayList(columns.get(0), columns.get(2)); mv.setIndexMeta(2L, "index_name", indexColumns, 0, - 222, (short) 1, TStorageType.COLUMN, KeysType.AGG_KEYS, null); + 222, (short) 1, TStorageType.COLUMN, KeysType.AGG_KEYS, null); mv.rebuildFullSchema(); Assert.assertEquals("index_name", mv.getIndexNameById(2L)); } @@ -157,9 +156,9 @@ public void testPartition() { MaterializedView.MvRefreshScheme refreshScheme = new MaterializedView.MvRefreshScheme(); MaterializedView mv = new MaterializedView(1000, 100, "mv_name", columns, KeysType.AGG_KEYS, - partitionInfo, distributionInfo, refreshScheme); + partitionInfo, distributionInfo, refreshScheme); mv.setIndexMeta(1L, "mv_name", columns, 0, - 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); + 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); Assert.assertEquals("mv_name", mv.getName()); mv.setName("new_name"); Assert.assertEquals("new_name", mv.getName()); @@ -184,9 +183,9 @@ public void testPartition() { rangePartitionInfo.setTabletType(1, TTabletType.TABLET_TYPE_DISK); MaterializedView mv2 = new MaterializedView(1000, 100, "mv_name_2", columns, KeysType.AGG_KEYS, - rangePartitionInfo, distributionInfo, refreshScheme); + rangePartitionInfo, distributionInfo, refreshScheme); mv2.setIndexMeta(1L, "mv_name_2", columns, 0, - 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); + 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); Assert.assertEquals("mv_name_2", mv2.getName()); mv2.setName("new_name_2"); Assert.assertEquals("new_name_2", mv2.getName()); @@ -214,16 +213,16 @@ public void testDistribution() { MaterializedView.MvRefreshScheme refreshScheme = new MaterializedView.MvRefreshScheme(); MaterializedView mv = new MaterializedView(1000, 100, "mv_name", columns, KeysType.AGG_KEYS, - partitionInfo, distributionInfo, refreshScheme); + partitionInfo, distributionInfo, refreshScheme); mv.setIndexMeta(1L, "mv_name", columns, 0, - 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); + 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); DistributionInfo distributionInfo1 = mv.getDefaultDistributionInfo(); Assert.assertTrue(distributionInfo1 instanceof RandomDistributionInfo); Assert.assertEquals(0, mv.getDistributionColumnNames().size()); HashDistributionInfo hashDistributionInfo = new HashDistributionInfo(10, Lists.newArrayList(columns.get(0))); MaterializedView mv2 = new MaterializedView(1000, 100, "mv_name", columns, KeysType.AGG_KEYS, - partitionInfo, hashDistributionInfo, refreshScheme); + partitionInfo, hashDistributionInfo, refreshScheme); DistributionInfo distributionInfo2 = mv2.getDefaultDistributionInfo(); Assert.assertTrue(distributionInfo2 instanceof HashDistributionInfo); Assert.assertEquals(1, mv2.getDistributionColumnNames().size()); @@ -240,9 +239,9 @@ public void testToThrift() { MaterializedView.MvRefreshScheme refreshScheme = new MaterializedView.MvRefreshScheme(); MaterializedView mv = new MaterializedView(1000, 100, "mv_name", columns, KeysType.AGG_KEYS, - partitionInfo, distributionInfo, refreshScheme); + partitionInfo, distributionInfo, refreshScheme); mv.setIndexMeta(1L, "mv_name", columns, 0, - 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); + 111, (short) 2, TStorageType.COLUMN, KeysType.AGG_KEYS, null); TTableDescriptor tableDescriptor = mv.toThrift(null); Assert.assertEquals(TTableType.MATERIALIZED_VIEW, tableDescriptor.getTableType()); Assert.assertEquals(1000, tableDescriptor.getId()); @@ -252,40 +251,40 @@ public void testToThrift() { @Test public void testRenameMaterializedView() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withMaterializedView("create materialized view mv_to_rename\n" + - "PARTITION BY k1\n" + - "distributed by hash(k2) buckets 3\n" + - "refresh async\n" + - "as select k1, k2, sum(v1) as total from tbl1 group by k1, k2;") - .withMaterializedView("create materialized view mv_to_rename2\n" + - "PARTITION BY date_trunc('month', k1)\n" + - "distributed by hash(k2) buckets 3\n" + - "refresh async\n" + - "as select k1, k2, sum(v1) as total from tbl1 group by k1, k2;"); - - Database db = connectContext.getGlobalStateMgr().getDb("test"); + .withTable("CREATE TABLE test.tbl1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withMaterializedView("create materialized view mv_to_rename\n" + + "PARTITION BY k1\n" + + "distributed by hash(k2) buckets 3\n" + + "refresh async\n" + + "as select k1, k2, sum(v1) as total from tbl1 group by k1, k2;") + .withMaterializedView("create materialized view mv_to_rename2\n" + + "PARTITION BY date_trunc('month', k1)\n" + + "distributed by hash(k2) buckets 3\n" + + "refresh async\n" + + "as select k1, k2, sum(v1) as total from tbl1 group by k1, k2;"); + + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); Assert.assertNotNull(db); - Table table = db.getTable("mv_to_rename"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv_to_rename"); Assert.assertNotNull(table); // test partition related info MaterializedView oldMv = (MaterializedView) table; Assert.assertTrue(oldMv.getRefreshScheme().isAsync()); Assert.assertTrue(oldMv.getRefreshScheme().toString().contains("MvRefreshScheme")); Map partitionMap = oldMv.getRefBaseTablePartitionColumns(); - Table table1 = db.getTable("tbl1"); + Table table1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); Assert.assertTrue(partitionMap.containsKey(table1)); List baseTableType = oldMv.getBaseTableTypes(); Assert.assertEquals(1, baseTableType.size()); @@ -301,8 +300,9 @@ public void testRenameMaterializedView() throws Exception { StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement); stmtExecutor.execute(); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable("mv_new_name")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_new_name")); Assert.assertNotNull(mv); Assert.assertEquals("mv_new_name", mv.getName()); ExpressionRangePartitionInfo partitionInfo = (ExpressionRangePartitionInfo) mv.getPartitionInfo(); @@ -317,7 +317,8 @@ public void testRenameMaterializedView() throws Exception { statement = SqlParser.parseSingleStatement(alterSql2, connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor2 = new StmtExecutor(connectContext, statement); stmtExecutor2.execute(); - MaterializedView mv2 = ((MaterializedView) testDb.getTable("mv_new_name2")); + MaterializedView mv2 = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_new_name2")); Assert.assertNotNull(mv2); Assert.assertEquals("mv_new_name2", mv2.getName()); ExpressionRangePartitionInfo partitionInfo2 = (ExpressionRangePartitionInfo) mv2.getPartitionInfo(); @@ -334,34 +335,34 @@ public void testRenameMaterializedView() throws Exception { @Test public void testReplay() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withMaterializedView("create materialized view mv_replay\n" + - "PARTITION BY k1\n" + - "distributed by hash(k2) buckets 3\n" + - "refresh async\n" + - "as select k1, k2, sum(v1) as total from tbl1 group by k1, k2;"); + .withTable("CREATE TABLE test.tbl1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withMaterializedView("create materialized view mv_replay\n" + + "PARTITION BY k1\n" + + "distributed by hash(k2) buckets 3\n" + + "refresh async\n" + + "as select k1, k2, sum(v1) as total from tbl1 group by k1, k2;"); connectContext.executeSql("insert into test.tbl1 values('2022-02-01', 2, 3)"); connectContext.executeSql("insert into test.tbl1 values('2022-02-16', 3, 5)"); connectContext.executeSql("refresh materialized view mv_replay with sync mode"); - Database db = connectContext.getGlobalStateMgr().getDb("test"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); Assert.assertNotNull(db); - Table table = db.getTable("mv_replay"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv_replay"); MaterializedView mv = (MaterializedView) table; AlterMaterializedViewBaseTableInfosLog log = new AlterMaterializedViewBaseTableInfosLog(db.getId(), mv.getId(), null, - mv.getBaseTableInfos(), mv.getRefreshScheme().getAsyncRefreshContext().getBaseTableVisibleVersionMap()); + mv.getBaseTableInfos(), mv.getRefreshScheme().getAsyncRefreshContext().getBaseTableVisibleVersionMap()); mv.replayAlterMaterializedViewBaseTableInfos(log); starRocksAssert.dropMaterializedView("mv_replay"); @@ -370,25 +371,26 @@ public void testReplay() throws Exception { @Test public void testMvAfterDropBaseTable() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl_drop\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withMaterializedView("create materialized view mv_to_check\n" + - "distributed by hash(k2) buckets 3\n" + - "refresh async\n" + - "as select k2, sum(v1) as total from tbl_drop group by k2;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable("mv_to_check")); + .withTable("CREATE TABLE test.tbl_drop\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withMaterializedView("create materialized view mv_to_check\n" + + "distributed by hash(k2) buckets 3\n" + + "refresh async\n" + + "as select k2, sum(v1) as total from tbl_drop group by k2;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_to_check")); String dropSql = "drop table tbl_drop;"; StatementBase statement = SqlParser.parseSingleStatement(dropSql, connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement); @@ -400,29 +402,30 @@ public void testMvAfterDropBaseTable() throws Exception { @Test public void testMvAfterBaseTableRename() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl_to_rename\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withMaterializedView("create materialized view mv_to_check\n" + - "distributed by hash(k2) buckets 3\n" + - "refresh async\n" + - "as select k2, sum(v1) as total from tbl_to_rename group by k2;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + .withTable("CREATE TABLE test.tbl_to_rename\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withMaterializedView("create materialized view mv_to_check\n" + + "distributed by hash(k2) buckets 3\n" + + "refresh async\n" + + "as select k2, sum(v1) as total from tbl_to_rename group by k2;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSql = "alter table tbl_to_rename rename new_tbl_name;"; StatementBase statement = SqlParser.parseSingleStatement(alterSql, connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement); stmtExecutor.execute(); - MaterializedView mv = ((MaterializedView) testDb.getTable("mv_to_check")); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_to_check")); Assert.assertNotNull(mv); Assert.assertFalse(mv.isActive()); } @@ -430,26 +433,28 @@ public void testMvAfterBaseTableRename() throws Exception { @Test public void testMaterializedViewWithHint() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withMaterializedView("create materialized view mv_with_hint\n" + - "PARTITION BY k1\n" + - "distributed by hash(k2) buckets 3\n" + - "refresh async\n" + - "as select /*+ SET_VAR(query_timeout = 500) */ k1, k2, sum(v1) as total from tbl1 group by k1, k2;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable("mv_with_hint")); + .withTable("CREATE TABLE test.tbl1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withMaterializedView("create materialized view mv_with_hint\n" + + "PARTITION BY k1\n" + + "distributed by hash(k2) buckets 3\n" + + "refresh async\n" + + "as select /*+ SET_VAR(query_timeout = 500) */ k1, k2, sum(v1) " + + "as total from tbl1 group by k1, k2;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_with_hint")); String mvTaskName = "mv-" + mv.getId(); Task task = connectContext.getGlobalStateMgr().getTaskManager().getTask(mvTaskName); Assert.assertNotNull(task); @@ -463,24 +468,24 @@ public void testMaterializedViewWithHint() throws Exception { @Test public void testRollupMaterializedViewWithScalarFunction() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE `part_with_mv` (\n" + - " `p_partkey` int(11) NOT NULL COMMENT \"\",\n" + - " `p_name` varchar(55) NOT NULL COMMENT \"\",\n" + - " `p_mfgr` varchar(25) NOT NULL COMMENT \"\",\n" + - " `p_brand` varchar(10) NOT NULL COMMENT \"\",\n" + - " `p_type` varchar(25) NOT NULL COMMENT \"\",\n" + - " `p_size` int(11) NOT NULL COMMENT \"\",\n" + - " `p_container` varchar(10) NOT NULL COMMENT \"\",\n" + - " `p_retailprice` decimal64(15, 2) NOT NULL COMMENT \"\",\n" + - " `p_comment` varchar(23) NOT NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`p_partkey`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 24\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\");"); + .withTable("CREATE TABLE `part_with_mv` (\n" + + " `p_partkey` int(11) NOT NULL COMMENT \"\",\n" + + " `p_name` varchar(55) NOT NULL COMMENT \"\",\n" + + " `p_mfgr` varchar(25) NOT NULL COMMENT \"\",\n" + + " `p_brand` varchar(10) NOT NULL COMMENT \"\",\n" + + " `p_type` varchar(25) NOT NULL COMMENT \"\",\n" + + " `p_size` int(11) NOT NULL COMMENT \"\",\n" + + " `p_container` varchar(10) NOT NULL COMMENT \"\",\n" + + " `p_retailprice` decimal64(15, 2) NOT NULL COMMENT \"\",\n" + + " `p_comment` varchar(23) NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`p_partkey`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 24\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\");"); String createMvSql = "create materialized view mv1 as select p_partkey, p_name, length(p_brand) as v1 " + - "from part_with_mv;"; + "from part_with_mv;"; StatementBase statement = SqlParser.parseSingleStatement(createMvSql, connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement); stmtExecutor.execute(); @@ -490,56 +495,57 @@ public void testRollupMaterializedViewWithScalarFunction() throws Exception { @Test(expected = SemanticException.class) public void testNonPartitionMvSupportedProperties() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE goods(\n" + - "item_id1 INT,\n" + - "item_name STRING,\n" + - "price FLOAT\n" + - ") DISTRIBUTED BY HASH(item_id1)\n" + - "PROPERTIES(\"replication_num\" = \"1\");"); + .withTable("CREATE TABLE goods(\n" + + "item_id1 INT,\n" + + "item_name STRING,\n" + + "price FLOAT\n" + + ") DISTRIBUTED BY HASH(item_id1)\n" + + "PROPERTIES(\"replication_num\" = \"1\");"); starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW order_mv\n" + - "DISTRIBUTED BY HASH(item_id1) BUCKETS 12\n" + - "PROPERTIES (\n" + - "\"partition_refresh_number\" = \"10\"\n" + - ")\n" + - "REFRESH ASYNC\n" + - "AS SELECT\n" + - "item_id1,\n" + - "sum(price) as total\n" + - "FROM goods\n" + - "GROUP BY item_id1;"); + "DISTRIBUTED BY HASH(item_id1) BUCKETS 12\n" + + "PROPERTIES (\n" + + "\"partition_refresh_number\" = \"10\"\n" + + ")\n" + + "REFRESH ASYNC\n" + + "AS SELECT\n" + + "item_id1,\n" + + "sum(price) as total\n" + + "FROM goods\n" + + "GROUP BY item_id1;"); } @Test public void testCreateMaterializedViewWithInactiveMaterializedView() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE base_table\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withMaterializedView("CREATE MATERIALIZED VIEW base_mv\n" + - "PARTITION BY k1\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "REFRESH manual\n" + - "as select k1,k2,v1 from base_table;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView baseMv = ((MaterializedView) testDb.getTable("base_mv")); + .withTable("CREATE TABLE base_table\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withMaterializedView("CREATE MATERIALIZED VIEW base_mv\n" + + "PARTITION BY k1\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "REFRESH manual\n" + + "as select k1,k2,v1 from base_table;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView baseMv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "base_mv")); baseMv.setInactiveAndReason(""); SinglePartitionInfo singlePartitionInfo = new SinglePartitionInfo(); MaterializedView.MvRefreshScheme refreshScheme = new MaterializedView.MvRefreshScheme(); HashDistributionInfo hashDistributionInfo = new HashDistributionInfo(3, Lists.newArrayList(columns.get(0))); MaterializedView mv = new MaterializedView(1000, testDb.getId(), "mv", columns, KeysType.AGG_KEYS, - singlePartitionInfo, hashDistributionInfo, refreshScheme); + singlePartitionInfo, hashDistributionInfo, refreshScheme); List baseTableInfos = Lists.newArrayList(); BaseTableInfo baseTableInfo = new BaseTableInfo(testDb.getId(), testDb.getFullName(), baseMv.getName(), baseMv.getId()); baseTableInfos.add(baseTableInfo); @@ -572,22 +578,22 @@ public void testShouldRefreshBy() { @Test public void testShowSyncMV() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl_sync_mv\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withMaterializedView("create materialized view sync_mv_to_check\n" + - "distributed by hash(k2) buckets 3\n" + - "as select k2, sum(v1) as total from tbl_sync_mv group by k2;"); + .withTable("CREATE TABLE test.tbl_sync_mv\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withMaterializedView("create materialized view sync_mv_to_check\n" + + "distributed by hash(k2) buckets 3\n" + + "as select k2, sum(v1) as total from tbl_sync_mv group by k2;"); String showSql = "show create materialized view sync_mv_to_check;"; StatementBase statement = SqlParser.parseSingleStatement(showSql, connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement); @@ -615,7 +621,7 @@ public void testAlterMVWithIndex() throws Exception { "distributed by hash(k2) buckets 3\n" + "as select k2, sum(v1) as total from table1 group by k2;"); - Database db = connectContext.getGlobalStateMgr().getDb("test"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); Assert.assertNotNull(db); Table table = db.getTable("index_mv_to_check"); Assert.assertNotNull(table); @@ -641,7 +647,7 @@ public void testShowMVWithIndex() throws Exception { testAlterMVWithIndex(); String showCreateSql = "show create materialized view test.index_mv_to_check;"; ShowCreateTableStmt showCreateTableStmt = - (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateSql, connectContext); + (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateSql, connectContext); ShowResultSet showResultSet = ShowExecutor.execute(showCreateTableStmt, connectContext); System.out.println(showResultSet.getMetaData().toString()); System.out.println(showResultSet.getResultRows()); @@ -650,47 +656,47 @@ public void testShowMVWithIndex() throws Exception { @Test public void testAlterViewWithIndex() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.table1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');") - .withView("create view index_view_to_check\n" + - "as select k2, sum(v1) as total from table1 group by k2;"); + .withTable("CREATE TABLE test.table1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');") + .withView("create view index_view_to_check\n" + + "as select k2, sum(v1) as total from table1 group by k2;"); String bitmapSql = "create index index1 ON test.index_view_to_check (k2) USING BITMAP COMMENT 'balabala'"; AlterTableStmt alterViewStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(bitmapSql, connectContext); Assert.assertThrows("Do not support alter non-native table/materialized-view[index_view_to_check]", - SemanticException.class, - () -> DDLStmtExecutor.execute(alterViewStmt, connectContext)); + SemanticException.class, + () -> DDLStmtExecutor.execute(alterViewStmt, connectContext)); } public void testCreateMV(String mvSql) throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.table1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.table1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); starRocksAssert.withMaterializedView(mvSql); String showCreateSql = "show create materialized view test.index_mv_to_check;"; ShowCreateTableStmt showCreateTableStmt = - (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateSql, connectContext); + (ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(showCreateSql, connectContext); ShowResultSet showResultSet = ShowExecutor.execute(showCreateTableStmt, connectContext); System.out.println(showResultSet.getResultRows()); } @@ -698,109 +704,108 @@ public void testCreateMV(String mvSql) throws Exception { @Test public void testCreateMVWithIndex() throws Exception { String mvSqlWithBitMapAndBloomfilter = "create materialized view index_mv_to_check " + - "(k2 ," + - " total ," + - "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala' " + - ")" + - "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + - "REFRESH MANUAL\n" + - "PROPERTIES " + - "(" - + "\"replicated_storage\" = \"true\"," - + "\"replication_num\" = \"1\"," - + "\"storage_medium\" = \"HDD\"," - + "\"bloom_filter_columns\" = \"k2\"" - + ")" + - "as select k2, sum(v1) as total from table1 group by k2;"; + "(k2 ," + + " total ," + + "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala' " + + ")" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + + "REFRESH MANUAL\n" + + "PROPERTIES " + + "(" + + "\"replicated_storage\" = \"true\"," + + "\"replication_num\" = \"1\"," + + "\"storage_medium\" = \"HDD\"," + + "\"bloom_filter_columns\" = \"k2\"" + + ")" + + "as select k2, sum(v1) as total from table1 group by k2;"; String mvSqlWithBitMap = "create materialized view index_mv_to_check " + - "(k2 ," + - " total ," + - "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala' " + - ")" + - "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + - "REFRESH MANUAL\n" + - "PROPERTIES " + - "(" - + "\"replicated_storage\" = \"true\"," - + "\"replication_num\" = \"1\"," - + "\"storage_medium\" = \"HDD\"" - + ")" + - "as select k2, sum(v1) as total from table1 group by k2;"; + "(k2 ," + + " total ," + + "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala' " + + ")" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + + "REFRESH MANUAL\n" + + "PROPERTIES " + + "(" + + "\"replicated_storage\" = \"true\"," + + "\"replication_num\" = \"1\"," + + "\"storage_medium\" = \"HDD\"" + + ")" + + "as select k2, sum(v1) as total from table1 group by k2;"; String mvSqlWithBloomFilter = "create materialized view index_mv_to_check " + - "(k2 ," + - " total" + - ")" + - "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + - "REFRESH MANUAL\n" + - "PROPERTIES " + - "(" - + "\"replicated_storage\" = \"true\"," - + "\"replication_num\" = \"1\"," - + "\"storage_medium\" = \"HDD\"," - + "\"bloom_filter_columns\" = \"k2\"" - + ")" + - "as select k2, sum(v1) as total from table1 group by k2;"; + "(k2 ," + + " total" + + ")" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + + "REFRESH MANUAL\n" + + "PROPERTIES " + + "(" + + "\"replicated_storage\" = \"true\"," + + "\"replication_num\" = \"1\"," + + "\"storage_medium\" = \"HDD\"," + + "\"bloom_filter_columns\" = \"k2\"" + + ")" + + "as select k2, sum(v1) as total from table1 group by k2;"; testCreateMV(mvSqlWithBitMapAndBloomfilter); testCreateMV(mvSqlWithBitMap); testCreateMV(mvSqlWithBloomFilter); } - @Test public void testCreateMVWithDuplicateIndexOrDuplicateColumn() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.table1\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.table1\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String mvSql = "create materialized view index_mv_to_check " + - "(k2 ," + - " total ," + - "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala', " + - "INDEX index1 (`total`) USING BITMAP COMMENT 'balabala' " + - ")" + - "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + - "REFRESH MANUAL\n" + - "PROPERTIES " + - "(" - + "\"replicated_storage\" = \"true\"," - + "\"replication_num\" = \"1\"," - + "\"storage_medium\" = \"HDD\"," - + "\"bloom_filter_columns\" = \"k2\"" - + ")" + - "as select k2, sum(v1) as total from table1 group by k2;"; + "(k2 ," + + " total ," + + "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala', " + + "INDEX index1 (`total`) USING BITMAP COMMENT 'balabala' " + + ")" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + + "REFRESH MANUAL\n" + + "PROPERTIES " + + "(" + + "\"replicated_storage\" = \"true\"," + + "\"replication_num\" = \"1\"," + + "\"storage_medium\" = \"HDD\"," + + "\"bloom_filter_columns\" = \"k2\"" + + ")" + + "as select k2, sum(v1) as total from table1 group by k2;"; Assert.assertThrows("Duplicate index name 'index1'", - UserException.class, - () -> starRocksAssert.withMaterializedView(mvSql)); + UserException.class, + () -> starRocksAssert.withMaterializedView(mvSql)); String mvSql2 = "create materialized view index_mv_to_check " + - "(k2 ," + - " total ," + - "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala', " + - "INDEX index2 (`k2`) USING BITMAP COMMENT 'balabala' " + - ")" + - "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + - "REFRESH MANUAL\n" + - "PROPERTIES " + - "(" - + "\"replicated_storage\" = \"true\"," - + "\"replication_num\" = \"1\"," - + "\"storage_medium\" = \"HDD\"," - + "\"bloom_filter_columns\" = \"k2\"" - + ")" + - "as select k2, sum(v1) as total from table1 group by k2;"; + "(k2 ," + + " total ," + + "INDEX index1 (`k2`) USING BITMAP COMMENT 'balabala', " + + "INDEX index2 (`k2`) USING BITMAP COMMENT 'balabala' " + + ")" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3 \n" + + "REFRESH MANUAL\n" + + "PROPERTIES " + + "(" + + "\"replicated_storage\" = \"true\"," + + "\"replication_num\" = \"1\"," + + "\"storage_medium\" = \"HDD\"," + + "\"bloom_filter_columns\" = \"k2\"" + + ")" + + "as select k2, sum(v1) as total from table1 group by k2;"; Assert.assertThrows("Duplicate column name 'k2' in index", - UserException.class, - () -> starRocksAssert.withMaterializedView(mvSql2)); + UserException.class, + () -> starRocksAssert.withMaterializedView(mvSql2)); } @Test @@ -813,6 +818,6 @@ public void testBasePartitionInfo() { Assert.assertEquals(100, basePartitionInfo.getExtLastFileModifiedTime()); Assert.assertEquals(10, basePartitionInfo.getFileNumber()); Assert.assertTrue(basePartitionInfo.toString().contains( - "BasePartitionInfo{id=-1, version=-1, lastRefreshTime=123456, lastFileModifiedTime=100, fileNumber=10}")); + "BasePartitionInfo{id=-1, version=-1, lastRefreshTime=123456, lastFileModifiedTime=100, fileNumber=10}")); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/MetadataViewerTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/MetadataViewerTest.java index 0f49ed124eaff..e3b1eab9679bb 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/MetadataViewerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/MetadataViewerTest.java @@ -96,9 +96,13 @@ public void before() { minTimes = 0; result = globalStateMgr; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = db; + + globalStateMgr.getLocalMetastore().getTable(anyString, anyString); + minTimes = 0; + result = db.getTable(CatalogMocker.TEST_TBL_NAME); } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/StorageCoolDownTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/StorageCoolDownTest.java index 7983b395f4db8..0afeeb9ba98b3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/StorageCoolDownTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/StorageCoolDownTest.java @@ -142,7 +142,7 @@ public void testDateWithTTLLessThan() throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") .getTable("site_access_datetime_with_1_day_ttl_less_than"); RangePartitionInfo partitionInfo = (RangePartitionInfo) table.getPartitionInfo(); @@ -200,7 +200,7 @@ public void testDateWithTTLUpperLower() throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") .getTable("site_access_date_upper_lower_ttl"); RangePartitionInfo partitionInfo = (RangePartitionInfo) table.getPartitionInfo(); @@ -250,7 +250,7 @@ public void testDateWithTTLStartEnd() throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") .getTable("site_access_date_with_1_day_ttl_start_end"); RangePartitionInfo partitionInfo = (RangePartitionInfo) table.getPartitionInfo(); @@ -303,7 +303,7 @@ public void testDateTimeWithTTLLessThan() throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") .getTable("site_access_date_with_1_day_ttl_less_than"); RangePartitionInfo partitionInfo = (RangePartitionInfo) table.getPartitionInfo(); @@ -356,7 +356,7 @@ public void testDateTimeWithMaxPartition() throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test") + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") .getTable("site_access_with_max_partition"); RangePartitionInfo partitionInfo = (RangePartitionInfo) table.getPartitionInfo(); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/StorageMediumInferTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/StorageMediumInferTest.java index 9fd815f8416f8..c91ada87c2f03 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/StorageMediumInferTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/StorageMediumInferTest.java @@ -85,14 +85,14 @@ private static void alterTableWithNewParser(String sql) throws Exception { @Test public void testCreateTable() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); be1.setStorageMediumForAllDisks(TStorageMedium.HDD); be2.setStorageMediumForAllDisks(TStorageMedium.HDD); createTable("create table test.tbl1(key1 int, key2 varchar(10)) \n" + "distributed by hash(key1) buckets 10 properties('replication_num' = '1');"); - OlapTable tbl1 = (OlapTable) db.getTable("tbl1"); + OlapTable tbl1 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); List partitionList1 = Lists.newArrayList(tbl1.getPartitions()); DataProperty dataProperty1 = globalStateMgr.getLocalMetastore().getDataPropertyIncludeRecycleBin(tbl1.getPartitionInfo(), @@ -105,7 +105,7 @@ public void testCreateTable() throws Exception { + "duplicate key(k1)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; createTable(sql); - OlapTable tbl2 = (OlapTable) db.getTable("tbl2"); + OlapTable tbl2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl2"); List partitionList2 = Lists.newArrayList(tbl2.getPartitions()); DataProperty dataProperty2 = globalStateMgr.getLocalMetastore().getDataPropertyIncludeRecycleBin(tbl2.getPartitionInfo(), @@ -117,7 +117,7 @@ public void testCreateTable() throws Exception { Config.tablet_sched_storage_cooldown_second = 123123213L; createTable("create table test.tbl3(key1 int, key2 varchar(10)) \n" + "distributed by hash(key1) buckets 10 properties('replication_num' = '1');"); - OlapTable tbl3 = (OlapTable) db.getTable("tbl3"); + OlapTable tbl3 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl3"); List partitionList3 = Lists.newArrayList(tbl3.getPartitions()); DataProperty dataProperty3 = globalStateMgr.getLocalMetastore().getDataPropertyIncludeRecycleBin(tbl3.getPartitionInfo(), @@ -127,7 +127,7 @@ public void testCreateTable() throws Exception { Config.tablet_sched_storage_cooldown_second = -1L; // default value, no storage cool down createTable("create table test.tbl4(key1 int, key2 varchar(10)) \n" + "distributed by hash(key1) buckets 10 properties('replication_num' = '1');"); - OlapTable tbl4 = (OlapTable) db.getTable("tbl4"); + OlapTable tbl4 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl4"); List partitionList4 = Lists.newArrayList(tbl4.getPartitions()); DataProperty dataProperty4 = globalStateMgr.getLocalMetastore().getDataPropertyIncludeRecycleBin(tbl4.getPartitionInfo(), @@ -137,7 +137,7 @@ public void testCreateTable() throws Exception { @Test public void testAlterTableAddPartition() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); be1.setStorageMediumForAllDisks(TStorageMedium.SSD); be2.setStorageMediumForAllDisks(TStorageMedium.SSD); @@ -146,7 +146,7 @@ public void testAlterTableAddPartition() throws Exception { + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "; createTable(sql); alterTableWithNewParser("ALTER TABLE test.tblp2 ADD PARTITION IF NOT EXISTS p2 VALUES LESS THAN (\"20\")"); - OlapTable tbl2 = (OlapTable) db.getTable("tblp2"); + OlapTable tbl2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tblp2"); List partitionList2 = Lists.newArrayList(tbl2.getPartitions()); Assert.assertEquals(2, partitionList2.size()); for (Partition partition : partitionList2) { diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java index cb4a0244d79d8..a25ca833f465d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java @@ -379,7 +379,7 @@ public void testForMultiPartitionTable() throws Exception { "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1');"); - Database db2 = GlobalStateMgr.getCurrentState().getDb("db2"); + Database db2 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db2"); OlapTable tbl2 = (OlapTable) db2.getTable("tbl2"); Map originPartitionTabletIds = Maps.newHashMap(); @@ -608,7 +608,7 @@ public void testForMultiPartitionTable() throws Exception { } OlapTable olapTable = - (OlapTable) GlobalStateMgr.getCurrentState().getDb("db2").getTable("tbl2"); + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db2").getTable("tbl2"); // waiting table state to normal int retryTimes = 5; @@ -675,7 +675,7 @@ public void testForStrictRangeCheck() throws Exception { "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1');"); - Database db3 = GlobalStateMgr.getCurrentState().getDb("db3"); + Database db3 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db3"); OlapTable tbl3 = (OlapTable) db3.getTable("tbl3"); // base range is [min, 10), [10, 20), [20, 30) diff --git a/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java b/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java index 2fb08ff1d8aeb..5687eb8d18e61 100644 --- a/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/clone/ColocateTableBalancerTest.java @@ -143,24 +143,25 @@ private ColocateTableIndex createColocateIndex(GroupId groupId, List flatL } private void addTabletsToScheduler(String dbName, String tableName, boolean setGroupId) { - Database database = GlobalStateMgr.getCurrentState().getDb(dbName); - OlapTable table = (OlapTable) database.getTable(tableName); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), tableName); // add its tablet to TabletScheduler TabletScheduler tabletScheduler = GlobalStateMgr.getCurrentState().getTabletScheduler(); for (Partition partition : table.getPartitions()) { MaterializedIndex materializedIndex = partition.getBaseIndex(); for (Tablet tablet : materializedIndex.getTablets()) { TabletSchedCtx ctx = new TabletSchedCtx(TabletSchedCtx.Type.REPAIR, - database.getId(), - table.getId(), - partition.getId(), - materializedIndex.getId(), - tablet.getId(), - System.currentTimeMillis()); + database.getId(), + table.getId(), + partition.getId(), + materializedIndex.getId(), + tablet.getId(), + System.currentTimeMillis()); ctx.setOrigPriority(TabletSchedCtx.Priority.LOW); if (setGroupId) { ctx.setColocateGroupId( - GlobalStateMgr.getCurrentState().getColocateTableIndex().getGroup(table.getId())); + GlobalStateMgr.getCurrentState().getColocateTableIndex().getGroup(table.getId())); } tabletScheduler.addTablet(ctx, false); } @@ -170,12 +171,13 @@ private void addTabletsToScheduler(String dbName, String tableName, boolean setG @Test public void test1MatchGroup() throws Exception { starRocksAssert.withDatabase("db1").useDatabase("db1") - .withTable("CREATE TABLE db1.tbl(id INT NOT NULL) " + - "distributed by hash(`id`) buckets 3 " + - "properties('replication_num' = '1', 'colocate_with' = 'group1');"); + .withTable("CREATE TABLE db1.tbl(id INT NOT NULL) " + + "distributed by hash(`id`) buckets 3 " + + "properties('replication_num' = '1', 'colocate_with' = 'group1');"); - Database database = GlobalStateMgr.getCurrentState().getDb("db1"); - OlapTable table = (OlapTable) database.getTable("tbl"); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "tbl"); addTabletsToScheduler("db1", "tbl", false); ColocateTableIndex colocateIndex = GlobalStateMgr.getCurrentState().getColocateTableIndex(); @@ -203,12 +205,13 @@ public void test3RepairWithBadReplica() throws Exception { UtFrameUtils.addMockBackend(10003); UtFrameUtils.addMockBackend(10004); starRocksAssert.withDatabase("db3").useDatabase("db3") - .withTable("CREATE TABLE db3.tbl3(id INT NOT NULL) " + - "distributed by hash(`id`) buckets 1 " + - "properties('replication_num' = '1', 'colocate_with' = 'group3');"); + .withTable("CREATE TABLE db3.tbl3(id INT NOT NULL) " + + "distributed by hash(`id`) buckets 1 " + + "properties('replication_num' = '1', 'colocate_with' = 'group3');"); - Database database = GlobalStateMgr.getCurrentState().getDb("db3"); - OlapTable table = (OlapTable) database.getTable("tbl3"); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db3"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "tbl3"); ColocateTableIndex colocateTableIndex = GlobalStateMgr.getCurrentState().getColocateTableIndex(); List partitions = Lists.newArrayList(table.getPartitions()); @@ -309,7 +312,7 @@ public void testRepairPrecedeBalance(@Mocked SystemInfoService infoService, GroupId groupId = new GroupId(10005, 10006); short replicationNUm = 3; ColocateTableIndex colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(1L, 2L, 3L, 1L, 2L, 3L, 1L, 2L, 4L, 1L, 2L, 5L), replicationNUm); + Lists.newArrayList(1L, 2L, 3L, 1L, 2L, 3L, 1L, 2L, 4L, 1L, 2L, 5L), replicationNUm); setGroup2Schema(groupId, colocateTableIndex, 4, replicationNUm); Set unavailableBeIds = Sets.newHashSet(5L); @@ -318,23 +321,23 @@ public void testRepairPrecedeBalance(@Mocked SystemInfoService infoService, boolean changed = false; ColocateTableBalancer.disableRepairPrecedence = true; changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); Assert.assertTrue(changed); System.out.println(balancedBackendsPerBucketSeq); List> expected = Lists.partition( - Lists.newArrayList(4L, 2L, 3L, 1L, 2L, 3L, 1L, 3L, 4L, 1L, 2L, 4L), 3); + Lists.newArrayList(4L, 2L, 3L, 1L, 2L, 3L, 1L, 3L, 4L, 1L, 2L, 4L), 3); Assert.assertEquals(expected, balancedBackendsPerBucketSeq); ColocateTableBalancer.disableRepairPrecedence = false; balancedBackendsPerBucketSeq.clear(); changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); Assert.assertTrue(changed); System.out.println(balancedBackendsPerBucketSeq); expected = Lists.partition( - Lists.newArrayList(1L, 2L, 3L, 1L, 2L, 3L, 1L, 2L, 4L, 1L, 2L, 4L), 3); + Lists.newArrayList(1L, 2L, 3L, 1L, 2L, 3L, 1L, 2L, 4L, 1L, 2L, 4L), 3); Assert.assertEquals(expected, balancedBackendsPerBucketSeq); } @@ -389,7 +392,6 @@ public void testPerGroupBalance(@Mocked SystemInfoService infoService, } }; - GroupId groupId = new GroupId(10000, 10001); List distributionCols = Lists.newArrayList(); distributionCols.add(new Column("k1", Type.INT)); @@ -401,27 +403,27 @@ public void testPerGroupBalance(@Mocked SystemInfoService infoService, // [[1, 2, 3], [4, 1, 2], [3, 4, 1], [2, 3, 4], [1, 2, 3]] FeConstants.runningUnitTest = true; ColocateTableIndex colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(1L, 2L, 3L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); + Lists.newArrayList(1L, 2L, 3L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); List> balancedBackendsPerBucketSeq = Lists.newArrayList(); List allAvailBackendIds = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); boolean changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, new HashSet(), allAvailBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, new HashSet(), allAvailBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); List> expected = Lists.partition( - Lists.newArrayList(9L, 5L, 3L, 4L, 6L, 8L, 7L, 6L, 1L, 2L, 9L, 4L, 1L, 2L, 3L), 3); + Lists.newArrayList(9L, 5L, 3L, 4L, 6L, 8L, 7L, 6L, 1L, 2L, 9L, 4L, 1L, 2L, 3L), 3); Assert.assertTrue(changed); Assert.assertEquals(expected, balancedBackendsPerBucketSeq); // 2. balance an already balanced group colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(9L, 8L, 7L, 8L, 6L, 5L, 9L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); + Lists.newArrayList(9L, 8L, 7L, 8L, 6L, 5L, 9L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); balancedBackendsPerBucketSeq.clear(); changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, new HashSet(), allAvailBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, new HashSet(), allAvailBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); System.out.println(balancedBackendsPerBucketSeq); Assert.assertFalse(changed); Assert.assertTrue(balancedBackendsPerBucketSeq.isEmpty()); @@ -516,10 +518,10 @@ public synchronized Map getTabletsNumInScheduleForEachCG() { ColocateTableIndex colocateTableIndex = GlobalStateMgr.getCurrentState().getColocateTableIndex(); // For group 1, bucket: 3, replication_num: 1, backend list per bucket seq: [[1], [2], [3]] colocateTableIndex.addBackendsPerBucketSeq(groupId1, - Lists.partition(Lists.newArrayList(1L, 2L, 3L), 1)); + Lists.partition(Lists.newArrayList(1L, 2L, 3L), 1)); // For group 2, bucket: 3, replication_num: 1, backend list per bucket seq: [[1], [2], [3]] colocateTableIndex.addBackendsPerBucketSeq(groupId2, - Lists.partition(Lists.newArrayList(1L, 2L, 3L), 1)); + Lists.partition(Lists.newArrayList(1L, 2L, 3L), 1)); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); Multimap group2Tables = ArrayListMultimap.create(); group2Tables.put(groupId1, 20001L); @@ -546,7 +548,7 @@ public synchronized Map getTabletsNumInScheduleForEachCG() { // totally 6 replicas, after adding 2 backends and overall balance, // every backend should have 1 replica, except one Assert.assertEquals(Lists.newArrayList(1, 1, 1, 1, 2), - result.values().stream().sorted().collect(Collectors.toList())); + result.values().stream().sorted().collect(Collectors.toList())); } } @@ -555,7 +557,7 @@ private void setGroup2Schema(GroupId groupId, ColocateTableIndex colocateTableIn List distributionCols = Lists.newArrayList(); distributionCols.add(new Column("k1", Type.INT)); ColocateGroupSchema groupSchema = - new ColocateGroupSchema(groupId, distributionCols, bucketNum, replicationNum); + new ColocateGroupSchema(groupId, distributionCols, bucketNum, replicationNum); Map group2Schema = Maps.newHashMap(); group2Schema.put(groupId, groupSchema); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); @@ -611,7 +613,7 @@ public void testBalanceWithOnlyOneAvailableBackend(@Mocked SystemInfoService inf GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getIdToBackend(); GroupId groupId = new GroupId(10000, 10001); ColocateTableIndex colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(4L, 2L, 3L, 4L, 2L, 3L, 4L, 2L, 3L), 3); + Lists.newArrayList(4L, 2L, 3L, 4L, 2L, 3L, 4L, 2L, 3L), 3); setGroup2Schema(groupId, colocateTableIndex, 3, (short) 3); Set unavailableBeIds = Sets.newHashSet(3L, 4L); @@ -619,8 +621,8 @@ public void testBalanceWithOnlyOneAvailableBackend(@Mocked SystemInfoService inf List availBackendIds = Lists.newArrayList(2L); boolean changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); // in this case, there is only on available backend, no need to make balancing decision. Assert.assertFalse(changed); } @@ -677,7 +679,7 @@ public void testBalanceWithSingleReplica(@Mocked SystemInfoService infoService, GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getIdToBackend(); GroupId groupId = new GroupId(10000, 10001); ColocateTableIndex colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(4L, 2L, 3L, 4L, 2L, 3L, 4L, 2L, 3L), 1); + Lists.newArrayList(4L, 2L, 3L, 4L, 2L, 3L, 4L, 2L, 3L), 1); setGroup2Schema(groupId, colocateTableIndex, 9, (short) 1); Set unavailableBeIds = Sets.newHashSet(4L); @@ -686,22 +688,22 @@ public void testBalanceWithSingleReplica(@Mocked SystemInfoService infoService, boolean changed = false; changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); // there is unavailable backend, but the replication number is 1 and replicas on available backends are // already balanced, so the bucket sequence will remain unchanged. Assert.assertFalse(changed); colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(2L, 2L, 4L, 2L, 2L, 4L, 3L, 2L, 4L), 1); + Lists.newArrayList(2L, 2L, 4L, 2L, 2L, 4L, 3L, 2L, 4L), 1); setGroup2Schema(groupId, colocateTableIndex, 9, (short) 1); changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); Assert.assertTrue(changed); System.out.println(balancedBackendsPerBucketSeq); List> expected = Lists.partition( - Lists.newArrayList(3L, 3L, 4L, 2L, 2L, 4L, 3L, 2L, 4L), 1); + Lists.newArrayList(3L, 3L, 4L, 2L, 2L, 4L, 3L, 2L, 4L), 1); // there is unavailable backend, but the replication number is 1 and replicas on available backends are // not balanced, check the balancer actually working. Assert.assertEquals(expected, balancedBackendsPerBucketSeq); @@ -715,15 +717,15 @@ public void testBalanceWithSingleReplica(@Mocked SystemInfoService infoService, }; balancedBackendsPerBucketSeq = Lists.newArrayList(); colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(2L, 2L, 4L, 2L, 2L, 4L, 3L, 2L, 4L), 1); + Lists.newArrayList(2L, 2L, 4L, 2L, 2L, 4L, 3L, 2L, 4L), 1); setGroup2Schema(groupId, colocateTableIndex, 9, (short) 1); changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, availBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); Assert.assertTrue(changed); System.out.println(balancedBackendsPerBucketSeq); List> expected3 = Lists.partition( - Lists.newArrayList(2L, 2L, 3L, 2L, 2L, 3L, 3L, 2L, 3L), 1); + Lists.newArrayList(2L, 2L, 3L, 2L, 2L, 3L, 3L, 2L, 3L), 1); // there is unavailable backend, but the replication number is 1 and there is decommissioned backend, // so we need to do relocation first. Assert.assertEquals(expected3, balancedBackendsPerBucketSeq); @@ -777,42 +779,42 @@ public void testFixBalanceEndlessLoop(@Mocked SystemInfoService infoService, // 1. only one available backend // [[7], [7], [7], [7], [7]] ColocateTableIndex colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(7L, 7L, 7L, 7L, 7L), 3); + Lists.newArrayList(7L, 7L, 7L, 7L, 7L), 3); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); List> balancedBackendsPerBucketSeq = Lists.newArrayList(); List allAvailBackendIds = Lists.newArrayList(7L); boolean changed = - Deencapsulation.invoke(balancer, "doRelocateAndBalance", - groupId, new HashSet(), allAvailBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + Deencapsulation.invoke(balancer, "doRelocateAndBalance", + groupId, new HashSet(), allAvailBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); Assert.assertFalse(changed); // 2. all backends are checked but this round is not changed // [[7], [7], [7], [7], [7]] // and add new backends 8, 9 that are on the same host with 7 colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(7L, 7L, 7L, 7L, 7L), 3); + Lists.newArrayList(7L, 7L, 7L, 7L, 7L), 3); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); balancedBackendsPerBucketSeq = Lists.newArrayList(); allAvailBackendIds = Lists.newArrayList(7L, 8L, 9L); changed = - Deencapsulation.invoke(balancer, "doRelocateAndBalance", - groupId, new HashSet(), allAvailBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + Deencapsulation.invoke(balancer, "doRelocateAndBalance", + groupId, new HashSet(), allAvailBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); Assert.assertFalse(changed); // 3. all backends are not available colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(7L, 7L, 7L, 7L, 7L), 3); + Lists.newArrayList(7L, 7L, 7L, 7L, 7L), 3); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); balancedBackendsPerBucketSeq = Lists.newArrayList(); allAvailBackendIds = Lists.newArrayList(); Set unAvailableBackendIds = Sets.newHashSet(7L); changed = Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unAvailableBackendIds, allAvailBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unAvailableBackendIds, allAvailBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); Assert.assertFalse(changed); } @@ -836,15 +838,15 @@ BackendLoadStatistic delegate(Long beId) { Set unavailBackendIds = Sets.newHashSet(9L); List flatBackendsPerBucketSeq = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); List> backends = Deencapsulation.invoke(balancer, "getSortedBackendReplicaNumPairs", - allAvailBackendIds, unavailBackendIds, statistic, flatBackendsPerBucketSeq); + allAvailBackendIds, unavailBackendIds, statistic, flatBackendsPerBucketSeq); long[] backendIds = backends.stream().mapToLong(Map.Entry::getKey).toArray(); Assert.assertArrayEquals(new long[] {7L, 8L, 6L, 2L, 3L, 5L, 4L, 1L}, backendIds); // 0,1 bucket on same be and 5, 6 on same be flatBackendsPerBucketSeq = Lists.newArrayList(1L, 1L, 3L, 4L, 5L, 6L, 7L, 7L, 9L); backends = Deencapsulation - .invoke(balancer, "getSortedBackendReplicaNumPairs", allAvailBackendIds, unavailBackendIds, - statistic, flatBackendsPerBucketSeq); + .invoke(balancer, "getSortedBackendReplicaNumPairs", allAvailBackendIds, unavailBackendIds, + statistic, flatBackendsPerBucketSeq); backendIds = backends.stream().mapToLong(Map.Entry::getKey).toArray(); Assert.assertArrayEquals(new long[] {7L, 1L, 6L, 3L, 5L, 4L, 8L, 2L}, backendIds); } @@ -865,7 +867,7 @@ public double getMixLoadScore() { public void testGetBeSeqIndexes() { List flatBackendsPerBucketSeq = Lists.newArrayList(1L, 2L, 2L, 3L, 4L, 2L); List indexes = Deencapsulation.invoke(balancer, - "getBeSeqIndexes", flatBackendsPerBucketSeq, 2L); + "getBeSeqIndexes", flatBackendsPerBucketSeq, 2L); Assert.assertArrayEquals(new int[] {1, 2, 5}, indexes.stream().mapToInt(i -> i).toArray()); System.out.println("backend1 id is " + backend1.getId()); } @@ -901,9 +903,9 @@ public void testGetUnavailableBeIdsInGroup() { GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getIdToBackend(); Set unavailableBeIds = Deencapsulation - .invoke(balancer, "getUnavailableBeIdsInGroup", infoService, colocateTableIndex, groupId); + .invoke(balancer, "getUnavailableBeIdsInGroup", infoService, colocateTableIndex, groupId); Assert.assertArrayEquals(new long[] {1L, 3L, 5L}, - unavailableBeIds.stream().mapToLong(i -> i).sorted().toArray()); + unavailableBeIds.stream().mapToLong(i -> i).sorted().toArray()); } @Test @@ -1032,18 +1034,18 @@ BackendLoadStatistic delegate(Long beId) { // group is balanced before backend 9 is dropped ColocateTableIndex colocateTableIndex = createColocateIndex(groupId, - Lists.newArrayList(9L, 8L, 7L, 8L, 6L, 5L, 9L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); + Lists.newArrayList(9L, 8L, 7L, 8L, 6L, 5L, 9L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); Deencapsulation.setField(colocateTableIndex, "group2Schema", group2Schema); List> balancedBackendsPerBucketSeq = Lists.newArrayList(); Set unavailableBeIds = Sets.newHashSet(9L); List allAvailBackendIds = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L); boolean changed = (Boolean) Deencapsulation - .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, allAvailBackendIds, - colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); + .invoke(balancer, "doRelocateAndBalance", groupId, unavailableBeIds, allAvailBackendIds, + colocateTableIndex, infoService, statistic, balancedBackendsPerBucketSeq); System.out.println(balancedBackendsPerBucketSeq); Assert.assertTrue(changed); List> expected = Lists.partition( - Lists.newArrayList(5L, 8L, 7L, 8L, 6L, 5L, 6L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); + Lists.newArrayList(5L, 8L, 7L, 8L, 6L, 5L, 6L, 4L, 1L, 2L, 3L, 4L, 1L, 2L, 3L), 3); Assert.assertEquals(expected, balancedBackendsPerBucketSeq); } @@ -1063,10 +1065,10 @@ public void testSystemStable() throws Exception { // set stable last time to 1s, and sleep 1s, the system becomes to stable Config.tablet_sched_colocate_balance_wait_system_stable_time_s = 1; System.out.println("before sleep, time: " + System.currentTimeMillis() - + "alive backend is: " + infoService.getBackendIds(true)); + + "alive backend is: " + infoService.getBackendIds(true)); Thread.sleep(2000L); System.out.println("after sleep, time: " + System.currentTimeMillis() - + "alive backend is: " + infoService.getBackendIds(true)); + + "alive backend is: " + infoService.getBackendIds(true)); Assert.assertTrue(balancer.isSystemStable(infoService)); Assert.assertTrue(balancer.isSystemStable(infoService)); @@ -1075,10 +1077,10 @@ public void testSystemStable() throws Exception { Assert.assertFalse(balancer.isSystemStable(infoService)); Assert.assertFalse(balancer.isSystemStable(infoService)); System.out.println("before sleep, time: " + System.currentTimeMillis() - + "alive backend is: " + infoService.getBackendIds(true)); + + "alive backend is: " + infoService.getBackendIds(true)); Thread.sleep(2000L); System.out.println("after sleep, time: " + System.currentTimeMillis() - + "alive backend is: " + infoService.getBackendIds(true)); + + "alive backend is: " + infoService.getBackendIds(true)); Assert.assertTrue(balancer.isSystemStable(infoService)); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/clone/DynamicPartitionSchedulerTest.java b/fe/fe-core/src/test/java/com/starrocks/clone/DynamicPartitionSchedulerTest.java index 8ff094ff6ae2a..e5e63ae28a760 100644 --- a/fe/fe-core/src/test/java/com/starrocks/clone/DynamicPartitionSchedulerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/clone/DynamicPartitionSchedulerTest.java @@ -58,28 +58,28 @@ public static void beforeClass() throws Exception { @Test public void testPartitionTTLProperties() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.tbl1\n" + - "(\n" + - " k1 date,\n" + - " v1 int \n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01'),\n" + - " PARTITION p3 values less than('2020-04-01'),\n" + - " PARTITION p4 values less than('2020-05-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + - "PROPERTIES" + - "(" + - " 'replication_num' = '1'\n" + - ");"); + .withTable("CREATE TABLE test.tbl1\n" + + "(\n" + + " k1 date,\n" + + " v1 int \n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01'),\n" + + " PARTITION p3 values less than('2020-04-01'),\n" + + " PARTITION p4 values less than('2020-05-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH (k1) BUCKETS 3\n" + + "PROPERTIES" + + "(" + + " 'replication_num' = '1'\n" + + ");"); DynamicPartitionScheduler dynamicPartitionScheduler = GlobalStateMgr.getCurrentState() - .getDynamicPartitionScheduler(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("tbl1"); + .getDynamicPartitionScheduler(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); // Now the table does not actually support partition ttl, // so in order to simplify the test, it is directly set like this tbl.getTableProperty().getProperties().put("partition_ttl_number", "3"); @@ -88,37 +88,36 @@ public void testPartitionTTLProperties() throws Exception { dynamicPartitionScheduler.registerTtlPartitionTable(db.getId(), tbl.getId()); dynamicPartitionScheduler.runAfterCatalogReady(); - Assert.assertEquals(3, tbl.getPartitions().size()); } @Test public void testPartitionTTLPropertiesZero() throws Exception { starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.base\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " v1 int sum\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values less than('2020-02-01'),\n" + - " PARTITION p2 values less than('2020-03-01')\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES('replication_num' = '1');"); + .withTable("CREATE TABLE test.base\n" + + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " v1 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); String sql = "create materialized view mv1 " + - "partition by k1 " + - "distributed by hash(k2) " + - "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"partition_ttl_number\" = \"0\"\n" + - ") " + - "as select k1, k2 from test.base;"; + "partition by k1 " + + "distributed by hash(k2) " + + "refresh async START('2122-12-31') EVERY(INTERVAL 1 HOUR) " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"partition_ttl_number\" = \"0\"\n" + + ") " + + "as select k1, k2 from test.base;"; CreateMaterializedViewStatement createMaterializedViewStatement = - (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); try { GlobalStateMgr.getCurrentState().getLocalMetastore().createMaterializedView(createMaterializedViewStatement); Assert.fail(); @@ -132,33 +131,34 @@ public void testAutoPartitionPartitionLiveNumber() throws Exception { new MockUp() { @Mock public LocalDateTime now() { - return LocalDateTime.of(2023, 3, 30, 1, 1, 1); + return LocalDateTime.of(2023, 3, 30, 1, 1, 1); } }; starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE site_access(\n" + - " event_day datetime,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day)(\n" + - " START (\"2023-03-27\") END (\"2023-03-31\") EVERY (INTERVAL 1 day),\n" + - " START (\"9999-12-30\") END (\"9999-12-31\") EVERY (INTERVAL 1 day)\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\n" + - " \"partition_live_number\" = \"3\",\n" + - " \"replication_num\" = \"1\"\n" + - ");"); + .withTable("CREATE TABLE site_access(\n" + + " event_day datetime,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day)(\n" + + " START (\"2023-03-27\") END (\"2023-03-31\") EVERY (INTERVAL 1 day),\n" + + " START (\"9999-12-30\") END (\"9999-12-31\") EVERY (INTERVAL 1 day)\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\n" + + " \"partition_live_number\" = \"3\",\n" + + " \"replication_num\" = \"1\"\n" + + ");"); DynamicPartitionScheduler dynamicPartitionScheduler = GlobalStateMgr.getCurrentState() - .getDynamicPartitionScheduler(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("site_access"); + .getDynamicPartitionScheduler(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access"); dynamicPartitionScheduler.registerTtlPartitionTable(db.getId(), tbl.getId()); dynamicPartitionScheduler.runOnceForTest(); @@ -176,33 +176,34 @@ public void testAutoRandomPartitionFPartitionLiveNumber() throws Exception { new MockUp() { @Mock public LocalDateTime now() { - return LocalDateTime.of(2023, 3, 30, 1, 1, 1); + return LocalDateTime.of(2023, 3, 30, 1, 1, 1); } }; starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE site_access(\n" + - " event_day datetime,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day)(\n" + - " START (\"2023-03-27\") END (\"2023-03-31\") EVERY (INTERVAL 1 day),\n" + - " START (\"9999-12-30\") END (\"9999-12-31\") EVERY (INTERVAL 1 day)\n" + - ")\n" + - "DISTRIBUTED BY RANDOM BUCKETS 32\n" + - "PROPERTIES(\n" + - " \"partition_live_number\" = \"3\",\n" + - " \"replication_num\" = \"1\"\n" + - ");"); + .withTable("CREATE TABLE site_access(\n" + + " event_day datetime,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day)(\n" + + " START (\"2023-03-27\") END (\"2023-03-31\") EVERY (INTERVAL 1 day),\n" + + " START (\"9999-12-30\") END (\"9999-12-31\") EVERY (INTERVAL 1 day)\n" + + ")\n" + + "DISTRIBUTED BY RANDOM BUCKETS 32\n" + + "PROPERTIES(\n" + + " \"partition_live_number\" = \"3\",\n" + + " \"replication_num\" = \"1\"\n" + + ");"); DynamicPartitionScheduler dynamicPartitionScheduler = GlobalStateMgr.getCurrentState() - .getDynamicPartitionScheduler(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("site_access"); + .getDynamicPartitionScheduler(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access"); dynamicPartitionScheduler.registerTtlPartitionTable(db.getId(), tbl.getId()); dynamicPartitionScheduler.runOnceForTest(); @@ -220,34 +221,35 @@ public void testRandomDynamicPartitionShouldMatchConfig() throws Exception { new MockUp() { @Mock public LocalDateTime now() { - return LocalDateTime.of(2023, 3, 30, 1, 1, 1); + return LocalDateTime.of(2023, 3, 30, 1, 1, 1); } }; starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test_random_bucket (\n" + - " uid String,\n" + - " tdbank_imp_date Date\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`uid`) \n" + - "PARTITION BY RANGE(`tdbank_imp_date`) ()\n" + - "DISTRIBUTED BY RANDOM BUCKETS 1\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\", \n" + - " \"dynamic_partition.enable\" = \"true\", \n" + - " \"dynamic_partition.time_unit\" = \"DAY\", \n" + - " \"dynamic_partition.time_zone\" = \"Asia/Shanghai\", \n" + - " \"dynamic_partition.start\" = \"-180\", \n" + - " \"dynamic_partition.end\" = \"3\", \n" + - " \"dynamic_partition.prefix\" = \"p\", \n" + - " \"dynamic_partition.buckets\" = \"4\", \n" + - " \"dynamic_partition.history_partition_num\" = \"0\",\n" + - " \"compression\" = \"LZ4\" );"); + .withTable("CREATE TABLE test_random_bucket (\n" + + " uid String,\n" + + " tdbank_imp_date Date\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`uid`) \n" + + "PARTITION BY RANGE(`tdbank_imp_date`) ()\n" + + "DISTRIBUTED BY RANDOM BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\", \n" + + " \"dynamic_partition.enable\" = \"true\", \n" + + " \"dynamic_partition.time_unit\" = \"DAY\", \n" + + " \"dynamic_partition.time_zone\" = \"Asia/Shanghai\", \n" + + " \"dynamic_partition.start\" = \"-180\", \n" + + " \"dynamic_partition.end\" = \"3\", \n" + + " \"dynamic_partition.prefix\" = \"p\", \n" + + " \"dynamic_partition.buckets\" = \"4\", \n" + + " \"dynamic_partition.history_partition_num\" = \"0\",\n" + + " \"compression\" = \"LZ4\" );"); DynamicPartitionScheduler dynamicPartitionScheduler = GlobalStateMgr.getCurrentState() - .getDynamicPartitionScheduler(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("test_random_bucket"); + .getDynamicPartitionScheduler(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_random_bucket"); dynamicPartitionScheduler.registerTtlPartitionTable(db.getId(), tbl.getId()); dynamicPartitionScheduler.runOnceForTest(); @@ -263,42 +265,43 @@ public void testPartitionColumnDateUseDynamicHour() throws Exception { new MockUp() { @Mock public LocalDateTime now() { - return LocalDateTime.of(2023, 3, 30, 1, 1, 1); + return LocalDateTime.of(2023, 3, 30, 1, 1, 1); } }; starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE `test_hour_partition2` (\n" + - " `event_day` date NULL COMMENT \"\",\n" + - " `site_id` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" + - " `city_code` varchar(100) NULL COMMENT \"\",\n" + - " `user_name` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" + - " `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`event_day`, `site_id`, `city_code`, `user_name`)\n" + - "PARTITION BY RANGE(`event_day`)\n" + - "()\n" + - "DISTRIBUTED BY HASH(`event_day`, `site_id`) BUCKETS 32 \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"dynamic_partition.enable\" = \"true\",\n" + - "\"dynamic_partition.time_unit\" = \"DAY\",\n" + - "\"dynamic_partition.time_zone\" = \"Asia/Shanghai\",\n" + - "\"dynamic_partition.start\" = \"-1\",\n" + - "\"dynamic_partition.end\" = \"10\",\n" + - "\"dynamic_partition.prefix\" = \"p\",\n" + - "\"dynamic_partition.buckets\" = \"3\",\n" + - "\"dynamic_partition.history_partition_num\" = \"0\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"storage_format\" = \"DEFAULT\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"compression\" = \"LZ4\"\n" + - ");"); + .withTable("CREATE TABLE `test_hour_partition2` (\n" + + " `event_day` date NULL COMMENT \"\",\n" + + " `site_id` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" + + " `city_code` varchar(100) NULL COMMENT \"\",\n" + + " `user_name` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" + + " `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`event_day`, `site_id`, `city_code`, `user_name`)\n" + + "PARTITION BY RANGE(`event_day`)\n" + + "()\n" + + "DISTRIBUTED BY HASH(`event_day`, `site_id`) BUCKETS 32 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.time_unit\" = \"DAY\",\n" + + "\"dynamic_partition.time_zone\" = \"Asia/Shanghai\",\n" + + "\"dynamic_partition.start\" = \"-1\",\n" + + "\"dynamic_partition.end\" = \"10\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"3\",\n" + + "\"dynamic_partition.history_partition_num\" = \"0\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"DEFAULT\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"compression\" = \"LZ4\"\n" + + ");"); DynamicPartitionScheduler dynamicPartitionScheduler = GlobalStateMgr.getCurrentState() - .getDynamicPartitionScheduler(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("test_hour_partition2"); + .getDynamicPartitionScheduler(); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_hour_partition2"); DynamicPartitionProperty dynamicPartitionProperty = tbl.getTableProperty().getDynamicPartitionProperty(); dynamicPartitionProperty.setTimeUnit("HOUR"); boolean result = dynamicPartitionScheduler.executeDynamicPartitionForTable(db.getId(), tbl.getId()); diff --git a/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedCtxTest.java b/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedCtxTest.java index f61390bbfbdaf..1b1e35c8e16b1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedCtxTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedCtxTest.java @@ -128,7 +128,7 @@ public void setUp() { olapTable.setIndexMeta(INDEX_ID, TB_NAME, TB_BASE_SCHEMA, 0, SCHEMA_HASH, (short) 1, TStorageType.COLUMN, KeysType.AGG_KEYS); olapTable.addPartition(partition); - Database db = new Database(); + Database db = new Database(DB_ID, ""); db.registerTableUnlocked(olapTable); GlobalStateMgr.getCurrentState().getLocalMetastore().getIdToDb().put(DB_ID, db); diff --git a/fe/fe-core/src/test/java/com/starrocks/cluster/SystemInfoServiceTest.java b/fe/fe-core/src/test/java/com/starrocks/cluster/SystemInfoServiceTest.java index 6469c7d54589f..22168d1ad5ae0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/cluster/SystemInfoServiceTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/cluster/SystemInfoServiceTest.java @@ -112,7 +112,7 @@ public void setUp() throws IOException { minTimes = 0; result = editLog; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; @@ -140,6 +140,14 @@ public void setUp() throws IOException { } }; + new Expectations(localMetastore) { + { + localMetastore.getDb(anyLong); + minTimes = 0; + result = db; + } + }; + new Expectations(nodeMgr) { { systemInfoService = new SystemInfoService(); diff --git a/fe/fe-core/src/test/java/com/starrocks/common/proc/DbsProcDirTest.java b/fe/fe-core/src/test/java/com/starrocks/common/proc/DbsProcDirTest.java index 81abd72804060..925b535b3ca7d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/common/proc/DbsProcDirTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/common/proc/DbsProcDirTest.java @@ -83,27 +83,27 @@ public void testRegister() { public void testLookupNormal() throws AnalysisException { new Expectations(globalStateMgr) { { - globalStateMgr.getDb("db1"); + globalStateMgr.getLocalMetastore().getDb("db1"); minTimes = 0; result = db1; - globalStateMgr.getDb("db2"); + globalStateMgr.getLocalMetastore().getDb("db2"); minTimes = 0; result = db2; - globalStateMgr.getDb("db3"); + globalStateMgr.getLocalMetastore().getDb("db3"); minTimes = 0; result = null; - globalStateMgr.getDb(db1.getId()); + globalStateMgr.getLocalMetastore().getDb(db1.getId()); minTimes = 0; result = db1; - globalStateMgr.getDb(db2.getId()); + globalStateMgr.getLocalMetastore().getDb(db2.getId()); minTimes = 0; result = db2; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = null; } @@ -164,27 +164,27 @@ public void testFetchResultNormal() throws AnalysisException { minTimes = 0; result = Lists.newArrayList("db1", "db2"); - globalStateMgr.getDb("db1"); + globalStateMgr.getLocalMetastore().getDb("db1"); minTimes = 0; result = db1; - globalStateMgr.getDb("db2"); + globalStateMgr.getLocalMetastore().getDb("db2"); minTimes = 0; result = db2; - globalStateMgr.getDb("db3"); + globalStateMgr.getLocalMetastore().getDb("db3"); minTimes = 0; result = null; - globalStateMgr.getDb(db1.getId()); + globalStateMgr.getLocalMetastore().getDb(db1.getId()); minTimes = 0; result = db1; - globalStateMgr.getDb(db2.getId()); + globalStateMgr.getLocalMetastore().getDb(db2.getId()); minTimes = 0; result = db2; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = null; } diff --git a/fe/fe-core/src/test/java/com/starrocks/common/proc/TablesProcDirTest.java b/fe/fe-core/src/test/java/com/starrocks/common/proc/TablesProcDirTest.java index f82d43b70e90d..6631919250343 100644 --- a/fe/fe-core/src/test/java/com/starrocks/common/proc/TablesProcDirTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/common/proc/TablesProcDirTest.java @@ -26,11 +26,15 @@ import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.PartitionType; import com.starrocks.catalog.RangePartitionInfo; +import com.starrocks.catalog.Table; import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; import com.starrocks.common.DdlException; import com.starrocks.common.FeConstants; +import com.starrocks.server.LocalMetastore; import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -105,6 +109,28 @@ public void setUp() throws DdlException, AnalysisException { db.registerTableUnlocked(tb1); db.registerTableUnlocked(tb2); db.registerTableUnlocked(tb3); + + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + + @Mock + public List
getTables(Long dbId) { + return db.getTables(); + } + }; } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/common/util/SmallFileMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/common/util/SmallFileMgrTest.java index 7acc45626cdd0..dec03e731d355 100644 --- a/fe/fe-core/src/test/java/com/starrocks/common/util/SmallFileMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/common/util/SmallFileMgrTest.java @@ -81,7 +81,7 @@ public void test(@Injectable CreateFileStmt stmt1, @Injectable CreateFileStmt st db.getId(); minTimes = 0; result = 1L; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = db; stmt1.getDbName(); diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/elasticsearch/EsShardPartitionsTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/elasticsearch/EsShardPartitionsTest.java index 3f746ba8245d6..f4575562c9b64 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/elasticsearch/EsShardPartitionsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/elasticsearch/EsShardPartitionsTest.java @@ -47,8 +47,8 @@ public class EsShardPartitionsTest extends EsTestCase { @Test public void testPartition() throws Exception { EsTable esTable = (EsTable) GlobalStateMgr.getCurrentState() - .getDb(GlobalStateMgrTestUtil.testDb1) - .getTable(GlobalStateMgrTestUtil.testEsTableId1); + .getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDb1, GlobalStateMgrTestUtil.testEsTable1); EsShardPartitions esShardPartitions = EsShardPartitions.findShardPartitions("doe", loadJsonFromFile("data/es/test_search_shards.json")); EsTablePartitions esTablePartitions = EsTablePartitions.fromShardPartitions(esTable, esShardPartitions); diff --git a/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java b/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java index b16da048e53ce..b7de28181515c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/consistency/ConsistencyCheckerTest.java @@ -77,9 +77,13 @@ public void testChooseTablets(@Mocked GlobalStateMgr globalStateMgr) { result = Lists.newArrayList(dbId); minTimes = 0; - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = database; minTimes = 0; + + globalStateMgr.getLocalMetastore().getTables(dbId); + result = database.getTables(); + minTimes = 0; } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java b/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java index f830c7306897d..fa07ad4e1430b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/consistency/MetaRecoveryDaemonTest.java @@ -53,16 +53,16 @@ public static void tearDown() throws Exception { public void testRecover() throws Exception { PseudoCluster cluster = PseudoCluster.getInstance(); String sql = "CREATE TABLE test.`tbl_recover` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 8\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 8\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); cluster.runSql("test", "insert into test.tbl_recover values (1, 'a'), (2, 'b')"); @@ -70,7 +70,8 @@ public void testRecover() throws Exception { Thread.sleep(2000L); Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); - OlapTable table = (OlapTable) database.getTable("tbl_recover"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(database.getFullName(), "tbl_recover"); Partition partition = table.getPartition("tbl_recover"); MaterializedIndex index = partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).get(0); for (Tablet tablet : index.getTablets()) { @@ -88,19 +89,19 @@ public void testRecover() throws Exception { for (Backend backend : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackends()) { backend.getBackendStatus().lastSuccessReportTabletsTime = TimeUtils - .longToTimeString(System.currentTimeMillis()); + .longToTimeString(System.currentTimeMillis()); } // add a committed txn TransactionState transactionState = new TransactionState(database.getId(), Lists.newArrayList(table.getId()), - 11111, "xxxx", null, TransactionState.LoadJobSourceType.FRONTEND, null, 2222, 100000); + 11111, "xxxx", null, TransactionState.LoadJobSourceType.FRONTEND, null, 2222, 100000); TableCommitInfo tableCommitInfo = new TableCommitInfo(table.getId()); PartitionCommitInfo partitionCommitInfo = new PartitionCommitInfo(partition.getId(), 4, -1L); tableCommitInfo.addPartitionCommitInfo(partitionCommitInfo); transactionState.putIdToTableCommitInfo(table.getId(), tableCommitInfo); transactionState.setTransactionStatus(TransactionStatus.COMMITTED); GlobalStateMgr.getCurrentState().getGlobalTransactionMgr() - .getDatabaseTransactionMgr(database.getId()).replayUpsertTransactionState(transactionState); + .getDatabaseTransactionMgr(database.getId()).replayUpsertTransactionState(transactionState); // recover will fail, because there is a committed txn on that partition MetaRecoveryDaemon recovery = new MetaRecoveryDaemon(); @@ -121,13 +122,13 @@ public void testRecover() throws Exception { // change replica version LocalTablet localTablet = (LocalTablet) partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL) - .get(0).getTablets().get(0); + .get(0).getTablets().get(0); long version = 3; for (Replica replica : localTablet.getAllReplicas()) { replica.updateForRestore(++version, 10, 10); } LocalTablet localTablet2 = (LocalTablet) partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL) - .get(0).getTablets().get(0); + .get(0).getTablets().get(0); for (Replica replica : localTablet2.getAllReplicas()) { replica.updateForRestore(4, 10, 10); } @@ -172,7 +173,7 @@ public void testCheckTabletReportCacheUp() { MetaRecoveryDaemon metaRecoveryDaemon = new MetaRecoveryDaemon(); for (Backend backend : GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackends()) { backend.getBackendStatus().lastSuccessReportTabletsTime = TimeUtils - .longToTimeString(timeMs); + .longToTimeString(timeMs); } Assert.assertFalse(metaRecoveryDaemon.checkTabletReportCacheUp(timeMs + 1000L)); Assert.assertTrue(metaRecoveryDaemon.checkTabletReportCacheUp(timeMs - 1000L)); diff --git a/fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java b/fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java index 6a68a5b792f33..593d35a1c8a6a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java @@ -127,7 +127,7 @@ public void syncTableMeta() throws Exception { LeaderImpl leader = new LeaderImpl(); TGetTableMetaResponse response = leader.getTableMeta(request); - Table table = GlobalStateMgr.getCurrentState().getDb("test_db").getTable("test_ext_table"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test_db").getTable("test_ext_table"); ExternalOlapTable extTable = (ExternalOlapTable) table; // remove the thread local meta context MetaContext.remove(); diff --git a/fe/fe-core/src/test/java/com/starrocks/http/ShowDataActionTest.java b/fe/fe-core/src/test/java/com/starrocks/http/ShowDataActionTest.java index 78fb347734f9f..3e01b1ce332c6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/http/ShowDataActionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/http/ShowDataActionTest.java @@ -45,10 +45,14 @@ public void doSetUp() { ConcurrentHashMap fullNameToDb = GlobalStateMgr.getCurrentState() .getLocalMetastore().getFullNameToDb(); fullNameToDb.put(SHOW_DATA_DB_NAME, db); + + ConcurrentHashMap idToDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getIdToDb(); + idToDb.put(1000 + testDbId, db); } @Test public void testGetShowData() throws IOException { + Config.http_slow_request_threshold_ms = 0; Request request = new Request.Builder() .get() diff --git a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java index cdff97ea22e2b..1bcc2c1ffe10e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java +++ b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java @@ -284,23 +284,15 @@ private static GlobalStateMgr newDelegateCatalog() { new Expectations(globalStateMgr) { { - globalStateMgr.getDb(db.getId()); - minTimes = 0; - result = db; - - globalStateMgr.getDb(DB_NAME); - minTimes = 0; - result = db; - globalStateMgr.isLeader(); minTimes = 0; result = true; - globalStateMgr.getDb("emptyDb"); + globalStateMgr.getLocalMetastore().getDb("emptyDb"); minTimes = 0; result = null; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = new Database(); @@ -320,13 +312,13 @@ private static GlobalStateMgr newDelegateCatalog() { new Expectations(localMetastore) { { - localMetastore.listDbNames(); + localMetastore.getDb("testDb"); minTimes = 0; - result = Lists.newArrayList("testDb"); + result = db; - localMetastore.getFullNameToDb(); + localMetastore.getDb(testDbId); minTimes = 0; - result = nameToDb; + result = db; } }; @@ -352,26 +344,10 @@ private static GlobalStateMgr newDelegateGlobalStateMgr() { new Expectations(globalStateMgr) { { - globalStateMgr.getDb(db.getId()); - minTimes = 0; - result = db; - - globalStateMgr.getDb(DB_NAME); - minTimes = 0; - result = db; - globalStateMgr.isLeader(); minTimes = 0; result = true; - globalStateMgr.getDb("emptyDb"); - minTimes = 0; - result = null; - - globalStateMgr.getDb(anyString); - minTimes = 0; - result = new Database(); - globalStateMgr.getLoadInstance(); minTimes = 0; result = new Load(); @@ -383,6 +359,10 @@ private static GlobalStateMgr newDelegateGlobalStateMgr() { globalStateMgr.getMetadataMgr(); minTimes = 0; result = metadataMgr; + + globalStateMgr.getLocalMetastore(); + minTimes = 0; + result = localMetastore; } }; @@ -401,7 +381,18 @@ private static GlobalStateMgr newDelegateGlobalStateMgr() { result = newEmptyTable; } }; - ; + + new Expectations(localMetastore) { + { + localMetastore.getDb("testDb"); + minTimes = 0; + result = db; + + localMetastore.getDb(testDbId); + minTimes = 0; + result = db; + } + }; return globalStateMgr; } diff --git a/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java b/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java index 72473198ee4d0..bace9e50e5ff4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TablePartitionActionTest.java @@ -106,7 +106,7 @@ public void testNonOlapTable() throws Exception { @Test public void testOlapTable() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(testDbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(testDbId); db.registerTableUnlocked(newOlapTable( TB_OLAP_TABLE_ID, TB_OLAP_TABLE_NAME, PARTITION_SIZE)); @@ -153,7 +153,7 @@ public void testOlapTable() throws Exception { @Test public void testLakeTable() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(testDbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(testDbId); db.registerTableUnlocked(newLakeTable( TB_LAKE_TABLE_ID, TB_LAKE_TABLE_NAME, PARTITION_SIZE)); @@ -200,7 +200,7 @@ public void testLakeTable() throws Exception { @Test public void testPages() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(testDbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(testDbId); db.registerTableUnlocked(newOlapTable( TB_OLAP_TABLE_ID, TB_OLAP_TABLE_NAME, PARTITION_SIZE)); diff --git a/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TableSchemaActionTest.java b/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TableSchemaActionTest.java index 3a61bc79162da..e26de38babf8f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TableSchemaActionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/http/rest/v2/TableSchemaActionTest.java @@ -75,7 +75,7 @@ public class TableSchemaActionTest extends StarRocksHttpTestCase { @Override protected void doSetUp() { - Database db = GlobalStateMgr.getCurrentState().getDb(testDbId); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(testDbId); db.registerTableUnlocked(newOlapTable(TB_GET_TABLE_SCHEMA_ID, TB_GET_TABLE_SCHEMA_NAME)); } diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/AlterTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/AlterTest.java index 2290194741781..16ee0473f61c4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/AlterTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/AlterTest.java @@ -63,35 +63,35 @@ public void testAddPartitionForLakeTable() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.test_lake_partition (\n" + - " k1 DATE,\n" + - " k2 INT,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "DUPLICATE KEY(k1, k2, k3)\n" + - "PARTITION BY RANGE (k1, k2, k3) (\n" + - " PARTITION p1 VALUES [(\"2014-01-01\", \"10\", \"200\"), (\"2014-01-01\", \"20\", \"300\")),\n" + - " PARTITION p2 VALUES [(\"2014-06-01\", \"100\", \"200\"), (\"2014-07-01\", \"100\", \"300\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES (\n" + - " \"datacache.enable\" = \"true\"\n" + - ")"; + " k1 DATE,\n" + + " k2 INT,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "DUPLICATE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE (k1, k2, k3) (\n" + + " PARTITION p1 VALUES [(\"2014-01-01\", \"10\", \"200\"), (\"2014-01-01\", \"20\", \"300\")),\n" + + " PARTITION p2 VALUES [(\"2014-06-01\", \"100\", \"200\"), (\"2014-07-01\", \"100\", \"300\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"datacache.enable\" = \"true\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE test_lake_partition ADD\n" + - " PARTITION p3 VALUES LESS THAN (\"2014-01-01\")"; + " PARTITION p3 VALUES LESS THAN (\"2014-01-01\")"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "test_lake_partition", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "test_lake_partition", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("test_lake_partition"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("test_lake_partition"); Assert.assertNotNull(table.getPartition("p1")); Assert.assertNotNull(table.getPartition("p2")); @@ -109,35 +109,35 @@ public void testMultiRangePartitionForLakeTable() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE site_access (\n" + - " datekey INT,\n" + - " site_id INT,\n" + - " city_code SMALLINT,\n" + - " user_name VARCHAR(32),\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(datekey, site_id, city_code, user_name)\n" + - "PARTITION BY RANGE (datekey) (\n" + - " START (\"1\") END (\"5\") EVERY (1)\n" + - ")\n" + - "DISTRIBUTED BY HASH(site_id) BUCKETS 3\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"1\"\n" + - ")"; + " datekey INT,\n" + + " site_id INT,\n" + + " city_code SMALLINT,\n" + + " user_name VARCHAR(32),\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(datekey, site_id, city_code, user_name)\n" + + "PARTITION BY RANGE (datekey) (\n" + + " START (\"1\") END (\"5\") EVERY (1)\n" + + ")\n" + + "DISTRIBUTED BY HASH(site_id) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String alterSQL = "ALTER TABLE site_access \n" + - " ADD PARTITIONS START (\"7\") END (\"9\") EVERY (1)"; + " ADD PARTITIONS START (\"7\") END (\"9\") EVERY (1)"; AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(alterSQL, ctx); AddPartitionClause addPartitionClause = (AddPartitionClause) alterTableStmt.getAlterClauseList().get(0); GlobalStateMgr.getCurrentState().getLocalMetastore() - .addPartitions(Util.getOrCreateConnectContext(), db, "site_access", addPartitionClause); + .addPartitions(Util.getOrCreateConnectContext(), db, "site_access", addPartitionClause); - Table table = GlobalStateMgr.getCurrentState().getDb("test") - .getTable("site_access"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test") + .getTable("site_access"); Assert.assertNotNull(table.getPartition("p1")); Assert.assertNotNull(table.getPartition("p2")); @@ -155,26 +155,27 @@ public void testMultiRangePartitionForLakeTable() throws Exception { public void testSingleRangePartitionPersistInfo() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); String createSQL = "CREATE TABLE test.new_table (\n" + - " k1 DATE,\n" + - " k2 INT,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "DUPLICATE KEY(k1, k2, k3)\n" + - "PARTITION BY RANGE (k1, k2, k3) (\n" + - " PARTITION p1 VALUES [(\"2014-01-01\", \"10\", \"200\"), (\"2014-01-01\", \"20\", \"300\")),\n" + - " PARTITION p2 VALUES [(\"2014-06-01\", \"100\", \"200\"), (\"2014-07-01\", \"100\", \"300\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES (\n" + - " \"datacache.enable\" = \"true\"\n" + - ")"; + " k1 DATE,\n" + + " k2 INT,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "DUPLICATE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE (k1, k2, k3) (\n" + + " PARTITION p1 VALUES [(\"2014-01-01\", \"10\", \"200\"), (\"2014-01-01\", \"20\", \"300\")),\n" + + " PARTITION p2 VALUES [(\"2014-06-01\", \"100\", \"200\"), (\"2014-07-01\", \"100\", \"300\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"datacache.enable\" = \"true\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("new_table"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "new_table"); RangePartitionInfo partitionInfo = (RangePartitionInfo) table.getPartitionInfo(); long dbId = db.getId(); @@ -188,7 +189,7 @@ public void testSingleRangePartitionPersistInfo() throws Exception { Range range = partitionInfo.getRange(partitionId); DataCacheInfo dataCacheInfo = partitionInfo.getDataCacheInfo(partitionId); RangePartitionPersistInfo partitionPersistInfoOut = new RangePartitionPersistInfo(dbId, tableId, partition, - dataProperty, replicationNum, isInMemory, isTempPartition, range, dataCacheInfo); + dataProperty, replicationNum, isInMemory, isTempPartition, range, dataCacheInfo); // write log File file = new File("./test_serial.log"); @@ -229,21 +230,21 @@ public void testAlterTableCompactionForLakeTable() throws Exception { DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser(dropSQL, ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); String createSQL = "CREATE TABLE test.t1 (\n" + - " k1 DATE,\n" + - " k2 INT,\n" + - " k3 SMALLINT,\n" + - " v1 VARCHAR(2048),\n" + - " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + - ")\n" + - "DUPLICATE KEY(k1, k2, k3)\n" + - "PARTITION BY RANGE (k1, k2, k3) (\n" + - " PARTITION p1 VALUES [(\"2014-01-01\", \"10\", \"200\"), (\"2014-01-01\", \"20\", \"300\")),\n" + - " PARTITION p2 VALUES [(\"2014-06-01\", \"100\", \"200\"), (\"2014-07-01\", \"100\", \"300\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "PROPERTIES (\n" + - " \"datacache.enable\" = \"true\"\n" + - ")"; + " k1 DATE,\n" + + " k2 INT,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "DUPLICATE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE (k1, k2, k3) (\n" + + " PARTITION p1 VALUES [(\"2014-01-01\", \"10\", \"200\"), (\"2014-01-01\", \"20\", \"300\")),\n" + + " PARTITION p2 VALUES [(\"2014-06-01\", \"100\", \"200\"), (\"2014-07-01\", \"100\", \"300\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"datacache.enable\" = \"true\"\n" + + ")"; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createSQL, ctx); StarRocksAssert.utCreateTableWithRetry(createTableStmt); diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/CreateLakeTableTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/CreateLakeTableTest.java index 7d78066921ef1..bbba4a7dfe580 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/CreateLakeTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/CreateLakeTableTest.java @@ -67,14 +67,14 @@ private static void createTable(String sql) throws Exception { } private void checkLakeTable(String dbName, String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); Assert.assertTrue(table.isCloudNativeTable()); } private LakeTable getLakeTable(String dbName, String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); Assert.assertTrue(table.isCloudNativeTable()); return (LakeTable) table; } @@ -120,7 +120,7 @@ public void testCreateLakeTable() throws UserException { "properties('replication_num' = '1');")); checkLakeTable("lake_test", "multi_partition_unique_key"); - Database db = GlobalStateMgr.getCurrentState().getDb("lake_test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("lake_test"); LakeTable table = getLakeTable("lake_test", "multi_partition_unique_key"); String defaultFullPath = getDefaultStorageVolumeFullPath(); String defaultTableFullPath = String.format("%s/db%d/%d", defaultFullPath, db.getId(), table.getId()); diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/LakeMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/LakeMaterializedViewTest.java index e876029060843..e7425d1143dbf 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/LakeMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/LakeMaterializedViewTest.java @@ -94,17 +94,17 @@ public static void setUp() throws Exception { starRocksAssert.withDatabase(DB).useDatabase(DB); starRocksAssert.withTable("CREATE TABLE base_table\n" + - "(\n" + - " k1 date,\n" + - " k2 int,\n" + - " k3 int\n" + - ")\n" + - "PARTITION BY RANGE(k1)\n" + - "(\n" + - " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + - " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + - ")\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3"); + "(\n" + + " k1 date,\n" + + " k2 int,\n" + + " k3 int\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" + + " PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3"); } @AfterClass @@ -157,7 +157,7 @@ int getCurrentStateJournalVersion() { // Lake mv LakeMaterializedView mv = new LakeMaterializedView(mvId, dbId, "mv1", columns, KeysType.AGG_KEYS, - partitionInfo, distributionInfo, mvRefreshScheme); + partitionInfo, distributionInfo, mvRefreshScheme); Deencapsulation.setField(mv, "baseIndexId", indexId); mv.addPartition(partition); mv.setIndexMeta(indexId, "mv1", columns, 0, 0, (short) 1, TStorageType.COLUMN, KeysType.AGG_KEYS); @@ -202,17 +202,18 @@ int getCurrentStateJournalVersion() { @Test public void testCreateMaterializedView() throws Exception { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "distributed by hash(k2) buckets 3\n" + - "PROPERTIES(\n" + - " 'datacache.enable' = 'true',\n" + - " 'enable_async_write_back' = 'false',\n" + - " 'datacache.partition_duration' = '6 day'\n" + - ")\n" + - "refresh async\n" + - "as select k2, sum(k3) as total from base_table group by k2;"); + "distributed by hash(k2) buckets 3\n" + + "PROPERTIES(\n" + + " 'datacache.enable' = 'true',\n" + + " 'enable_async_write_back' = 'false',\n" + + " 'datacache.partition_duration' = '6 day'\n" + + ")\n" + + "refresh async\n" + + "as select k2, sum(k3) as total from base_table group by k2;"); - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - MaterializedView mv = (MaterializedView) db.getTable("mv1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv1"); Assert.assertTrue(mv.isCloudNativeMaterializedView()); Assert.assertTrue(mv.isActive()); @@ -243,46 +244,48 @@ public void testCreateMaterializedView() throws Exception { Assert.assertNotNull(task); starRocksAssert.dropMaterializedView("mv1"); - Assert.assertNull(db.getTable("mv1")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv1")); } @Test public void testInactiveMaterializedView() throws Exception { starRocksAssert.withTable("create table base_table2\n" + - "(\n" + - " k4 date,\n" + - " k5 int\n" + - ")\n" + - "DISTRIBUTED BY HASH(k4) BUCKETS 3"); + "(\n" + + " k4 date,\n" + + " k5 int\n" + + ")\n" + + "DISTRIBUTED BY HASH(k4) BUCKETS 3"); starRocksAssert.withMaterializedView("create materialized view mv2\n" + - "distributed by hash(k2) buckets 3\n" + - "refresh async\n" + - "as select k1, k2, sum(k3) as total from base_table, base_table2 where k1 = k4 group by k1, k2;"); + "distributed by hash(k2) buckets 3\n" + + "refresh async\n" + + "as select k1, k2, sum(k3) as total from base_table, base_table2 where k1 = k4 group by k1, k2;"); - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - MaterializedView mv = (MaterializedView) db.getTable("mv2"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv2"); Assert.assertTrue(mv.isCloudNativeMaterializedView()); Assert.assertTrue(mv.isActive()); // drop base table and inactive mv starRocksAssert.dropTable("base_table2"); - Assert.assertNull(db.getTable("base_table2")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "base_table2")); Assert.assertFalse(mv.isActive()); starRocksAssert.dropMaterializedView("mv2"); - Assert.assertNull(db.getTable("mv2")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv2")); } @Test public void testAlterAsyncMaterializedViewInterval() throws Exception { starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW mv3\n" + - "PARTITION BY k1\n" + - "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + - "REFRESH async START('2122-12-31 20:45:11') EVERY(INTERVAL 1 DAY)\n" + - "as select k1,k2 from base_table;"); - - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - MaterializedView mv = (MaterializedView) db.getTable("mv3"); + "PARTITION BY k1\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "REFRESH async START('2122-12-31 20:45:11') EVERY(INTERVAL 1 DAY)\n" + + "as select k1,k2 from base_table;"); + + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv3"); Assert.assertTrue(mv.isCloudNativeMaterializedView()); MaterializedView.AsyncRefreshContext asyncRefreshContext = mv.getRefreshScheme().getAsyncRefreshContext(); @@ -299,25 +302,26 @@ public void testAlterAsyncMaterializedViewInterval() throws Exception { Assert.assertEquals(2, asyncRefreshContext.getStep()); starRocksAssert.dropMaterializedView("mv3"); - Assert.assertNull(db.getTable("mv3")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv3")); } @Test public void testModifyRelatedColumnWithMaterializedView() { try { starRocksAssert.withTable("create table base_table4\n" + - "(\n" + - " k4 date,\n" + - " k5 int\n" + - ")\n" + - "duplicate key(k4) distributed by hash(k4) buckets 3;"); + "(\n" + + " k4 date,\n" + + " k5 int\n" + + ")\n" + + "duplicate key(k4) distributed by hash(k4) buckets 3;"); starRocksAssert.withMaterializedView("create materialized view mv4\n" + - "distributed by hash(k1) buckets 3\n" + - "refresh async\n" + - "as select k1, k5, sum(k3) as total from base_table, base_table4 where k1 = k4 group by k1, k5;"); + "distributed by hash(k1) buckets 3\n" + + "refresh async\n" + + "as select k1, k5, sum(k3) as total from base_table, base_table4 where k1 = k4 group by k1, k5;"); - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - MaterializedView mv = (MaterializedView) db.getTable("mv4"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv4"); Assert.assertTrue(mv.isCloudNativeMaterializedView()); Assert.assertTrue(mv.isActive()); @@ -332,9 +336,9 @@ public void testModifyRelatedColumnWithMaterializedView() { Assert.assertFalse(mv.isActive()); starRocksAssert.dropMaterializedView("mv4"); - Assert.assertNull(db.getTable("mv4")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv4")); starRocksAssert.dropTable("base_table4"); - Assert.assertNull(db.getTable("base_table4")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "base_table4")); } catch (Exception e) { System.out.println(e); Assert.fail(); @@ -345,27 +349,28 @@ public void testModifyRelatedColumnWithMaterializedView() { public void testNonPartitionMvEnableFillDataCache() { try { starRocksAssert.withTable("create table base_table5\n" + - "(\n" + - " k4 date,\n" + - " k5 int\n" + - ")\n" + - "duplicate key(k4) distributed by hash(k4) buckets 3;"); + "(\n" + + " k4 date,\n" + + " k5 int\n" + + ")\n" + + "duplicate key(k4) distributed by hash(k4) buckets 3;"); starRocksAssert.withMaterializedView("create materialized view mv5\n" + - "distributed by hash(k1) buckets 3\n" + - "refresh async\n" + - "as select k1, k5, sum(k3) as total from base_table, base_table5 where k1 = k4 group by k1, k5;"); + "distributed by hash(k1) buckets 3\n" + + "refresh async\n" + + "as select k1, k5, sum(k3) as total from base_table, base_table5 where k1 = k4 group by k1, k5;"); - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - MaterializedView mv = (MaterializedView) db.getTable("mv5"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + MaterializedView mv = + (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv5"); Assert.assertTrue(mv.isCloudNativeMaterializedView()); Partition p = mv.getPartition("mv5"); Assert.assertTrue(mv.isEnableFillDataCache(p)); starRocksAssert.dropMaterializedView("mv5"); - Assert.assertNull(db.getTable("mv5")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "mv5")); starRocksAssert.dropTable("base_table5"); - Assert.assertNull(db.getTable("base_table5")); + Assert.assertNull(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "base_table5")); } catch (Exception e) { System.out.println(e); Assert.fail(); @@ -418,7 +423,7 @@ int getCurrentStateJournalVersion() { LocalDate lower1 = upper1.minus(duration); Range range1 = Range.closedOpen(PartitionKey.ofDate(lower1), PartitionKey.ofDate(upper1)); partitionInfo.addPartition(partition1Id, false, range1, DataProperty.DEFAULT_DATA_PROPERTY, (short) 1, false, - new DataCacheInfo(true, false)); + new DataCacheInfo(true, false)); // partition2 MaterializedIndex index2 = new MaterializedIndex(indexId, MaterializedIndex.IndexState.NORMAL); @@ -431,7 +436,7 @@ int getCurrentStateJournalVersion() { LocalDate lower2 = upper2.minus(duration); Range range2 = Range.closedOpen(PartitionKey.ofDate(lower2), PartitionKey.ofDate(upper2)); partitionInfo.addPartition(partition2Id, false, range2, DataProperty.DEFAULT_DATA_PROPERTY, (short) 1, false, - new DataCacheInfo(true, false)); + new DataCacheInfo(true, false)); // refresh scheme MvRefreshScheme mvRefreshScheme = new MvRefreshScheme(); @@ -439,7 +444,7 @@ int getCurrentStateJournalVersion() { // Lake mv LakeMaterializedView mv = new LakeMaterializedView(mvId, dbId, "mv1", columns, KeysType.AGG_KEYS, - partitionInfo, distributionInfo, mvRefreshScheme); + partitionInfo, distributionInfo, mvRefreshScheme); Deencapsulation.setField(mv, "baseIndexId", indexId); mv.addPartition(partition1); mv.addPartition(partition2); diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/delete/DeleteTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/delete/DeleteTest.java index dcbc2d935cde0..5247c9177d6af 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/delete/DeleteTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/delete/DeleteTest.java @@ -149,9 +149,12 @@ public void setUpExpectation() { GlobalStateMgr.getCurrentState(); result = globalStateMgr; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); result = db; + globalStateMgr.getLocalMetastore().getTable(anyString, anyString); + result = db.getTable(tableId); + GlobalStateMgr.getCurrentState().getGlobalTransactionMgr(); result = globalTransactionMgr; @@ -324,9 +327,12 @@ public void setUpExpectationWithoutExec() { GlobalStateMgr.getCurrentState(); result = globalStateMgr; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); result = db; + globalStateMgr.getLocalMetastore().getTable(anyString, anyString); + result = db.getTable(tableId); + GlobalStateMgr.getCurrentState().getGlobalTransactionMgr(); result = globalTransactionMgr; diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/qe/ShowExecutorTest.java index cd64e34088b19..4d0d334a1ce96 100644 --- a/fe/fe-core/src/test/java/com/starrocks/lake/qe/ShowExecutorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/lake/qe/ShowExecutorTest.java @@ -81,11 +81,11 @@ public RunMode getCurrentRunMode() { globalStateMgr = Deencapsulation.newInstance(GlobalStateMgr.class); new Expectations(globalStateMgr) { { - globalStateMgr.getDb("testDb"); + globalStateMgr.getLocalMetastore().getDb("testDb"); minTimes = 0; result = db; - globalStateMgr.getDb("testDb1"); + globalStateMgr.getLocalMetastore().getDb("testDb1"); minTimes = 0; result = db1; } diff --git a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java index 5b5ff4fdc1c20..5785533027497 100644 --- a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java @@ -85,22 +85,22 @@ public static void beforeClass() throws Exception { StarRocksAssert starRocksAssert = new StarRocksAssert(connectContext); starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE test.properties_change_test(k1 int, v1 int) " + - "primary key(k1) distributed by hash(k1) properties('replication_num' = '1');") - .withTable("CREATE TABLE test.binlog_report_handler_test(k1 int, v1 int) " + - "duplicate key(k1) distributed by hash(k1) buckets 50 properties('replication_num' = '1', " + - "'binlog_enable' = 'true', 'binlog_max_size' = '100');") - .withTable("CREATE TABLE test.primary_index_cache_expire_sec_test(k1 int, v1 int) " + - "primary key(k1) distributed by hash(k1) buckets 5 properties('replication_num' = '1', " + - "'primary_index_cache_expire_sec' = '3600');") - .withTable("CREATE TABLE test.update_schema(k1 int, v1 int) " + - "primary key(k1) distributed by hash(k1) buckets 5 properties('replication_num' = '1', " + - "'primary_index_cache_expire_sec' = '3600');"); + .withTable("CREATE TABLE test.properties_change_test(k1 int, v1 int) " + + "primary key(k1) distributed by hash(k1) properties('replication_num' = '1');") + .withTable("CREATE TABLE test.binlog_report_handler_test(k1 int, v1 int) " + + "duplicate key(k1) distributed by hash(k1) buckets 50 properties('replication_num' = '1', " + + "'binlog_enable' = 'true', 'binlog_max_size' = '100');") + .withTable("CREATE TABLE test.primary_index_cache_expire_sec_test(k1 int, v1 int) " + + "primary key(k1) distributed by hash(k1) buckets 5 properties('replication_num' = '1', " + + "'primary_index_cache_expire_sec' = '3600');") + .withTable("CREATE TABLE test.update_schema(k1 int, v1 int) " + + "primary key(k1) distributed by hash(k1) buckets 5 properties('replication_num' = '1', " + + "'primary_index_cache_expire_sec' = '3600');"); } @Test public void testHandleSetTabletEnablePersistentIndex() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); long dbId = db.getId(); long backendId = 10001L; List tabletIds = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletIdsByBackendId(10001); @@ -124,9 +124,10 @@ public void testHandleSetTabletEnablePersistentIndex() { @Test public void testHandleSetPrimaryIndexCacheExpireSec() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); long dbId = db.getId(); - OlapTable olapTable = (OlapTable) db.getTable("primary_index_cache_expire_sec_test"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "primary_index_cache_expire_sec_test"); long backendId = 10001L; List tabletIds = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletIdsByBackendId(10001); Assert.assertFalse(tabletIds.isEmpty()); @@ -149,21 +150,22 @@ public void testHandleSetPrimaryIndexCacheExpireSec() { @Test public void testHandleUpdateTableSchema() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); long dbId = db.getId(); - OlapTable olapTable = (OlapTable) db.getTable("update_schema"); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "update_schema"); String stmt = "alter table update_schema add column add_v int default '1'"; StarRocksAssert starRocksAssert = new StarRocksAssert(connectContext); AlterTableStmt alterTableStmt = - (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); + (AlterTableStmt) UtFrameUtils.parseStmtWithNewParser(stmt, starRocksAssert.getCtx()); SchemaChangeHandler schemaChangeHandler = GlobalStateMgr.getCurrentState().getSchemaChangeHandler(); schemaChangeHandler.process(alterTableStmt.getAlterClauseList(), db, olapTable); Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); long backendId = 10001L; List tabletIds = - GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletIdsByBackendId(10001); + GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletIdsByBackendId(10001); Assert.assertFalse(tabletIds.isEmpty()); Map backendTablets = new HashMap(); @@ -184,9 +186,10 @@ public void testHandleUpdateTableSchema() throws Exception { @Test public void testHandleSetTabletBinlogConfig() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); long dbId = db.getId(); - OlapTable olapTable = (OlapTable) db.getTable("binlog_report_handler_test"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "binlog_report_handler_test"); long backendId = 10001L; List tabletIds = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletIdsByBackendId(10001); Assert.assertFalse(tabletIds.isEmpty()); @@ -375,7 +378,7 @@ public void testHandleMigration() throws TException { for (int i = 0; i < tabletMetaList.size(); i++) { long tabletId = tabletIds.get(i); TabletMeta tabletMeta = tabletMetaList.get(i); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); if (db == null) { continue; } @@ -383,7 +386,8 @@ public void testHandleMigration() throws TException { Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - table = (OlapTable) db.getTable(tabletMeta.getTableId()); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getId(), tabletMeta.getTableId()); } finally { locker.unLockDatabase(db, LockType.READ); } @@ -413,11 +417,11 @@ public void submit(AgentBatchTask task) { }; OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState() - .getDb("test").getTable("binlog_report_handler_test"); + .getLocalMetastore().getDb("test").getTable("binlog_report_handler_test"); ListMultimap tabletMetaMigrationMap = ArrayListMultimap.create(); List allTablets = new ArrayList<>(); for (MaterializedIndex index : olapTable.getPartition("binlog_report_handler_test") - .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { tabletMetaMigrationMap.put(TStorageMedium.HDD, tablet.getId()); allTablets.add(tablet.getId()); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/BrokerFileGroupTest.java b/fe/fe-core/src/test/java/com/starrocks/load/BrokerFileGroupTest.java index 1e7b2bdf2926c..84322205d5897 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/BrokerFileGroupTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/BrokerFileGroupTest.java @@ -31,11 +31,14 @@ import com.starrocks.catalog.Type; import com.starrocks.common.CsvFormat; import com.starrocks.common.UserException; +import com.starrocks.server.LocalMetastore; import com.starrocks.sql.ast.DataDescription; import com.starrocks.sql.ast.UserIdentity; import com.starrocks.utframe.StarRocksAssert; import com.starrocks.utframe.UtFrameUtils; import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; import mockit.Mocked; import org.junit.Assert; import org.junit.BeforeClass; @@ -154,6 +157,13 @@ public void testParseHiveTable() throws UserException { } }; + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + }; + BrokerFileGroup fileGroup = new BrokerFileGroup(desc); fileGroup.parse(db, desc); Assert.assertEquals(Lists.newArrayList("k1", "k2"), fileGroup.getFileFieldNames()); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/DeleteHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/DeleteHandlerTest.java index c79ff8ed77e84..bd08f4abd5128 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/DeleteHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/DeleteHandlerTest.java @@ -44,6 +44,7 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.qe.QueryStateException; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.sql.analyzer.Analyzer; import com.starrocks.sql.ast.DeleteStmt; import com.starrocks.sql.ast.PartitionNames; @@ -136,14 +137,26 @@ public void logInsertTransactionState(TransactionState transactionState) { new Expectations() { { - globalStateMgr.getDb(anyString); + GlobalStateMgr.getCurrentState(); + minTimes = 0; + result = globalStateMgr; + + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = db; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = db; + globalStateMgr.getLocalMetastore().getTable("test_db", "test_tbl"); + minTimes = 0; + result = db.getTable("test_tbl"); + + globalStateMgr.getLocalMetastore().getTable(CatalogMocker.TEST_DB_ID, CatalogMocker.TEST_TBL_ID); + minTimes = 0; + result = db.getTable("test_tbl"); + globalStateMgr.getEditLog(); minTimes = 0; result = editLog; @@ -190,6 +203,13 @@ public void logInsertTransactionState(TransactionState transactionState) { minTimes = 0; } }; + + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + }; } @Test(expected = DdlException.class) @@ -535,6 +555,18 @@ public void lock(long rid, LockType lockType, long timeout) throws LockException @Test public void testRemoveOldOnReplay() throws Exception { + new Expectations(globalStateMgr) { + { + globalStateMgr.getLocalMetastore().getDb(1L); + minTimes = 0; + result = db; + + globalStateMgr.getLocalMetastore().getTable(anyLong, anyLong); + minTimes = 0; + result = db.getTable("test_tbl"); + } + }; + Config.label_keep_max_second = 1; Config.label_keep_max_num = 10; diff --git a/fe/fe-core/src/test/java/com/starrocks/load/DeletePruneTest.java b/fe/fe-core/src/test/java/com/starrocks/load/DeletePruneTest.java index c001875db12f5..bbbbe3233fed2 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/DeletePruneTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/DeletePruneTest.java @@ -17,6 +17,7 @@ import com.starrocks.catalog.Database; import com.starrocks.catalog.OlapTable; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.DeleteStmt; import com.starrocks.utframe.StarRocksAssert; import com.starrocks.utframe.UtFrameUtils; @@ -41,55 +42,56 @@ public static void beforeClass() throws Exception { deleteHandler = new DeleteMgr(); starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE `test_delete` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `k2` datetime NULL COMMENT \"\",\n" + - " `k3` char(20) NULL COMMENT \"\",\n" + - " `k4` varchar(20) NULL COMMENT \"\",\n" + - " `k5` boolean NULL COMMENT \"\",\n" + - " `k6` tinyint(4) NULL COMMENT \"\",\n" + - " `k7` smallint(6) NULL COMMENT \"\",\n" + - " `k8` int(11) NULL COMMENT \"\",\n" + - " `k9` bigint(20) NULL COMMENT \"\",\n" + - " `k10` largeint(40) NULL COMMENT \"\",\n" + - " `k11` float NULL COMMENT \"\",\n" + - " `k12` double NULL COMMENT \"\",\n" + - " `k13` decimal128(27, 9) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE(`k1`) (\n" + - " START (\"2020-01-01\") END (\"2021-01-01\") EVERY (INTERVAL 1 day)\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3 \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\"\n" + - ");") - .withTable("CREATE TABLE `test_delete2` (\n" + - " `date` date NULL COMMENT \"\",\n" + - " `id` int(11) NULL COMMENT \"\",\n" + - " `value` char(20) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`date`, `id`)\n" + - "COMMENT \"OLAP\"\n" + - "PARTITION BY RANGE(`date`, `id`)\n" + - "(\n" + - " PARTITION `p202001_1000` VALUES LESS THAN (\"2020-02-01\", \"1000\"),\n" + - " PARTITION `p202002_2000` VALUES LESS THAN (\"2020-03-01\", \"2000\"),\n" + - " PARTITION `p202003_all` VALUES LESS THAN (\"2020-04-01\")\n" + - ")\n" + - "DISTRIBUTED BY HASH(`date`, `id`) BUCKETS 3 \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");"); + .withTable("CREATE TABLE `test_delete` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` datetime NULL COMMENT \"\",\n" + + " `k3` char(20) NULL COMMENT \"\",\n" + + " `k4` varchar(20) NULL COMMENT \"\",\n" + + " `k5` boolean NULL COMMENT \"\",\n" + + " `k6` tinyint(4) NULL COMMENT \"\",\n" + + " `k7` smallint(6) NULL COMMENT \"\",\n" + + " `k8` int(11) NULL COMMENT \"\",\n" + + " `k9` bigint(20) NULL COMMENT \"\",\n" + + " `k10` largeint(40) NULL COMMENT \"\",\n" + + " `k11` float NULL COMMENT \"\",\n" + + " `k12` double NULL COMMENT \"\",\n" + + " `k13` decimal128(27, 9) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE(`k1`) (\n" + + " START (\"2020-01-01\") END (\"2021-01-01\") EVERY (INTERVAL 1 day)\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\"\n" + + ");") + .withTable("CREATE TABLE `test_delete2` (\n" + + " `date` date NULL COMMENT \"\",\n" + + " `id` int(11) NULL COMMENT \"\",\n" + + " `value` char(20) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`date`, `id`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE(`date`, `id`)\n" + + "(\n" + + " PARTITION `p202001_1000` VALUES LESS THAN (\"2020-02-01\", \"1000\"),\n" + + " PARTITION `p202002_2000` VALUES LESS THAN (\"2020-03-01\", \"2000\"),\n" + + " PARTITION `p202003_all` VALUES LESS THAN (\"2020-04-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`date`, `id`) BUCKETS 3 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); } @Test public void testDeletePrune() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); - Database db = ctx.getGlobalStateMgr().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("test_delete"); + Database db = ctx.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable tbl = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_delete"); String deleteSQL = "delete from test_delete where k1 = '2020-01-01'"; DeleteStmt deleteStmt = (DeleteStmt) UtFrameUtils.parseStmtWithNewParser(deleteSQL, ctx); @@ -153,8 +155,9 @@ public void testDeletePrune() throws Exception { @Test public void testDeletePruneMultiPartition() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); - Database db = ctx.getGlobalStateMgr().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("test_delete2"); + Database db = ctx.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable tbl = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_delete2"); String deleteSQL = "delete from test_delete2 where date in ('2020-02-02') and id = 1000"; DeleteStmt deleteStmt = (DeleteStmt) UtFrameUtils.parseStmtWithNewParser(deleteSQL, ctx); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/InsertOverwriteJobRunnerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/InsertOverwriteJobRunnerTest.java index 2de8ebd435a7f..944cf9c976db4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/InsertOverwriteJobRunnerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/InsertOverwriteJobRunnerTest.java @@ -83,8 +83,8 @@ public static void beforeClass() throws Exception { @Test public void testReplayInsertOverwrite() { - Database database = GlobalStateMgr.getCurrentState().getDb("insert_overwrite_test"); - Table table = database.getTable("t1"); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("insert_overwrite_test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "t1"); Assert.assertTrue(table instanceof OlapTable); OlapTable olapTable = (OlapTable) table; InsertOverwriteJob insertOverwriteJob = new InsertOverwriteJob(100L, database.getId(), olapTable.getId(), @@ -123,8 +123,8 @@ public void testInsertOverwrite() throws Exception { String sql = "insert overwrite t1 select * from t2"; InsertStmt insertStmt = (InsertStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); StmtExecutor executor = new StmtExecutor(connectContext, insertStmt); - Database database = GlobalStateMgr.getCurrentState().getDb("insert_overwrite_test"); - Table table = database.getTable("t1"); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("insert_overwrite_test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "t1"); Assert.assertTrue(table instanceof OlapTable); OlapTable olapTable = (OlapTable) table; InsertOverwriteJob insertOverwriteJob = new InsertOverwriteJob(100L, insertStmt, database.getId(), olapTable.getId(), diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/BrokerLoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/BrokerLoadJobTest.java index adca22a76c759..1731834094c10 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/BrokerLoadJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/BrokerLoadJobTest.java @@ -108,21 +108,17 @@ public void testFromLoadStmt(@Injectable LoadStmt loadStmt, loadStmt.getLabel(); minTimes = 0; result = labelName; + labelName.getDbName(); minTimes = 0; result = databaseName; - globalStateMgr.getDb(databaseName); - minTimes = 0; - result = database; + loadStmt.getDataDescriptions(); minTimes = 0; result = dataDescriptionList; dataDescription.getTableName(); minTimes = 0; result = tableName; - database.getTable(tableName); - minTimes = 0; - result = null; } }; @@ -162,7 +158,7 @@ public void testFromLoadStmt2(@Injectable LoadStmt loadStmt, labelName.getLabelName(); minTimes = 0; result = label; - globalStateMgr.getDb(databaseName); + globalStateMgr.getLocalMetastore().getDb(databaseName); minTimes = 0; result = database; loadStmt.getDataDescriptions(); @@ -171,7 +167,7 @@ public void testFromLoadStmt2(@Injectable LoadStmt loadStmt, dataDescription.getTableName(); minTimes = 0; result = tableName; - database.getTable(tableName); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), tableName); minTimes = 0; result = olapTable; dataDescription.getPartitionNames(); @@ -231,7 +227,7 @@ public void testAlterLoad(@Injectable LoadStmt loadStmt, labelName.getLabelName(); minTimes = 0; result = label; - globalStateMgr.getDb(databaseName); + globalStateMgr.getLocalMetastore().getDb(databaseName); minTimes = 0; result = database; loadStmt.getDataDescriptions(); @@ -240,7 +236,7 @@ public void testAlterLoad(@Injectable LoadStmt loadStmt, dataDescription.getTableName(); minTimes = 0; result = tableName; - database.getTable(tableName); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), tableName); minTimes = 0; result = olapTable; dataDescription.getPartitionNames(); @@ -296,10 +292,10 @@ public void testGetTableNames(@Injectable BrokerFileGroupAggInfo fileGroupAggInf fileGroupAggInfo.getAllTableIds(); minTimes = 0; result = Sets.newHashSet(1L); - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; - database.getTable(1L); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), 1L); minTimes = 0; result = table; table.getName(); @@ -567,7 +563,7 @@ public void testLoadingTaskOnFinished(@Injectable BrokerLoadingTaskAttachment at attachment1.getTaskId(); minTimes = 0; result = 1L; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; } @@ -719,7 +715,7 @@ public void testCommitRateExceeded(@Injectable BrokerLoadingTaskAttachment attac attachment1.getTaskId(); minTimes = 0; result = 1L; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; globalStateMgr.getCurrentState().getGlobalTransactionMgr(); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/InsertLoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/InsertLoadJobTest.java index 8bbc74ce9eb05..39688d0477c05 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/InsertLoadJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/InsertLoadJobTest.java @@ -62,9 +62,9 @@ public void testGetTableNames(@Mocked GlobalStateMgr globalStateMgr, String tableName = "table1"; new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); result = database; - database.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), anyLong); result = table; table.getName(); result = tableName; @@ -84,9 +84,9 @@ public void testUpdateProgress(@Mocked GlobalStateMgr globalStateMgr, @Injectable Table table) throws MetaNotFoundException { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); result = database; - database.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), anyLong); result = table; table.getName(); result = "some_table"; diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java index 291a688cc33a8..64f9baf3d1548 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java @@ -80,7 +80,7 @@ public void testGetDbNotExists(@Mocked GlobalStateMgr globalStateMgr) { Deencapsulation.setField(loadJob, "dbId", 1L); new Expectations() { { - globalStateMgr.getDb(1L); + globalStateMgr.getLocalMetastore().getDb(1L); minTimes = 0; result = null; } diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadMgrTest.java index 88aaf3e3deb41..5734ed0eab571 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadMgrTest.java @@ -83,10 +83,10 @@ public void testSerializationNormal(@Mocked GlobalStateMgr globalStateMgr, @Injectable Table table) throws Exception { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; - database.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), anyLong); minTimes = 0; result = table; table.getName(); @@ -115,10 +115,10 @@ public void testSerializationWithJobRemoved(@Mocked MetaContext metaContext, @Injectable Table table) throws Exception { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; - database.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), anyLong); minTimes = 0; result = table; table.getName(); @@ -150,10 +150,10 @@ public void testDeserializationWithJobRemoved(@Mocked MetaContext metaContext, @Injectable Table table) throws Exception { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; - database.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), anyLong); minTimes = 0; result = table; table.getName(); @@ -206,7 +206,7 @@ public void testRemoveOldLoadJob(@Mocked GlobalStateMgr globalStateMgr, @Injectable Database db) throws Exception { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); result = db; } }; @@ -320,7 +320,7 @@ public void testLoadJsonImage(@Mocked GlobalStateMgr globalStateMgr, @Injectable Database db) throws Exception { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); result = db; } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java index 9afbd98c1e895..c0c5c5ce41285 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java @@ -157,11 +157,11 @@ public void testCreateFromLoadStmt(@Mocked GlobalStateMgr globalStateMgr, @Injec new Expectations() { { - globalStateMgr.getDb(dbName); + globalStateMgr.getLocalMetastore().getDb(dbName); result = db; globalStateMgr.getResourceMgr(); result = resourceMgr; - db.getTable(tableName); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); result = olapTable; db.getId(); result = dbId; @@ -372,9 +372,9 @@ public void testUpdateEtlStatusFinishedAndCommitTransaction( result = status; handler.getEtlFilePaths(etlOutputPath, (BrokerDesc) any); result = filePathToSize; - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = db; - db.getTable(tableId); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); result = table; table.getPartition(partitionId); result = partition; @@ -470,9 +470,9 @@ public void testUpdateEtlStatusFinishedAndCommitTransactionForLake( result = status; handler.getEtlFilePaths(etlOutputPath, (BrokerDesc) any); result = filePathToSize; - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = db; - db.getTable(tableId); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), tableId); result = table; table.getPartition(partitionId); result = partition; @@ -528,7 +528,7 @@ public void testSubmitTasksWhenStateFinished(@Mocked GlobalStateMgr globalStateM @Injectable Database db) throws Exception { new Expectations() { { - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = db; } }; @@ -619,7 +619,7 @@ public void testNoPartitionsHaveDataLoad(@Mocked GlobalStateMgr globalStateMgr, @Injectable Database db) throws Exception { new Expectations() { { - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = db; } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadPendingTaskTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadPendingTaskTest.java index 075cebd55cf8f..ce5c02c2bcf84 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadPendingTaskTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadPendingTaskTest.java @@ -125,11 +125,11 @@ public void testExecuteTask(@Injectable SparkLoadJob sparkLoadJob, new Expectations() { { - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = database; sparkLoadJob.getHandle(); result = handle; - database.getTable(tableId); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); result = table; table.getPartitions(); result = partitions; @@ -181,7 +181,7 @@ public void testNoDb(@Injectable SparkLoadJob sparkLoadJob, new Expectations() { { - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = null; } }; @@ -210,9 +210,9 @@ public void testNoTable(@Injectable SparkLoadJob sparkLoadJob, new Expectations() { { - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = database; - database.getTable(tableId); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); result = null; } }; @@ -286,9 +286,9 @@ public void testRangePartitionHashDistribution(@Injectable SparkLoadJob sparkLoa new Expectations() { { - globalStateMgr.getDb(dbId); + globalStateMgr.getLocalMetastore().getDb(dbId); result = database; - database.getTable(tableId); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), tableId); result = table; table.getPartitions(); result = partitions; diff --git a/fe/fe-core/src/test/java/com/starrocks/load/pipe/PipeManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/pipe/PipeManagerTest.java index 1cb027928b47b..a798131df1cdd 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/pipe/PipeManagerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/pipe/PipeManagerTest.java @@ -132,7 +132,7 @@ public static void tearDown() { @After public void after() { - long dbId = ctx.getGlobalStateMgr().getDb(PIPE_TEST_DB).getId(); + long dbId = ctx.getGlobalStateMgr().getLocalMetastore().getDb(PIPE_TEST_DB).getId(); PipeManager pm = ctx.getGlobalStateMgr().getPipeManager(); pm.dropPipesOfDb(PIPE_TEST_DB, dbId); } @@ -229,7 +229,7 @@ public void persistPipe() throws Exception { UtFrameUtils.PseudoJournalReplayer.resetFollowerJournalQueue(); UtFrameUtils.PseudoImage emptyImage = new UtFrameUtils.PseudoImage(); - long dbId = ctx.getGlobalStateMgr().getDb(PIPE_TEST_DB).getId(); + long dbId = ctx.getGlobalStateMgr().getLocalMetastore().getDb(PIPE_TEST_DB).getId(); pm.dropPipesOfDb(PIPE_TEST_DB, dbId); // create pipe 1 @@ -835,7 +835,7 @@ public void pipeCRUD() throws Exception { sql = "create pipe p_crud1 as insert into tbl1 select * from files('path'='fake://pipe', 'format'='parquet')"; createStmt = (CreatePipeStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); pm.createPipe(createStmt); - long dbId = ctx.getGlobalStateMgr().getDb(PIPE_TEST_DB).getId(); + long dbId = ctx.getGlobalStateMgr().getLocalMetastore().getDb(PIPE_TEST_DB).getId(); pm.dropPipesOfDb(PIPE_TEST_DB, dbId); Assert.assertEquals(0, pm.getPipesUnlock().size()); } @@ -1030,7 +1030,7 @@ public void testRecovery() throws Exception { UtFrameUtils.PseudoJournalReplayer.resetFollowerJournalQueue(); UtFrameUtils.PseudoImage emptyImage = new UtFrameUtils.PseudoImage(); - long dbId = ctx.getGlobalStateMgr().getDb(PIPE_TEST_DB).getId(); + long dbId = ctx.getGlobalStateMgr().getLocalMetastore().getDb(PIPE_TEST_DB).getId(); pm.dropPipesOfDb(PIPE_TEST_DB, dbId); // create pipe 1 diff --git a/fe/fe-core/src/test/java/com/starrocks/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/routineload/KafkaRoutineLoadJobTest.java index df2508a908f90..1b74d232461cb 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/routineload/KafkaRoutineLoadJobTest.java @@ -279,7 +279,7 @@ public void testFromCreateStmtWithErrorTable(@Mocked GlobalStateMgr globalStateM new Expectations() { { - database.getTable(tableNameString); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), tableNameString); minTimes = 0; result = null; } @@ -312,7 +312,7 @@ public void testFromCreateStmt(@Mocked GlobalStateMgr globalStateMgr, new Expectations() { { - database.getTable(tableNameString); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), tableNameString); minTimes = 0; result = table; database.getId(); @@ -392,7 +392,7 @@ public void testSerializationCsv(@Mocked GlobalStateMgr globalStateMgr, new Expectations() { { - database.getTable(tableNameString); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), tableNameString); minTimes = 0; result = table; database.getId(); @@ -460,7 +460,7 @@ public void testSerializationJson(@Mocked GlobalStateMgr globalStateMgr, new Expectations() { { - database.getTable(tableNameString); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), tableNameString); minTimes = 0; result = table; database.getId(); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobMetaTest.java b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobMetaTest.java index 21b72ccc8e72e..eded4599f2186 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobMetaTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobMetaTest.java @@ -63,7 +63,7 @@ public static void beforeClass() throws Exception { @Test public void testMetaNotFound() throws UserException { GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb("test"); + Database db = globalStateMgr.getLocalMetastore().getDb("test"); RoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "rj", db.getId(), 2L, "", ""); Exception e = Assert.assertThrows(MetaNotFoundException.class, @@ -73,8 +73,8 @@ public void testMetaNotFound() throws UserException { @Test public void testTxnNotFound() throws UserException { GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb("test"); - Table table = db.getTable("site_access_auto"); + Database db = globalStateMgr.getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_auto"); RoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "rj", db.getId(), table.getId(), "", ""); Exception e = Assert.assertThrows(MetaNotFoundException.class, diff --git a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobTest.java index 9a914d42d044e..69d8637968f26 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadJobTest.java @@ -350,7 +350,7 @@ public RunMode getCurrentRunMode() { public void testUpdateWhileDbDeleted(@Mocked GlobalStateMgr globalStateMgr) throws UserException { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = null; } @@ -367,10 +367,10 @@ public void testUpdateWhileTableDeleted(@Mocked GlobalStateMgr globalStateMgr, @Injectable Database database) throws UserException { new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; - database.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), anyLong); minTimes = 0; result = null; } @@ -389,10 +389,10 @@ public void testUpdateWhilePartitionChanged(@Mocked GlobalStateMgr globalStateMg new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; - database.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), anyLong); minTimes = 0; result = table; } diff --git a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadManagerTest.java index b0344dd90c844..7ad594182313f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadManagerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadManagerTest.java @@ -574,7 +574,7 @@ public void testGetJobByDb(@Injectable RoutineLoadJob routineLoadJob1, routineLoadJob3.getName(); minTimes = 0; result = "bbb"; - globalStateMgr.getDb("db1"); + globalStateMgr.getLocalMetastore().getDb("db1"); minTimes = 0; result = database; database.getId(); @@ -624,7 +624,7 @@ public void testGetJobByDbAndJobName(@Injectable RoutineLoadJob routineLoadJob1, routineLoadJob3.getName(); minTimes = 0; result = "bbb"; - globalStateMgr.getDb("db1"); + globalStateMgr.getLocalMetastore().getDb("db1"); minTimes = 0; result = database; database.getId(); @@ -664,7 +664,7 @@ public void testGetJobIncludeHistory(@Injectable RoutineLoadJob routineLoadJob1, routineLoadJob3.isFinal(); minTimes = 0; result = true; - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = database; database.getId(); @@ -718,7 +718,7 @@ public void testPauseRoutineLoadJob(@Injectable PauseRoutineLoadStmt pauseRoutin pauseRoutineLoadStmt.getName(); minTimes = 0; result = ""; - globalStateMgr.getDb(""); + globalStateMgr.getLocalMetastore().getDb(""); minTimes = 0; result = database; database.getId(); @@ -769,7 +769,7 @@ public void testResumeRoutineLoadJob(@Injectable ResumeRoutineLoadStmt resumeRou resumeRoutineLoadStmt.getName(); minTimes = 0; result = ""; - globalStateMgr.getDb(""); + globalStateMgr.getLocalMetastore().getDb(""); minTimes = 0; result = database; database.getId(); @@ -806,7 +806,7 @@ public void testStopRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineLo stopRoutineLoadStmt.getName(); minTimes = 0; result = ""; - globalStateMgr.getDb(""); + globalStateMgr.getLocalMetastore().getDb(""); minTimes = 0; result = database; database.getId(); @@ -946,7 +946,7 @@ public void testAlterRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineL stopRoutineLoadStmt.getName(); minTimes = 0; result = ""; - globalStateMgr.getDb(""); + globalStateMgr.getLocalMetastore().getDb(""); minTimes = 0; result = database; database.getId(); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadSchedulerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadSchedulerTest.java index a2cae89c82423..a25b61ca297d5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadSchedulerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/routineload/RoutineLoadSchedulerTest.java @@ -104,12 +104,6 @@ public void testNormalRunOneCycle(@Mocked GlobalStateMgr globalStateMgr, routineLoadManager.getRoutineLoadJobByState(Sets.newHashSet(RoutineLoadJob.JobState.NEED_SCHEDULE)); minTimes = 0; result = routineLoadJobList; - globalStateMgr.getDb(anyLong); - minTimes = 0; - result = database; - database.getTable(1L); - minTimes = 0; - result = olapTable; systemInfoService.getBackendIds(true); minTimes = 0; result = beIds; @@ -172,7 +166,7 @@ public void functionTest(@Mocked GlobalStateMgr globalStateMgr, globalStateMgr.getRoutineLoadMgr(); minTimes = 0; result = routineLoadManager; - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); minTimes = 0; result = database; systemInfoService.getBackendIds(true); diff --git a/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java index d399672912c43..34439ac4e5bff 100644 --- a/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java @@ -83,10 +83,14 @@ public void logInsertTransactionState(TransactionState transactionState) { new Expectations() { { - globalStateMgr.getDb(anyString); + globalStateMgr.getLocalMetastore().getDb(anyString); minTimes = 0; result = db; + globalStateMgr.getLocalMetastore().getTable(anyString, anyString); + minTimes = 0; + result = db.getTable(CatalogMocker.TEST_TBL_ID); + globalStateMgr.getEditLog(); minTimes = 0; result = editLog; diff --git a/fe/fe-core/src/test/java/com/starrocks/persist/ColumnRenameTest.java b/fe/fe-core/src/test/java/com/starrocks/persist/ColumnRenameTest.java index 7830c513826c3..b788b6827d4df 100644 --- a/fe/fe-core/src/test/java/com/starrocks/persist/ColumnRenameTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/persist/ColumnRenameTest.java @@ -84,24 +84,24 @@ public static void beforeClass() throws Exception { @Test public void testReplayRenameColumn() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = testDb.getTable("tbl1"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1"); ColumnRenameInfo columnRenameInfo = new ColumnRenameInfo(testDb.getId(), table.getId(), "k1", "k3"); GlobalStateMgr.getCurrentState().getLocalMetastore().replayRenameColumn(columnRenameInfo); Assert.assertEquals("k3", table.getColumn("k3").getName()); - table = testDb.getTable("tbl2"); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl2"); columnRenameInfo = new ColumnRenameInfo(testDb.getId(), table.getId(), "k1", "k3"); GlobalStateMgr.getCurrentState().getLocalMetastore().replayRenameColumn(columnRenameInfo); Assert.assertEquals("k3", table.getColumn("k3").getName()); - table = testDb.getTable("tbl3"); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl3"); columnRenameInfo = new ColumnRenameInfo(testDb.getId(), table.getId(), "k1", "k3"); GlobalStateMgr.getCurrentState().getLocalMetastore().replayRenameColumn(columnRenameInfo); Assert.assertEquals("k3", table.getColumn("k3").getName()); - table = testDb.getTable("tbl4"); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl4"); columnRenameInfo = new ColumnRenameInfo(testDb.getId(), table.getId(), "k1", "k3"); GlobalStateMgr.getCurrentState().getLocalMetastore().replayRenameColumn(columnRenameInfo); Assert.assertEquals("k3", table.getColumn("k3").getName()); diff --git a/fe/fe-core/src/test/java/com/starrocks/persist/RenameMaterializedViewLogTest.java b/fe/fe-core/src/test/java/com/starrocks/persist/RenameMaterializedViewLogTest.java index 842f09b2c4ea9..d7a7b75e9fabb 100644 --- a/fe/fe-core/src/test/java/com/starrocks/persist/RenameMaterializedViewLogTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/persist/RenameMaterializedViewLogTest.java @@ -65,10 +65,10 @@ public void testNormal(@Mocked GlobalStateMgr globalStateMgr, in.close(); new Expectations() { { - globalStateMgr.getDb(anyLong); + globalStateMgr.getLocalMetastore().getDb(anyLong); result = db; - db.getTable(anyLong); + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), anyLong); result = table; } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewLowCardTPCHTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewLowCardTPCHTest.java index 4b0a6127f76eb..ce8a7be5709eb 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewLowCardTPCHTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewLowCardTPCHTest.java @@ -39,9 +39,9 @@ public static void beforeClass() throws Exception { int scale = 1; GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); connectContext.getGlobalStateMgr().setStatisticStorage(new MockTpchStatisticStorage(connectContext, scale)); - OlapTable t4 = (OlapTable) globalStateMgr.getDb(MATERIALIZED_DB_NAME).getTable("customer"); + OlapTable t4 = (OlapTable) globalStateMgr.getLocalMetastore().getDb(MATERIALIZED_DB_NAME).getTable("customer"); setTableStatistics(t4, 150000 * scale); - OlapTable t7 = (OlapTable) globalStateMgr.getDb(MATERIALIZED_DB_NAME).getTable("lineitem"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb(MATERIALIZED_DB_NAME).getTable("lineitem"); setTableStatistics(t7, 6000000 * scale); connectContext.getSessionVariable().setEnableLowCardinalityOptimize(true); diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTPCHTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTPCHTest.java index f1b88d2a30862..02d777d36fb59 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTPCHTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTPCHTest.java @@ -48,9 +48,9 @@ public static void beforeClass() throws Exception { int scale = 1; GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); connectContext.getGlobalStateMgr().setStatisticStorage(new MockTpchStatisticStorage(connectContext, scale)); - OlapTable t4 = (OlapTable) globalStateMgr.getDb(MATERIALIZED_DB_NAME).getTable("customer"); + OlapTable t4 = (OlapTable) globalStateMgr.getLocalMetastore().getDb(MATERIALIZED_DB_NAME).getTable("customer"); setTableStatistics(t4, 150000 * scale); - OlapTable t7 = (OlapTable) globalStateMgr.getDb(MATERIALIZED_DB_NAME).getTable("lineitem"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb(MATERIALIZED_DB_NAME).getTable("lineitem"); setTableStatistics(t7, 6000000 * scale); // When force rule based rewrite is enabled, query will be transformed into scan in Rule Rewrite Phase. diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java index 70876e90c169d..bdeba6cbebe0a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java @@ -415,7 +415,7 @@ public static void beforeClass() throws Exception { ");"); GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t7 = (OlapTable) globalStateMgr.getDb(MATERIALIZED_DB_NAME).getTable("emps"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb(MATERIALIZED_DB_NAME).getTable("emps"); setTableStatistics(t7, 6000000); } @@ -4563,7 +4563,7 @@ public void testJoinWithToBitmapRewrite() throws Exception { ");"; starRocksAssert.withTable(table2); - OlapTable t4 = (OlapTable) GlobalStateMgr.getCurrentState().getDb(MATERIALIZED_DB_NAME) + OlapTable t4 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(MATERIALIZED_DB_NAME) .getTable("test_sr_table_join"); setTableStatistics(t4, 150000); diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java index 9d983f5cd5fe1..ebb8d8bc50609 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java @@ -288,8 +288,8 @@ protected MVRewriteChecker rewrite(String mv, String query, String properties) t } protected static Table getTable(String dbName, String mvName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(mvName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName); Assert.assertNotNull(table); return table; } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTextBasedRewriteTPCHTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTextBasedRewriteTPCHTest.java index 5c041bc3b446d..0d3c6ec905c7f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTextBasedRewriteTPCHTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTextBasedRewriteTPCHTest.java @@ -44,9 +44,9 @@ public static void beforeClass() throws Exception { int scale = 1; GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); connectContext.getGlobalStateMgr().setStatisticStorage(new MockTpchStatisticStorage(connectContext, scale)); - OlapTable t4 = (OlapTable) globalStateMgr.getDb(MATERIALIZED_DB_NAME).getTable("customer"); + OlapTable t4 = (OlapTable) globalStateMgr.getLocalMetastore().getDb(MATERIALIZED_DB_NAME).getTable("customer"); setTableStatistics(t4, 150000 * scale); - OlapTable t7 = (OlapTable) globalStateMgr.getDb(MATERIALIZED_DB_NAME).getTable("lineitem"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb(MATERIALIZED_DB_NAME).getTable("lineitem"); setTableStatistics(t7, 6000000 * scale); } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest2.java b/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest2.java index 15ff346f041e9..ec268ea4493ac 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest2.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/OlapTableSinkTest2.java @@ -58,8 +58,8 @@ public int getQuorumNum(long partitionId, TWriteQuorumType writeQuorum) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("db2"); - OlapTable olapTable = (OlapTable) db.getTable("tbl1"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db2"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); List partitionIds = olapTable.getAllPartitionIds(); diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java index b54009123101d..5287a5dbea28e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/mv/MVMetaVersionRepairerTest.java @@ -102,7 +102,7 @@ public void testRepairBaseTableVersionChanges1() { MVRepairHandler.PartitionRepairInfo partitionRepairInfo = toPartitionInfo(curPartition, 100L, currentTs); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Table baseTable = getTable("test", "m1"); MVMetaVersionRepairer.repairBaseTableVersionChanges(db, baseTable, ImmutableList.of(partitionRepairInfo)); @@ -156,7 +156,7 @@ public void testRepairBaseTableVersionChanges2() { MVRepairHandler.PartitionRepairInfo partitionRepairInfo = toPartitionInfo(curPartition, 100L, currentTs); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Table baseTable = getTable("test", "m1"); MVMetaVersionRepairer.repairBaseTableVersionChanges(db, baseTable, ImmutableList.of(partitionRepairInfo)); @@ -215,7 +215,7 @@ public void testRepairBaseTableVersionChanges3() { MVRepairHandler.PartitionRepairInfo partitionRepairInfo = toPartitionInfo(p2, 100L, currentTs); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Table baseTable = getTable("test", "m1"); MVMetaVersionRepairer.repairBaseTableVersionChanges(db, baseTable, ImmutableList.of(partitionRepairInfo)); @@ -282,7 +282,7 @@ public void testRepairBaseTableVersionChanges4() { MVRepairHandler.PartitionRepairInfo partitionRepairInfo = toPartitionInfo(p1, 100L, currentTs); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Table baseTable = getTable("test", "m1"); MVMetaVersionRepairer.repairBaseTableVersionChanges(db, baseTable, ImmutableList.of(partitionRepairInfo)); diff --git a/fe/fe-core/src/test/java/com/starrocks/privilege/AuthorizationMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/privilege/AuthorizationMgrTest.java index d65f0cd5a7d5e..85b9eac6d9c29 100644 --- a/fe/fe-core/src/test/java/com/starrocks/privilege/AuthorizationMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/privilege/AuthorizationMgrTest.java @@ -22,6 +22,7 @@ import com.starrocks.catalog.Column; import com.starrocks.catalog.HiveTable; import com.starrocks.catalog.InternalCatalog; +import com.starrocks.catalog.Table; import com.starrocks.catalog.Type; import com.starrocks.common.Config; import com.starrocks.common.DdlException; @@ -45,6 +46,7 @@ import com.starrocks.qe.ShowResultSet; import com.starrocks.qe.StmtExecutor; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.server.MetadataMgr; import com.starrocks.sql.analyzer.Authorizer; import com.starrocks.sql.ast.CreateCatalogStmt; @@ -1663,6 +1665,18 @@ public void testGrantBrief() throws Exception { @Test public void testPartialRevoke() throws Exception { + new MockUp() { + @Mock + public Table getTable(String dbName, String tblName) { + return GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(new TableName("db", "tbl1")).get(); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(new TableName("db", "tbl1")).get(); + } + }; + String sql = "grant select on table db.tbl0 to test_user"; DDLStmtExecutor.execute(UtFrameUtils.parseStmtWithNewParser(sql, ctx), ctx); diff --git a/fe/fe-core/src/test/java/com/starrocks/privilege/RBACMockedMetadataMgr.java b/fe/fe-core/src/test/java/com/starrocks/privilege/RBACMockedMetadataMgr.java index 49f1f0a792c19..e9d2a338d4c56 100644 --- a/fe/fe-core/src/test/java/com/starrocks/privilege/RBACMockedMetadataMgr.java +++ b/fe/fe-core/src/test/java/com/starrocks/privilege/RBACMockedMetadataMgr.java @@ -117,17 +117,6 @@ public Table getTable(String catalogName, String dbName, String tblName) { return tableMap.get(tblName); } - @Override - public Table getTable(Long databaseId, Long tableId) { - for (Table table : tableMap.values()) { - if (table.getId() == tableId) { - return table; - } - } - - return null; - } - @Override public List listTableNames(String catalogName, String dbName) { return new ArrayList<>(tableMap.keySet()); diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/BeRestartTest.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/BeRestartTest.java index 9749c2e1b6eb3..62ababccc94e0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/BeRestartTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/BeRestartTest.java @@ -64,7 +64,7 @@ public void testBeRestart() throws Exception { createTableSqls[i] = PseudoCluster.newCreateTableSqlBuilder().setTableName(name).build(); insertSqls[i] = PseudoCluster.buildInsertSql("test", name); cluster.runSqls("test", createTableSqls[i], insertSqls[i], insertSqls[i], insertSqls[i]); - tableIds[i] = GlobalStateMgr.getCurrentState().getDb("test").getTable(name).getId(); + tableIds[i] = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable(name).getId(); // insert 3 times -> version: 4 tableVersions[i] = 4; } diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java index b451036350beb..eeecf69483ac4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationLabeledTableBalanceTest.java @@ -105,7 +105,7 @@ public void test1BestEffortBalance() throws SQLException, InterruptedException { ");"; cluster.runSql("test", sql); - Database test = GlobalStateMgr.getCurrentState().getDb("test"); + Database test = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); OlapTable olapTable = (OlapTable) test.getTable("test_table_best_effort_balance"); // Add another backend without location property, with best-effort balance strategy, @@ -177,7 +177,7 @@ private void printTabletReplicaInfo(OlapTable table) { @Test @Ignore public void test2LocationMatchedBalance() throws InterruptedException, SQLException { - Database test = GlobalStateMgr.getCurrentState().getDb("test"); + Database test = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); OlapTable olapTable = (OlapTable) test.getTable("test_table_best_effort_balance"); long start = System.currentTimeMillis(); diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java index 9eee82727dba6..77652e2b487a7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/LocationMismatchRepairTest.java @@ -81,7 +81,7 @@ private void setBackendLocationProp() throws SQLException { int j = 0; for (PseudoBackend backend : cluster.getBackends()) { String stmtStr = "alter system modify backend '" + backend.getHostHeartbeatPort() + "' set ('" + - AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:r" + i + "')"; + AlterSystemStmtAnalyzer.PROP_KEY_LOCATION + "' = 'rack:r" + i + "')"; if (j % 2 == 1) { i++; } @@ -96,9 +96,9 @@ private Set getBackendIdsWithLocProp(String locationKey, String locationVa Set backendIds = Sets.newHashSet(); for (PseudoBackend pseudoBackend : cluster.getBackends()) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(pseudoBackend.getId()); + .getClusterInfo().getBackend(pseudoBackend.getId()); if (backend.getLocation().containsKey(locationKey) && - Objects.equals(backend.getLocation().get(locationKey), locationVal)) { + Objects.equals(backend.getLocation().get(locationKey), locationVal)) { backendIds.add(backend.getId()); } } @@ -112,9 +112,9 @@ private void printTabletReplicaInfo(OlapTable table) { stringBuffer.append("tablet ").append(tablet.getId()).append(": "); for (Replica replica : tablet.getAllReplicas()) { stringBuffer.append(replica.getBackendId()).append(" ") - .append(GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend( - replica.getBackendId()).getLocation()).append(", "); + .append(GlobalStateMgr.getCurrentState().getNodeMgr() + .getClusterInfo().getBackend( + replica.getBackendId()).getLocation()).append(", "); } System.out.println(stringBuffer); }); @@ -136,20 +136,21 @@ public void testRepairAfterChangeTableLocation() throws Exception { PseudoCluster cluster = PseudoCluster.getInstance(); String sql = "CREATE TABLE test.`test_table_backend_no_loc` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_table_backend_no_loc"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_backend_no_loc"); Assert.assertNotNull(table.getLocation()); Assert.assertTrue(table.getLocation().keySet().contains("*")); @@ -173,22 +174,22 @@ public void testRepairAfterChangeTableLocation() throws Exception { long oldCloneFinishedCnt = stat.counterCloneTaskSucceeded.get(); long oldLocationMismatchErr = stat.counterReplicaLocMismatchErr.get(); System.out.println("old clone finished: " + oldCloneFinishedCnt + ", old location mismatch: " + - oldLocationMismatchErr); + oldLocationMismatchErr); sql = "ALTER TABLE test.`test_table_backend_no_loc` SET ('" + - PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:r1,rack:r2,rack:r3');"; + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:r1,rack:r2,rack:r3');"; cluster.runSql("test", sql); System.out.println("========================="); long start = System.currentTimeMillis(); while (true) { if (stat.counterCloneTaskSucceeded.get() - oldCloneFinishedCnt >= locationMismatchFullCloneNeeded - && stat.counterReplicaLocMismatchErr.get() - oldLocationMismatchErr >= - locationMismatchFullCloneNeeded) { + && stat.counterReplicaLocMismatchErr.get() - oldLocationMismatchErr >= + locationMismatchFullCloneNeeded) { break; } System.out.println("wait for enough clone tasks for LOCATION_MISMATCH finished, " + - "current clone finished: " + stat.counterCloneTaskSucceeded.get() + - ", current location mismatch: " + stat.counterReplicaLocMismatchErr.get() + - ", expected clone finished: " + locationMismatchFullCloneNeeded); + "current clone finished: " + stat.counterCloneTaskSucceeded.get() + + ", current location mismatch: " + stat.counterReplicaLocMismatchErr.get() + + ", expected clone finished: " + locationMismatchFullCloneNeeded); Thread.sleep(1000); if (System.currentTimeMillis() - start > 180000) { Assert.fail("wait for enough clone tasks for LOCATION_MISMATCH finished timeout"); @@ -208,7 +209,7 @@ public void testRepairAfterChangeTableLocation() throws Exception { Set> racks = new HashSet<>(); for (long backendId : intersection) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(backendId); + .getClusterInfo().getBackend(backendId); racks.add(backend.getSingleLevelLocationKV()); } Assert.assertEquals(3, racks.size()); @@ -221,17 +222,17 @@ public void testRepairLocationAfterDecommissionBackend() throws Exception { PseudoCluster cluster = PseudoCluster.getInstance(); String sql = "CREATE TABLE test.`test_table_backend_no_loc` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r0,rack:r1,rack:r2,rack:r4\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:r0,rack:r1,rack:r2,rack:r4\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); // decommission backends with 'rack:r4' location @@ -251,7 +252,7 @@ public void testRepairLocationAfterDecommissionBackend() throws Exception { while (true) { int curTabletNum = decommissionBE.getTabletManager().getNumTablet(); System.out.printf("#tablets: %d/%d: fullClone: %d wait...\n", curTabletNum, pair.second, - Tablet.getTotalFullClone()); + Tablet.getTotalFullClone()); if (curTabletNum == 0) { break; } @@ -261,8 +262,9 @@ public void testRepairLocationAfterDecommissionBackend() throws Exception { // sleep to wait for redundant replicas cleaned Thread.sleep(5000); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_table_backend_no_loc"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_table_backend_no_loc"); printTabletReplicaInfo(table); Set stayedBackendIds = getBackendIdsWithLocProp("rack", "r0"); stayedBackendIds.addAll(getBackendIdsWithLocProp("rack", "r1")); @@ -277,14 +279,13 @@ public void testRepairLocationAfterDecommissionBackend() throws Exception { Set> racks = new HashSet<>(); for (long backendId : intersection) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(backendId); + .getClusterInfo().getBackend(backendId); racks.add(backend.getSingleLevelLocationKV()); } Assert.assertEquals(3, racks.size()); } } - @Test public void testMVRepairAfterChangeTableLocation() throws Exception { setBackendLocationProp(); @@ -292,32 +293,33 @@ public void testMVRepairAfterChangeTableLocation() throws Exception { PseudoCluster cluster = PseudoCluster.getInstance(); // create base table String sql = "CREATE TABLE test.`test_base_table` (\n" + - " k1 int,\n" + - " k2 VARCHAR NOT NULL\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + - "PROPERTIES (\n" + - " \"replication_num\" = \"3\",\n" + - " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:*\",\n" + - " \"in_memory\" = \"false\"\n" + - ");"; + " k1 int,\n" + + " k2 VARCHAR NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 5\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"3\",\n" + + " \"" + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "\" = \"rack:*\",\n" + + " \"in_memory\" = \"false\"\n" + + ");"; cluster.runSql("test", sql); // create mv sql = "CREATE MATERIALIZED VIEW test_mv01\n" + - "DISTRIBUTED BY HASH(`k1`)\n" + - "buckets 15\n" + - "REFRESH MANUAL\n" + - "properties(\n" + - " \"replication_num\" = \"3\"" + - ")\n" + - "AS SELECT test_base_table.k1 FROM test_base_table;"; + "DISTRIBUTED BY HASH(`k1`)\n" + + "buckets 15\n" + + "REFRESH MANUAL\n" + + "properties(\n" + + " \"replication_num\" = \"3\"" + + ")\n" + + "AS SELECT test_base_table.k1 FROM test_base_table;"; cluster.runSql("test", sql); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("test_mv01"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_mv01"); Assert.assertNotNull(table.getLocation()); Assert.assertTrue(table.getLocation().keySet().contains("*")); @@ -341,22 +343,22 @@ public void testMVRepairAfterChangeTableLocation() throws Exception { long oldCloneFinishedCnt = stat.counterCloneTaskSucceeded.get(); long oldLocationMismatchErr = stat.counterReplicaLocMismatchErr.get(); System.out.println("old clone finished: " + oldCloneFinishedCnt + ", old location mismatch: " + - oldLocationMismatchErr); + oldLocationMismatchErr); sql = "ALTER MATERIALIZED VIEW test.`test_mv01` SET ('" + - PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:r1,rack:r2,rack:r3');"; + PropertyAnalyzer.PROPERTIES_LABELS_LOCATION + "' = 'rack:r1,rack:r2,rack:r3');"; cluster.runSql("test", sql); System.out.println("========================="); long start = System.currentTimeMillis(); while (true) { if (stat.counterCloneTaskSucceeded.get() - oldCloneFinishedCnt >= locationMismatchFullCloneNeeded - && stat.counterReplicaLocMismatchErr.get() - oldLocationMismatchErr >= - locationMismatchFullCloneNeeded) { + && stat.counterReplicaLocMismatchErr.get() - oldLocationMismatchErr >= + locationMismatchFullCloneNeeded) { break; } System.out.println("wait for enough clone tasks for LOCATION_MISMATCH finished, " + - "current clone finished: " + stat.counterCloneTaskSucceeded.get() + - ", current location mismatch: " + stat.counterReplicaLocMismatchErr.get() + - ", expected clone finished: " + locationMismatchFullCloneNeeded); + "current clone finished: " + stat.counterCloneTaskSucceeded.get() + + ", current location mismatch: " + stat.counterReplicaLocMismatchErr.get() + + ", expected clone finished: " + locationMismatchFullCloneNeeded); Thread.sleep(1000); if (System.currentTimeMillis() - start > 180000) { Assert.fail("wait for enough clone tasks for LOCATION_MISMATCH finished timeout"); @@ -376,7 +378,7 @@ public void testMVRepairAfterChangeTableLocation() throws Exception { Set> racks = new HashSet<>(); for (long backendId : intersection) { Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr() - .getClusterInfo().getBackend(backendId); + .getClusterInfo().getBackend(backendId); racks.add(backend.getSingleLevelLocationKV()); } Assert.assertEquals(3, racks.size()); diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoCluster.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoCluster.java index ad5738a4bd967..94a0d17ea1fba 100644 --- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoCluster.java +++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoCluster.java @@ -284,14 +284,14 @@ public Connection getQueryConnection() throws SQLException { } public List listTablets(String dbName, String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); if (db == null) { return null; } Locker locker = new Locker(); locker.lockDatabase(db, LockType.READ); try { - Table table = db.getTable(tableName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (table == null) { return null; } diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/ReplicaMinReadableVersionTest.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/ReplicaMinReadableVersionTest.java index db274741ed990..f683987f05f0e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/ReplicaMinReadableVersionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/ReplicaMinReadableVersionTest.java @@ -50,7 +50,7 @@ public static void tearDown() throws Exception { @Test public void testReplicaMinReadableVersionReported() throws Exception { - OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test").getTable(tableName); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable(tableName); long tableId = olapTable.getId(); PseudoBackend be = cluster.getBackend(10001); Tablet tablet = be.getTabletsByTable(tableId).get(0); diff --git a/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java index bb9fb47f4832b..b842143bc7161 100644 --- a/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/qe/ShowExecutorTest.java @@ -82,6 +82,7 @@ import com.starrocks.privilege.PrivilegeBuiltinConstants; import com.starrocks.server.CatalogMgr; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.server.MetadataMgr; import com.starrocks.server.NodeMgr; import com.starrocks.server.RunMode; @@ -253,7 +254,7 @@ public void setUp() throws Exception { }; BaseTableInfo baseTableInfo = new BaseTableInfo( - "default_catalog", "testDb", "testTbl", null); + "default_catalog", "testDb", "testTbl", null); // mock materialized view MaterializedView mv = new MaterializedView(); @@ -302,10 +303,10 @@ public void setUp() throws Exception { mv.getPartitionInfo(); minTimes = 0; result = new ExpressionRangePartitionInfo( - Collections.singletonList( - ColumnIdExpr.create(new SlotRef( - new TableName("test", "testMv"), column1.getName()))), - Collections.singletonList(column1), PartitionType.RANGE); + Collections.singletonList( + ColumnIdExpr.create(new SlotRef( + new TableName("test", "testMv"), column1.getName()))), + Collections.singletonList(column1), PartitionType.RANGE); mv.getDefaultDistributionInfo(); minTimes = 0; @@ -326,7 +327,7 @@ public void setUp() throws Exception { mv.getTableProperty(); minTimes = 0; result = new TableProperty( - Collections.singletonMap(PROPERTIES_STORAGE_COOLDOWN_TIME, "100")); + Collections.singletonMap(PROPERTIES_STORAGE_COOLDOWN_TIME, "100")); mv.getIdToColumn(); minTimes = 0; @@ -366,20 +367,29 @@ public void setUp() throws Exception { // mock globalStateMgr. globalStateMgr = Deencapsulation.newInstance(GlobalStateMgr.class); + LocalMetastore localMetastore = new LocalMetastore(globalStateMgr, null, null); new Expectations(globalStateMgr) { { - globalStateMgr.getDb("testDb"); + /* + globalStateMgr.getLocalMetastore().getDb("testDb"); minTimes = 0; result = db; - globalStateMgr.getDb("emptyDb"); + + globalStateMgr.getLocalMetastore().getDb("emptyDb"); minTimes = 0; result = null; + */ + GlobalStateMgr.getCurrentState(); minTimes = 0; result = globalStateMgr; + globalStateMgr.getLocalMetastore(); + minTimes = 0; + result = localMetastore; + globalStateMgr.getMetadataMgr(); minTimes = 0; result = metadataMgr; @@ -397,12 +407,32 @@ public void setUp() throws Exception { result = null; metadataMgr.getTable("default_catalog", "testDb", - "testTbl"); + "testTbl"); minTimes = 0; result = table; } }; + new MockUp() { + @Mock + public Database getDb(String dbName) { + if (dbName.equalsIgnoreCase("emptyDb")) { + return null; + } + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public List
getTables(Long dbId) { + return db.getTables(); + } + }; + // mock scheduler ConnectScheduler scheduler = new ConnectScheduler(10); new Expectations(scheduler) { @@ -473,6 +503,7 @@ public List getAvailableBackendIds() { OlapTable olapTable = listPartitionInfoTest.findTableForMultiListPartition(); Database db = new Database(); + /* new Expectations(db) { { db.getTable(anyString); @@ -485,6 +516,8 @@ public List getAvailableBackendIds() { } }; + */ + new MockUp() { @Mock public Table getSessionAwareTable(ConnectContext ctx, Database db, TableName tableName) { @@ -494,15 +527,19 @@ public Table getSessionAwareTable(ConnectContext ctx, Database db, TableName tab new Expectations() { { - globalStateMgr.getDb(0); + globalStateMgr.getLocalMetastore().getDb(0); minTimes = 0; result = db; + + globalStateMgr.getLocalMetastore().getTable(anyLong, anyLong); + minTimes = 0; + result = olapTable; } }; // Ok to test ShowPartitionsStmt stmt = new ShowPartitionsStmt(new TableName("testDb", "testTbl"), - null, null, null, false); + null, null, null, false); com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx); @@ -549,7 +586,7 @@ public void testDescribe() throws DdlException { ctx.setQualifiedUser("testUser"); DescribeStmt stmt = (DescribeStmt) com.starrocks.sql.parser.SqlParser.parse("desc testTbl", - ctx.getSessionVariable().getSqlMode()).get(0); + ctx.getSessionVariable().getSqlMode()).get(0); com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); ShowResultSet resultSet; @@ -655,7 +692,7 @@ public void testShowCreateNoDb() { @Test(expected = SemanticException.class) public void testShowCreateTableEmptyDb() throws SemanticException, DdlException { ShowCreateTableStmt stmt = new ShowCreateTableStmt(new TableName("emptyDb", "testTable"), - ShowCreateTableStmt.CreateTableType.TABLE); + ShowCreateTableStmt.CreateTableType.TABLE); ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx); @@ -668,7 +705,7 @@ public void testShowColumn() throws AnalysisException, DdlException { ctx.setQualifiedUser("testUser"); ShowColumnStmt stmt = (ShowColumnStmt) com.starrocks.sql.parser.SqlParser.parse("show columns from testTbl in testDb", - ctx.getSessionVariable()).get(0); + ctx.getSessionVariable()).get(0); com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx); @@ -682,7 +719,7 @@ public void testShowColumn() throws AnalysisException, DdlException { // verbose stmt = (ShowColumnStmt) com.starrocks.sql.parser.SqlParser.parse("show full columns from testTbl in testDb", - ctx.getSessionVariable()).get(0); + ctx.getSessionVariable()).get(0); com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); resultSet = ShowExecutor.execute(stmt, ctx); @@ -697,7 +734,7 @@ public void testShowColumn() throws AnalysisException, DdlException { // show full fields stmt = (ShowColumnStmt) com.starrocks.sql.parser.SqlParser.parse("show full fields from testTbl in testDb", - ctx.getSessionVariable()).get(0); + ctx.getSessionVariable()).get(0); com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); resultSet = ShowExecutor.execute(stmt, ctx); @@ -712,7 +749,7 @@ public void testShowColumn() throws AnalysisException, DdlException { // pattern stmt = (ShowColumnStmt) com.starrocks.sql.parser.SqlParser.parse("show full columns from testTbl in testDb like \"%1\"", - ctx.getSessionVariable().getSqlMode()).get(0); + ctx.getSessionVariable().getSqlMode()).get(0); com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); resultSet = ShowExecutor.execute(stmt, ctx); @@ -895,10 +932,10 @@ RunMode getCurrentRunMode() { ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx); Assert.assertEquals(ComputeNodeProcDir.TITLE_NAMES_SHARED_DATA.size(), - resultSet.getMetaData().getColumnCount()); + resultSet.getMetaData().getColumnCount()); for (int i = 0; i < ComputeNodeProcDir.TITLE_NAMES_SHARED_DATA.size(); ++i) { Assert.assertEquals(ComputeNodeProcDir.TITLE_NAMES_SHARED_DATA.get(i), - resultSet.getMetaData().getColumn(i).getName()); + resultSet.getMetaData().getColumn(i).getName()); } Assert.assertTrue(resultSet.next()); @@ -1003,15 +1040,15 @@ public void testShowMaterializedViewPattern() throws AnalysisException, DdlExcep private void verifyShowMaterializedViewResult(ShowResultSet resultSet) throws AnalysisException, DdlException { String expectedSqlText = "CREATE MATERIALIZED VIEW `testMv` (`col1`, `col2`)\n" + - "COMMENT \"TEST MATERIALIZED VIEW\"\n" + - "PARTITION BY (`col1`)\n" + - "DISTRIBUTED BY HASH(`col1`) BUCKETS 10 \n" + - "REFRESH ASYNC\n" + - "PROPERTIES (\n" + - "\"storage_cooldown_time\" = \"1970-01-01 08:00:00\",\n" + - "\"storage_medium\" = \"SSD\"\n" + - ")\n" + - "AS select col1, col2 from table1;"; + "COMMENT \"TEST MATERIALIZED VIEW\"\n" + + "PARTITION BY (`col1`)\n" + + "DISTRIBUTED BY HASH(`col1`) BUCKETS 10 \n" + + "REFRESH ASYNC\n" + + "PROPERTIES (\n" + + "\"storage_cooldown_time\" = \"1970-01-01 08:00:00\",\n" + + "\"storage_medium\" = \"SSD\"\n" + + ")\n" + + "AS select col1, col2 from table1;"; Assert.assertTrue(resultSet.next()); List mvSchemaTable = MaterializedViewsSystemTable.create().getFullSchema(); @@ -1053,7 +1090,8 @@ public void testShowRoutineLoadNonExisted() throws AnalysisException, DdlExcepti @Test public void testShowAlterTable() throws AnalysisException, DdlException { ShowAlterStmt stmt = new ShowAlterStmt(ShowAlterStmt.AlterType.OPTIMIZE, "testDb", null, null, null); - stmt.setNode(new OptimizeProcDir(globalStateMgr.getSchemaChangeHandler(), globalStateMgr.getDb("testDb"))); + stmt.setNode(new OptimizeProcDir(globalStateMgr.getSchemaChangeHandler(), + globalStateMgr.getLocalMetastore().getDb("testDb"))); ShowExecutor.execute(stmt, ctx); } @@ -1082,34 +1120,34 @@ public Table getTable(String catalogName, String dbName, String tblName) { partitions.add("year"); partitions.add("dt"); HiveTable.Builder tableBuilder = HiveTable.builder() - .setId(1) - .setTableName("test_table") - .setCatalogName("hive_catalog") - .setResourceName(toResourceName("hive_catalog", "hive")) - .setHiveDbName("hive_db") - .setHiveTableName("test_table") - .setPartitionColumnNames(partitions) - .setFullSchema(fullSchema) - .setTableLocation("hdfs://hadoop/hive/warehouse/test.db/test") - .setCreateTime(10000); + .setId(1) + .setTableName("test_table") + .setCatalogName("hive_catalog") + .setResourceName(toResourceName("hive_catalog", "hive")) + .setHiveDbName("hive_db") + .setHiveTableName("test_table") + .setPartitionColumnNames(partitions) + .setFullSchema(fullSchema) + .setTableLocation("hdfs://hadoop/hive/warehouse/test.db/test") + .setCreateTime(10000); return tableBuilder.build(); } }; ShowCreateTableStmt stmt = new ShowCreateTableStmt(new TableName("hive_catalog", "hive_db", "test_table"), - ShowCreateTableStmt.CreateTableType.TABLE); + ShowCreateTableStmt.CreateTableType.TABLE); ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx); Assert.assertEquals("test_table", resultSet.getResultRows().get(0).get(0)); Assert.assertEquals("CREATE TABLE `test_table` (\n" + - " `id` int(11) DEFAULT NULL COMMENT \"id\",\n" + - " `name` varchar DEFAULT NULL,\n" + - " `year` int(11) DEFAULT NULL,\n" + - " `dt` int(11) DEFAULT NULL\n" + - ")\n" + - "PARTITION BY (year, dt)\n" + - "PROPERTIES (\"location\" = \"hdfs://hadoop/hive/warehouse/test.db/test\");", - resultSet.getResultRows().get(0).get(1)); + " `id` int(11) DEFAULT NULL COMMENT \"id\",\n" + + " `name` varchar DEFAULT NULL,\n" + + " `year` int(11) DEFAULT NULL,\n" + + " `dt` int(11) DEFAULT NULL\n" + + ")\n" + + "PARTITION BY (year, dt)\n" + + "PROPERTIES (\"location\" = \"hdfs://hadoop/hive/warehouse/test.db/test\");", + resultSet.getResultRows().get(0).get(1)); } @Test @@ -1136,35 +1174,35 @@ public Table getTable(String catalogName, String dbName, String tblName) { partitions.add("year"); partitions.add("dt"); HiveTable.Builder tableBuilder = HiveTable.builder() - .setId(1) - .setTableName("test_table") - .setCatalogName("hive_catalog") - .setResourceName(toResourceName("hive_catalog", "hive")) - .setHiveDbName("hive_db") - .setHiveTableName("test_table") - .setPartitionColumnNames(partitions) - .setFullSchema(fullSchema) - .setTableLocation("hdfs://hadoop/hive/warehouse/test.db/test") - .setCreateTime(10000) - .setHiveTableType(HiveTable.HiveTableType.EXTERNAL_TABLE); + .setId(1) + .setTableName("test_table") + .setCatalogName("hive_catalog") + .setResourceName(toResourceName("hive_catalog", "hive")) + .setHiveDbName("hive_db") + .setHiveTableName("test_table") + .setPartitionColumnNames(partitions) + .setFullSchema(fullSchema) + .setTableLocation("hdfs://hadoop/hive/warehouse/test.db/test") + .setCreateTime(10000) + .setHiveTableType(HiveTable.HiveTableType.EXTERNAL_TABLE); return tableBuilder.build(); } }; ShowCreateTableStmt stmt = new ShowCreateTableStmt(new TableName("hive_catalog", "hive_db", "test_table"), - ShowCreateTableStmt.CreateTableType.TABLE); + ShowCreateTableStmt.CreateTableType.TABLE); ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx); Assert.assertEquals("test_table", resultSet.getResultRows().get(0).get(0)); Assert.assertEquals("CREATE EXTERNAL TABLE `test_table` (\n" + - " `id` int(11) DEFAULT NULL COMMENT \"id\",\n" + - " `name` varchar DEFAULT NULL,\n" + - " `year` int(11) DEFAULT NULL,\n" + - " `dt` int(11) DEFAULT NULL\n" + - ")\n" + - "PARTITION BY (year, dt)\n" + - "PROPERTIES (\"location\" = \"hdfs://hadoop/hive/warehouse/test.db/test\");", - resultSet.getResultRows().get(0).get(1)); + " `id` int(11) DEFAULT NULL COMMENT \"id\",\n" + + " `name` varchar DEFAULT NULL,\n" + + " `year` int(11) DEFAULT NULL,\n" + + " `dt` int(11) DEFAULT NULL\n" + + ")\n" + + "PARTITION BY (year, dt)\n" + + "PROPERTIES (\"location\" = \"hdfs://hadoop/hive/warehouse/test.db/test\");", + resultSet.getResultRows().get(0).get(1)); } @Test @@ -1185,10 +1223,10 @@ public Catalog getCatalogByName(String name) { Assert.assertEquals("test_hive", resultSet.getResultRows().get(0).get(0)); Assert.assertEquals("CREATE EXTERNAL CATALOG `test_hive`\n" + - "comment \"hive_test\"\n" + - "PROPERTIES (\"type\" = \"hive\",\n" + - "\"hive.metastore.uris\" = \"thrift://hadoop:9083\"\n" + - ")", resultSet.getResultRows().get(0).get(1)); + "comment \"hive_test\"\n" + + "PROPERTIES (\"type\" = \"hive\",\n" + + "\"hive.metastore.uris\" = \"thrift://hadoop:9083\"\n" + + ")", resultSet.getResultRows().get(0).get(1)); } @Test @@ -1203,7 +1241,7 @@ public Catalog getCatalogByName(String name) { ShowCreateExternalCatalogStmt stmt = new ShowCreateExternalCatalogStmt("catalog_not_exist"); ExceptionChecker.expectThrowsWithMsg(SemanticException.class, "Unknown catalog 'catalog_not_exist'", - () -> ShowExecutor.execute(stmt, ctx)); + () -> ShowExecutor.execute(stmt, ctx)); } @Test @@ -1213,8 +1251,8 @@ public void testShowBasicStatsMeta() throws Exception { public Map getExternalBasicStatsMetaMap() { Map map = new HashMap<>(); map.put(new AnalyzeMgr.StatsMetaKey("hive0", "testDb", "testTable"), - new ExternalBasicStatsMeta("hive0", "testDb", "testTable", null, - StatsConstants.AnalyzeType.FULL, LocalDateTime.now(), Maps.newHashMap())); + new ExternalBasicStatsMeta("hive0", "testDb", "testTable", null, + StatsConstants.AnalyzeType.FULL, LocalDateTime.now(), Maps.newHashMap())); return map; } }; @@ -1235,13 +1273,13 @@ public void testShowGrants() throws Exception { ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx); resultSet.getResultRows().forEach(System.out::println); String expectString1 = "root, null, GRANT CREATE TABLE, DROP, ALTER, CREATE VIEW, CREATE FUNCTION, " + - "CREATE MATERIALIZED VIEW, CREATE PIPE ON ALL DATABASES TO ROLE 'root'"; + "CREATE MATERIALIZED VIEW, CREATE PIPE ON ALL DATABASES TO ROLE 'root'"; Assert.assertTrue(resultSet.getResultRows().stream().anyMatch(l -> - l.toString().contains(expectString1))); + l.toString().contains(expectString1))); String expectString2 = "root, null, GRANT DELETE, DROP, INSERT, SELECT, ALTER, EXPORT, " + - "UPDATE ON ALL TABLES IN ALL DATABASES TO ROLE 'root'"; + "UPDATE ON ALL TABLES IN ALL DATABASES TO ROLE 'root'"; Assert.assertTrue(resultSet.getResultRows().stream().anyMatch(l -> - l.toString().contains(expectString2))); + l.toString().contains(expectString2))); } @Test @@ -1264,12 +1302,12 @@ public Catalog getCatalogByName(String name) { Assert.assertEquals("test_hive", resultSet.getResultRows().get(0).get(0)); Assert.assertEquals("CREATE EXTERNAL CATALOG `test_hive`\n" + - "comment \"hive_test\"\n" + - "PROPERTIES (\"aws.s3.access_key\" = \"ia******ey\",\n" + - "\"aws.s3.secret_key\" = \"ia******ey\",\n" + - "\"hive.metastore.uris\" = \"thrift://hadoop:9083\",\n" + - "\"type\" = \"hive\"\n" + - ")", resultSet.getResultRows().get(0).get(1)); + "comment \"hive_test\"\n" + + "PROPERTIES (\"aws.s3.access_key\" = \"ia******ey\",\n" + + "\"aws.s3.secret_key\" = \"ia******ey\",\n" + + "\"hive.metastore.uris\" = \"thrift://hadoop:9083\",\n" + + "\"type\" = \"hive\"\n" + + ")", resultSet.getResultRows().get(0).get(1)); } @Test @@ -1282,7 +1320,7 @@ public void testShowDataCacheRules() throws DdlException, AnalysisException { properties.put("ni", "hao"); StringLiteral stringLiteral = new StringLiteral("hello"); dataCacheMgr.createCacheRule(QualifiedName.of(ImmutableList.of("test2", "test2", "test2")), - stringLiteral, -1, properties); + stringLiteral, -1, properties); ShowDataCacheRulesStmt stmt = new ShowDataCacheRulesStmt(NodePosition.ZERO); diff --git a/fe/fe-core/src/test/java/com/starrocks/qe/ShowTableMockMeta.java b/fe/fe-core/src/test/java/com/starrocks/qe/ShowTableMockMeta.java index 2858c4ae2c991..9cb52e455cdcc 100644 --- a/fe/fe-core/src/test/java/com/starrocks/qe/ShowTableMockMeta.java +++ b/fe/fe-core/src/test/java/com/starrocks/qe/ShowTableMockMeta.java @@ -130,17 +130,6 @@ public Table getTable(String catalogName, String dbName, String tblName) { return tableMap.get(tblName); } - @Override - public Table getTable(Long databaseId, Long tableId) { - for (Table table : tableMap.values()) { - if (table.getId() == tableId) { - return table; - } - } - - return null; - } - @Override public List listTableNames(String catalogName, String dbName) { if (catalogName.equals("hive_catalog")) { diff --git a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java index ef9e7de206ed8..7d31dda4894c5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationJobTest.java @@ -73,15 +73,16 @@ public static void beforeClass() throws Exception { starRocksAssert = new StarRocksAssert(AnalyzeTestUtil.getConnectContext()); starRocksAssert.withDatabase("test").useDatabase("test"); - db = GlobalStateMgr.getCurrentState().getDb("test"); + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String sql = "create table single_partition_duplicate_key (key1 int, key2 varchar(10))\n" + - "distributed by hash(key1) buckets 1\n" + - "properties('replication_num' = '1'); "; + "distributed by hash(key1) buckets 1\n" + + "properties('replication_num' = '1'); "; CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, - AnalyzeTestUtil.getConnectContext()); + AnalyzeTestUtil.getConnectContext()); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - table = (OlapTable) db.getTable("single_partition_duplicate_key"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "single_partition_duplicate_key"); srcTable = DeepCopy.copyWithGson(table, OlapTable.class); partition = table.getPartitions().iterator().next(); @@ -105,16 +106,16 @@ public void setUp() throws Exception { srcPartition.setNextDataVersion(99); job = new ReplicationJob(null, "test_token", db.getId(), table, srcTable, - GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); + GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); } @Test public void testJobId() { ReplicationJob jobWithoutId = new ReplicationJob(null, "test_token", db.getId(), table, srcTable, - GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); + GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); Assert.assertFalse(jobWithoutId.getJobId().isEmpty()); ReplicationJob jobWithId = new ReplicationJob("fake_id", "test_token", db.getId(), table, srcTable, - GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); + GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo()); Assert.assertEquals("fake_id", jobWithId.getJobId()); } @@ -140,7 +141,7 @@ public void testNormal() throws Exception { job.finishRemoteSnapshotTask((RemoteSnapshotTask) task, request); Deencapsulation.invoke(new LeaderImpl(), "finishRemoteSnapshotTask", - (RemoteSnapshotTask) task, request); + (RemoteSnapshotTask) task, request); ((RemoteSnapshotTask) task).toThrift(); task.toString(); } @@ -154,7 +155,7 @@ public void testNormal() throws Exception { job.finishReplicateSnapshotTask((ReplicateSnapshotTask) task, request); Deencapsulation.invoke(new LeaderImpl(), "finishReplicateSnapshotTask", - (ReplicateSnapshotTask) task, request); + (ReplicateSnapshotTask) task, request); ((ReplicateSnapshotTask) task).toThrift(); task.toString(); } @@ -346,7 +347,7 @@ public void testInitializedByThrift() { tabletInfo.replica_replication_infos = new ArrayList(); TReplicaReplicationInfo replicaInfo = new TReplicaReplicationInfo(); Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackends().iterator() - .next(); + .next(); replicaInfo.src_backend = new TBackend(backend.getHost(), backend.getBePort(), backend.getHttpPort()); tabletInfo.replica_replication_infos.add(replicaInfo); } diff --git a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java index 795e9d28f799d..fe327de3521f4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/replication/ReplicationMgrTest.java @@ -74,7 +74,7 @@ public static void beforeClass() throws Exception { starRocksAssert = new StarRocksAssert(AnalyzeTestUtil.getConnectContext()); starRocksAssert.withDatabase("test").useDatabase("test"); - db = GlobalStateMgr.getCurrentState().getDb("test"); + db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String sql = "create table single_partition_duplicate_key (key1 int, key2 varchar(10))\n" + "distributed by hash(key1) buckets 1\n" + @@ -83,7 +83,8 @@ public static void beforeClass() throws Exception { AnalyzeTestUtil.getConnectContext()); StarRocksAssert.utCreateTableWithRetry(createTableStmt); - table = (OlapTable) db.getTable("single_partition_duplicate_key"); + table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "single_partition_duplicate_key"); srcTable = DeepCopy.copyWithGson(table, OlapTable.class); partition = table.getPartitions().iterator().next(); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/MVRefreshTestBase.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/MVRefreshTestBase.java index 63e22ed180805..f07404bc265c0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/MVRefreshTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/MVRefreshTestBase.java @@ -97,8 +97,8 @@ public static void executeInsertSql(ConnectContext connectContext, String sql) t } protected MaterializedView getMv(String dbName, String mvName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(mvName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName); Assert.assertNotNull(table); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; @@ -106,8 +106,8 @@ protected MaterializedView getMv(String dbName, String mvName) { } protected Table getTable(String dbName, String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); Assert.assertNotNull(table); return table; } diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java index e8978fc654797..31e313001be19 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PCTRefreshListPartitionOlapTest.java @@ -53,75 +53,75 @@ public static void beforeClass() throws Exception { MVRefreshTestBase.beforeClass(); // table whose partitions have multiple values T1 = "CREATE TABLE t1 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (province) (\n" + - " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + - " PARTITION p2 VALUES IN (\"guangdong\") \n" + - ")\n" + - "DISTRIBUTED BY RANDOM\n"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\",\"chongqing\") ,\n" + + " PARTITION p2 VALUES IN (\"guangdong\") \n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n"; // table whose partitions have only single values T2 = "CREATE TABLE t2 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (province) (\n" + - " PARTITION p1 VALUES IN (\"beijing\") ,\n" + - " PARTITION p2 VALUES IN (\"guangdong\") \n" + - ")\n" + - "DISTRIBUTED BY RANDOM\n"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province) (\n" + + " PARTITION p1 VALUES IN (\"beijing\") ,\n" + + " PARTITION p2 VALUES IN (\"guangdong\") \n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n"; // table whose partitions have multi columns T3 = "CREATE TABLE t3 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10) not null,\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY LIST (province, dt) (\n" + - " PARTITION p1 VALUES IN ((\"beijing\", \"2024-01-01\")) ,\n" + - " PARTITION p2 VALUES IN ((\"guangdong\", \"2024-01-01\")), \n" + - " PARTITION p3 VALUES IN ((\"beijing\", \"2024-01-02\")) ,\n" + - " PARTITION p4 VALUES IN ((\"guangdong\", \"2024-01-02\")) \n" + - ")\n" + - "DISTRIBUTED BY RANDOM\n"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province, dt) (\n" + + " PARTITION p1 VALUES IN ((\"beijing\", \"2024-01-01\")) ,\n" + + " PARTITION p2 VALUES IN ((\"guangdong\", \"2024-01-01\")), \n" + + " PARTITION p3 VALUES IN ((\"beijing\", \"2024-01-02\")) ,\n" + + " PARTITION p4 VALUES IN ((\"guangdong\", \"2024-01-02\")) \n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n"; // table with partition expression whose partitions have multiple values T4 = "CREATE TABLE t4 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY (province) \n" + - "DISTRIBUTED BY RANDOM\n"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY (province) \n" + + "DISTRIBUTED BY RANDOM\n"; // table with partition expression whose partitions have multi columns T5 = "CREATE TABLE t5 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10) not null,\n" + - " province VARCHAR(64) not null\n" + - ")\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY (province, dt) \n" + - "DISTRIBUTED BY RANDOM\n"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY (province, dt) \n" + + "DISTRIBUTED BY RANDOM\n"; // table with partition expression whose partitions have multi columns(nullable partition columns) T6 = "CREATE TABLE t6 (\n" + - " id BIGINT,\n" + - " age SMALLINT,\n" + - " dt VARCHAR(10),\n" + - " province VARCHAR(64)\n" + - ")\n" + - "DUPLICATE KEY(id)\n" + - "PARTITION BY (province, dt) \n" + - "DISTRIBUTED BY RANDOM\n"; + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10),\n" + + " province VARCHAR(64)\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY (province, dt) \n" + + "DISTRIBUTED BY RANDOM\n"; } @AfterClass @@ -144,7 +144,7 @@ private ExecPlan getExecPlan(TaskRun taskRun) { initAndExecuteTaskRun(taskRun); PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) - taskRun.getProcessor(); + taskRun.getProcessor(); MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); return execPlan; @@ -178,12 +178,12 @@ private void addListPartition(String tbl, String pName, String pVal) { } private String toPartitionVal(String val) { - return val == null ? "NULL" : String.format("'%s'", val); + return val == null ? "NULL" : String.format("'%s'", val); } private void addListPartition(String tbl, String pName, String pVal1, String pVal2) { String addPartitionSql = String.format("ALTER TABLE %s ADD PARTITION %s VALUES IN ((%s, %s))", tbl, pName, - toPartitionVal(pVal1), toPartitionVal(pVal2)); + toPartitionVal(pVal1), toPartitionVal(pVal2)); StatementBase stmt = SqlParser.parseSingleStatement(addPartitionSql, connectContext.getSessionVariable().getSqlMode()); try { new StmtExecutor(connectContext, stmt).execute(); @@ -194,772 +194,794 @@ private void addListPartition(String tbl, String pName, String pVal1, String pVa @Test public void testRefreshNonPartitionedMV() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T2, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "as select dt, province, sum(age) from t2 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t2 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/2"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(1, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t2", "p3", "hangzhou"); - - String insertSql = "INSERT INTO t2 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=3/3"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(1, partitions.size()); - } - }); + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "as select dt, province, sum(age) from t2 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t2 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/2"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(1, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t2", "p3", "hangzhou"); + + String insertSql = "INSERT INTO t2 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=3/3"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(1, partitions.size()); + } + }); }); } @Test public void testRefreshSingleColumnMVWithSingleValues() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T2, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as select dt, province, sum(age) from t2 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t2 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t2", "p3", "hangzhou"); - - String insertSql = "INSERT INTO t2 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as select dt, province, sum(age) from t2 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t2 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t2", "p3", "hangzhou"); + + String insertSql = "INSERT INTO t2 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + }); }); } @Test public void testRefreshSingleColumnWithMultiValues() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T1, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as select dt, province, sum(age) from t1 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t1 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t1", "p3", "hangzhou"); - - String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as select dt, province, sum(age) from t1 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t1 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t1", "p3", "hangzhou"); + + String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + }); }); } @Test public void testRefreshMultiColumnsMV1() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T3, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as select dt, province, sum(age) from t3 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t3 partition(p1) values(1, 1, '2024-01-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " 0:OlapScanNode\n" + - " TABLE: t3\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/4"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t3 partition(p1) values(1, 1, '2024-01-01', 'beijing');"; - executeInsertSql(connectContext, insertSql); - insertSql = "insert into t3 partition(p3) values(1, 1, '2024-01-02', 'beijing');"; - executeInsertSql(connectContext, insertSql); - - ExecPlan execPlan = getExecPlan(taskRun); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t3\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/4"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t3", "p5", "hangzhou", "2022-01-01"); - String insertSql = "INSERT INTO t3 partition(p5) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t3\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/5"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as select dt, province, sum(age) from t3 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t3 partition(p1) values(1, 1, '2024-01-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " 0:OlapScanNode\n" + + " TABLE: t3\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/4"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t3 partition(p1) values(1, 1, '2024-01-01', 'beijing');"; + executeInsertSql(connectContext, insertSql); + insertSql = "insert into t3 partition(p3) values(1, 1, '2024-01-02', 'beijing');"; + executeInsertSql(connectContext, insertSql); + + ExecPlan execPlan = getExecPlan(taskRun); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t3\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/4"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t3", "p5", "hangzhou", "2022-01-01"); + String insertSql = "INSERT INTO t3 partition(p5) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t3\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/5"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + }); }); } @Test public void testRefreshMultiColumnsMV2() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T3, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by dt \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as select dt, province, sum(age) from t3 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t3 values (1, 1, '2024-01-01', 'beijing')," + - "(2, 20, '2024-01-01', 'guangdong'), (3, 30, '2024-01-02', 'guangdong');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t3\n" + - " PREAGGREGATION: ON\n" + - " partitions=4/4"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t3 partition(p1) values(1, 1, '2024-01-01', 'beijing');"; - executeInsertSql(connectContext, insertSql); - insertSql = "insert into t3 partition(p3) values(1, 1, '2024-01-02', 'beijing');"; - executeInsertSql(connectContext, insertSql); - - ExecPlan execPlan = getExecPlan(taskRun); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t3\n" + - " PREAGGREGATION: ON\n" + - " partitions=4/4"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t3", "p5", "hangzhou", "2024-01-01"); - String insertSql = "INSERT INTO t3 partition(p5) values(1, 1, '2024-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t3\n" + - " PREAGGREGATION: ON\n" + - " partitions=3/5"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - } - }); + "partition by dt \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as select dt, province, sum(age) from t3 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t3 values (1, 1, '2024-01-01', 'beijing')," + + "(2, 20, '2024-01-01', 'guangdong'), (3, 30, '2024-01-02', 'guangdong');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t3\n" + + " PREAGGREGATION: ON\n" + + " partitions=4/4"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t3 partition(p1) values(1, 1, '2024-01-01', 'beijing');"; + executeInsertSql(connectContext, insertSql); + insertSql = "insert into t3 partition(p3) values(1, 1, '2024-01-02', 'beijing');"; + executeInsertSql(connectContext, insertSql); + + ExecPlan execPlan = getExecPlan(taskRun); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t3\n" + + " PREAGGREGATION: ON\n" + + " partitions=4/4"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t3", "p5", "hangzhou", "2024-01-01"); + String insertSql = "INSERT INTO t3 partition(p5) values(1, 1, '2024-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t3\n" + + " PREAGGREGATION: ON\n" + + " partitions=3/5"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + } + }); }); } @Test public void testRefreshSingleColumnMVWithPartitionExpr() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T4, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as select dt, province, sum(age) from t4 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - addListPartition("t4", "p1", "beijing"); - String insertSql = "insert into t4 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t4\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(1, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t4", "p3", "hangzhou"); - - String insertSql = "INSERT INTO t4 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t4\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as select dt, province, sum(age) from t4 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + addListPartition("t4", "p1", "beijing"); + String insertSql = "insert into t4 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t4\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(1, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t4", "p3", "hangzhou"); + + String insertSql = "INSERT INTO t4 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t4\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + } + }); }); } @Test public void testRefreshMultiColumnsMVWithPartitionExpr() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T5, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as select dt, province, sum(age) from t5 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t5", "p1", "beijing", "2022-01-01"); - String insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t5\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(1, partitions.size()); - Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - executeInsertSql(connectContext, insertSql); - addListPartition("t5", "p2", "beijing", "2022-01-02"); - insertSql = "insert into t5 partition(p2) values(1, 1, '2021-12-02', 'beijing');"; - executeInsertSql(connectContext, insertSql); - - ExecPlan execPlan = getExecPlan(taskRun); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t5\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/3"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(1, partitions.size()); - Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t5", "p5", "hangzhou", "2022-01-01"); - String insertSql = "INSERT INTO t5 partition(p5) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t5\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/4"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as select dt, province, sum(age) from t5 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t5", "p1", "beijing", "2022-01-01"); + String insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t5\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(1, partitions.size()); + Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + executeInsertSql(connectContext, insertSql); + addListPartition("t5", "p2", "beijing", "2022-01-02"); + insertSql = "insert into t5 partition(p2) values(1, 1, '2021-12-02', 'beijing');"; + executeInsertSql(connectContext, insertSql); + + ExecPlan execPlan = getExecPlan(taskRun); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t5\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/3"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(1, partitions.size()); + Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t5", "p5", "hangzhou", "2022-01-01"); + String insertSql = "INSERT INTO t5 partition(p5) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t5\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/4"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + } + }); }); } @Test public void testRefreshMultiBaseTablesWithSingleColumn() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTables(ImmutableList.of(T2, T4), () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as " + - " select dt, province, sum(age) from t2 group by dt, province " + - " union all " + - " select dt, province, sum(age) from t4 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // only one table has updated - String insertSql = "insert into t2 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " 1:OlapScanNode\n" + - " TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2\n" + - " rollup: t2"); - PlanTestBase.assertContains(plan, " TABLE: t4\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: t4"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // t2 add a new partition - addListPartition("t2", "p3", "hangzhou"); - String insertSql = "INSERT INTO t2 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - executeInsertSql(connectContext, insertSql); - - // t4 add a new partition - addListPartition("t4", "p3", "hangzhou"); - insertSql = "INSERT INTO t4 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - executeInsertSql(connectContext, insertSql); - - ExecPlan execPlan = getExecPlan(taskRun); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); - PlanTestBase.assertContains(plan, " TABLE: t4\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as " + + " select dt, province, sum(age) from t2 group by dt, province " + + " union all " + + " select dt, province, sum(age) from t4 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // only one table has updated + String insertSql = "insert into t2 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " 1:OlapScanNode\n" + + " TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2\n" + + " rollup: t2"); + PlanTestBase.assertContains(plan, " TABLE: t4\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: t4"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // t2 add a new partition + addListPartition("t2", "p3", "hangzhou"); + String insertSql = "INSERT INTO t2 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + executeInsertSql(connectContext, insertSql); + + // t4 add a new partition + addListPartition("t4", "p3", "hangzhou"); + insertSql = "INSERT INTO t4 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + executeInsertSql(connectContext, insertSql); + + ExecPlan execPlan = getExecPlan(taskRun); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); + PlanTestBase.assertContains(plan, " TABLE: t4\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + }); }); } @Test public void testRefreshMultiBaseTablesWithMultiColumns() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTables(ImmutableList.of(T1, T5), () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1') \n" + - "as select dt, province, sum(age) from t1 group by dt, province \n" + - " union all\n" + - " select dt, province, sum(age) from t5 group by dt, province", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // only t1 has updated - String insertSql = "insert into t1 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2\n" + - " rollup: t1"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t1", "p3", "hangzhou"); - - String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); - PlanTestBase.assertContains(plan, " TABLE: t5\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: t5"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - - { - // t1 add a new partition - String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - executeInsertSql(connectContext, insertSql); - // t5 add a new partition - addListPartition("t5", "p1", "beijing", "2022-01-01"); - insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - executeInsertSql(connectContext, insertSql); - - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/3\n" + - " rollup: t1"); - PlanTestBase.assertContains(plan, " TABLE: t5\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1') \n" + + "as select dt, province, sum(age) from t1 group by dt, province \n" + + " union all\n" + + " select dt, province, sum(age) from t5 group by dt, province", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // only t1 has updated + String insertSql = "insert into t1 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2\n" + + " rollup: t1"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t1", "p3", "hangzhou"); + + String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); + PlanTestBase.assertContains(plan, " TABLE: t5\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: t5"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + + { + // t1 add a new partition + String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + executeInsertSql(connectContext, insertSql); + // t5 add a new partition + addListPartition("t5", "p1", "beijing", "2022-01-01"); + insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + executeInsertSql(connectContext, insertSql); + + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/3\n" + + " rollup: t1"); + PlanTestBase.assertContains(plan, " TABLE: t5\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + }); }); } @Test public void testRefreshJoinWithMultiColumns1() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTables(ImmutableList.of(T1, T5), () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1') \n" + - "as select t1.dt, t5.province, sum(t5.age) from t1 join t5 on t1.province=t5.province " + - "group by t1.dt, t5.province \n", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // only t1 has updated - String insertSql = "insert into t1 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2\n" + - " rollup: t1"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t1", "p3", "hangzhou"); - - String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); - PlanTestBase.assertContains(plan, " TABLE: t5\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: t5"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - - { - // t1 add a new partition - String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; - executeInsertSql(connectContext, insertSql); - // t5 add a new partition - addListPartition("t5", "p1", "beijing", "2022-01-01"); - insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - executeInsertSql(connectContext, insertSql); - - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/3"); - PlanTestBase.assertContains(plan, " TABLE: t5\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1') \n" + + "as select t1.dt, t5.province, sum(t5.age) from t1 join t5 on t1.province=t5.province " + + "group by t1.dt, t5.province \n", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // only t1 has updated + String insertSql = "insert into t1 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2\n" + + " rollup: t1"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t1", "p3", "hangzhou"); + + String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); + PlanTestBase.assertContains(plan, " TABLE: t5\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: t5"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + + { + // t1 add a new partition + String insertSql = "INSERT INTO t1 partition(p3) values(1, 1, '2022-01-01', 'hangzhou')"; + executeInsertSql(connectContext, insertSql); + // t5 add a new partition + addListPartition("t5", "p1", "beijing", "2022-01-01"); + insertSql = "insert into t5 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + executeInsertSql(connectContext, insertSql); + + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/3"); + PlanTestBase.assertContains(plan, " TABLE: t5\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + }); }); } @Test public void testRefreshMVWithMultiNulllalbeColumns() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable(T6, () -> { starRocksAssert.withMaterializedView("create materialized view mv1\n" + - "partition by province \n" + - "distributed by random \n" + - "REFRESH DEFERRED MANUAL \n" + - "properties ('partition_refresh_number' = '-1')" + - "as select dt, province, sum(age) from t6 group by dt, province;", - (obj) -> { - String mvName = (String) obj; - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - - { - // no partition has changed, no need to refresh - ExecPlan execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t6", "p1", "beijing", "2022-01-01"); - String insertSql = "insert into t6 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t6\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/2"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(1, partitions.size()); - Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - String insertSql = "insert into t6 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; - executeInsertSql(connectContext, insertSql); - addListPartition("t6", "p2", "beijing", "2022-01-02"); - insertSql = "insert into t6 partition(p2) values(1, 1, '2021-12-02', 'beijing');"; - executeInsertSql(connectContext, insertSql); - - ExecPlan execPlan = getExecPlan(taskRun); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t6\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/3"); - - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(1, partitions.size()); - Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); - // refresh again, refreshed partitions should not be refreshed again. - execPlan = getExecPlan(taskRun); - Assert.assertTrue(execPlan == null); - } - - { - // add a new partition - addListPartition("t6", "p5", "hangzhou", "2022-01-01"); - String insertSql = "INSERT INTO t6 partition(p5) values(1, 1, '2022-01-01', 'hangzhou')"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t6\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/4"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(2, partitions.size()); - } - - { - // add a null partition - addListPartition("t6", "p6", null, null); - String insertSql = "INSERT INTO t6 partition(p6) values(1, 1, NULL, NULL)"; - ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, " TABLE: t6\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/5"); - Collection partitions = materializedView.getPartitions(); - Assert.assertEquals(3, partitions.size()); - } - }); + "partition by province \n" + + "distributed by random \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties ('partition_refresh_number' = '-1')" + + "as select dt, province, sum(age) from t6 group by dt, province;", + (obj) -> { + String mvName = (String) obj; + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + + { + // no partition has changed, no need to refresh + ExecPlan execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t6", "p1", "beijing", "2022-01-01"); + String insertSql = "insert into t6 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t6\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/2"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(1, partitions.size()); + Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + String insertSql = "insert into t6 partition(p1) values(1, 1, '2021-12-01', 'beijing');"; + executeInsertSql(connectContext, insertSql); + addListPartition("t6", "p2", "beijing", "2022-01-02"); + insertSql = "insert into t6 partition(p2) values(1, 1, '2021-12-02', 'beijing');"; + executeInsertSql(connectContext, insertSql); + + ExecPlan execPlan = getExecPlan(taskRun); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t6\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/3"); + + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(1, partitions.size()); + Assert.assertTrue(partitions.iterator().next().getName().equals("p1")); + // refresh again, refreshed partitions should not be refreshed again. + execPlan = getExecPlan(taskRun); + Assert.assertTrue(execPlan == null); + } + + { + // add a new partition + addListPartition("t6", "p5", "hangzhou", "2022-01-01"); + String insertSql = "INSERT INTO t6 partition(p5) values(1, 1, '2022-01-01', 'hangzhou')"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t6\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/4"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(2, partitions.size()); + } + + { + // add a null partition + addListPartition("t6", "p6", null, null); + String insertSql = "INSERT INTO t6 partition(p6) values(1, 1, NULL, NULL)"; + ExecPlan execPlan = getExecPlanAfterInsert(taskRun, insertSql); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, " TABLE: t6\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/5"); + Collection partitions = materializedView.getPartitions(); + Assert.assertEquals(3, partitions.size()); + } + }); }); } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java index 5743bd650dde9..485895906e84c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorHiveTest.java @@ -149,8 +149,9 @@ public void testAutoRefreshPartitionLimitWithHiveTable() throws Exception { "\"storage_medium\" = \"HDD\"\n" + ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`lineitem_par` as a;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv1")); materializedView.getTableProperty().setAutoRefreshPartitionsLimit(2); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); @@ -194,8 +195,9 @@ public void testAutoRefreshPartitionLimitWithHiveTable() throws Exception { @Test public void testRefreshWithHiveTableJoin() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_join_mv")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_join_mv")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -229,8 +231,9 @@ public void testAutoPartitionRefreshWithPartitionChanged() throws Exception { "\"storage_medium\" = \"HDD\"\n" + ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`lineitem_par` as a;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -274,8 +277,9 @@ public void testAutoPartitionRefreshWithHiveTableJoin1() throws Exception { "AS SELECT t1.c1, t1.c2, t1_par.par_col, t1_par.par_date FROM `hive0`.`partitioned_db`.`t1` join " + "`hive0`.`partitioned_db`.`t1_par` using (par_col)"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_join_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_join_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -323,8 +327,9 @@ public void testAutoPartitionRefreshWithHiveTableJoin2() throws Exception { "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate`,`o_custkey` FROM `hive0`.`partitioned_db`.`lineitem_par` " + "as a join `hive0`.`tpch`.`orders` on l_orderkey = o_orderkey"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_join_mv2")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_join_mv2")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -371,8 +376,9 @@ public void testAutoPartitionRefreshWithUnPartitionedHiveTable() throws Exceptio ")\n" + "AS SELECT `n_nationkey`, `n_name`, `n_comment` FROM `hive0`.`tpch`.`nation`;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_tbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_tbl_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -414,8 +420,9 @@ public void testAutoPartitionRefreshWithPartitionedHiveTable1() throws Exception "\"storage_medium\" = \"HDD\"\n" + ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`lineitem_par` as a;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -464,8 +471,9 @@ public void testAutoPartitionRefreshWithPartitionedHiveTable2() throws Exception "\"storage_medium\" = \"HDD\"\n" + ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`lineitem_par` as a;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_tbl_mv2")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_tbl_mv2")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -509,8 +517,9 @@ public void testAutoPartitionRefreshWithPartitionedHiveTableJoinInternalTable() ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`lineitem_par` as a" + " join test.tbl1 b on a.l_suppkey=b.k2;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_join_internal_mv")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_join_internal_mv")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -554,8 +563,9 @@ public void testPartitionRefreshWithUpperCaseTable() throws Exception { ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`LINEITEM_PAR` as " + "`LINEITEM_PAR_ALIAS`;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -604,8 +614,9 @@ public void testPartitionRefreshWithUpperCaseDb() throws Exception { ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_DB`.`LINEITEM_PAR` as " + "`LINEITEM_PAR_ALIAS`;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -655,8 +666,9 @@ public void testPartitionRefreshWithLowerCase() throws Exception { "\"storage_medium\" = \"HDD\"\n" + ")\n" + "AS SELECT c1, c2, par_col FROM `hive0`.`partitioned_db2`.`t2`;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -691,8 +703,9 @@ public void testPartitionRefreshWithLowerCase() throws Exception { @Test public void testRangePartitionRefreshWithHiveTable() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv")); HashMap taskRunProperties = new HashMap<>(); taskRunProperties.put(PARTITION_START, "1998-01-01"); taskRunProperties.put(TaskRun.PARTITION_END, "1998-01-03"); @@ -718,8 +731,9 @@ public void testRangePartitionRefreshWithHiveTable() throws Exception { @Test public void testRefreshPartitionWithMulParColumnsHiveTable1() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_mul_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_mul_parttbl_mv1")); Map mvProperties = Maps.newHashMap(); mvProperties.put(PARTITION_START, "1998-01-01"); mvProperties.put(PARTITION_END, "1998-01-03"); @@ -740,8 +754,9 @@ public void testRefreshPartitionWithMulParColumnsHiveTable1() throws Exception { @Test public void testRefreshPartitionWithMulParColumnsHiveTable2() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_mul_parttbl_mv2")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_mul_parttbl_mv2")); Map mvProperties = Maps.newHashMap(); mvProperties.put(PARTITION_START, "2020-01-01"); mvProperties.put(PARTITION_END, "2020-01-03"); @@ -762,7 +777,7 @@ public void testRefreshPartitionWithMulParColumnsHiveTable2() throws Exception { @Test public void testHivePartitionPruneNonRefBaseTable1() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withMaterializedView( "CREATE MATERIALIZED VIEW `test`.`hive_partition_prune_non_ref_tables2`\n" + "COMMENT \"MATERIALIZED_VIEW\"\n" + @@ -778,7 +793,8 @@ public void testHivePartitionPruneNonRefBaseTable1() throws Exception { "`hive0`.`partitioned_db`.`part_tbl2` using (par_date)"); MaterializedView materializedView = - ((MaterializedView) testDb.getTable("hive_partition_prune_non_ref_tables2")); + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_partition_prune_non_ref_tables2")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -891,7 +907,7 @@ public void testHivePartitionPruneNonRefBaseTable1() throws Exception { @Test public void testHivePartitionPruneNonRefBaseTable2() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withMaterializedView( "CREATE MATERIALIZED VIEW `test`.`hive_partition_prune_non_ref_tables1`\n" + "COMMENT \"MATERIALIZED_VIEW\"\n" + @@ -907,7 +923,8 @@ public void testHivePartitionPruneNonRefBaseTable2() throws Exception { "`hive0`.`partitioned_db`.`t1_par` using (par_col)"); MaterializedView materializedView = - ((MaterializedView) testDb.getTable("hive_partition_prune_non_ref_tables1")); + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_partition_prune_non_ref_tables1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -1000,7 +1017,7 @@ public void testHivePartitionPruneNonRefBaseTable2() throws Exception { @Test public void testHivePartitionPruneNonRefBaseTable3() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withTable("CREATE TABLE `test_partition_prune_tbl1` (\n" + "`k1` date,\n" + "`k2` int,\n" + @@ -1034,7 +1051,8 @@ public void testHivePartitionPruneNonRefBaseTable3() throws Exception { " FROM test_partition_prune_tbl1 join test_partition_prune_tbl2 on " + " test_partition_prune_tbl1.k1=test_partition_prune_tbl2.k1;"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("partition_prune_mv1")); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "partition_prune_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); @@ -1118,8 +1136,9 @@ public void testCancelRefreshMV() throws Exception { "\"storage_medium\" = \"HDD\"\n" + ")\n" + "AS SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`lineitem_par` as a;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("hive_parttbl_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "hive_parttbl_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -1143,8 +1162,9 @@ public void testDropBaseVersionMetaOfExternalTable() throws Exception { "refresh async every (interval 1 day)\n" + "as SELECT `l_orderkey`, `l_suppkey`, `l_shipdate` FROM `hive0`.`partitioned_db`.`lineitem_par` as a;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable("test_drop_partition_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_drop_partition_mv1")); Map> versionMap = mv.getRefreshScheme().getAsyncRefreshContext().getBaseTableInfoVisibleVersionMap(); Map> mvPartitionNameRefBaseTablePartitionMap = diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java index 3162cf54b58c6..5faed2b0f4406 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorIcebergTest.java @@ -68,7 +68,7 @@ public void after() throws Exception { } private static void triggerRefreshMv(Database testDb, MaterializedView partitionedMaterializedView) - throws Exception { + throws Exception { Task task = TaskBuilder.buildMvTask(partitionedMaterializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); initAndExecuteTaskRun(taskRun); @@ -77,31 +77,32 @@ private static void triggerRefreshMv(Database testDb, MaterializedView partition @Test public void testCreateNonPartitionedMVForIceberg() throws Exception { starRocksAssert.useDatabase("test") - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_mv1` " + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT id, data, date FROM `iceberg0`.`unpartitioned_db`.`t0` as a;") - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_mv2` " + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1` as a;"); + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_mv1` " + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT id, data, date FROM `iceberg0`.`unpartitioned_db`.`t0` as a;") + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_mv2` " + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1` as a;"); // Partitioned base table { String mvName = "iceberg_mv2"; - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); refreshMVRange(mvName, true); List partitionNames = mv.getPartitions().stream().map(Partition::getName) - .sorted().collect(Collectors.toList()); + .sorted().collect(Collectors.toList()); Assert.assertEquals(ImmutableList.of(mvName), partitionNames); String querySql = "SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1`"; starRocksAssert.query(querySql).explainContains(mvName); @@ -111,11 +112,12 @@ public void testCreateNonPartitionedMVForIceberg() throws Exception { // Non-Partitioned base table { String mvName = "iceberg_mv1"; - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); refreshMVRange(mvName, true); List partitionNames = mv.getPartitions().stream().map(Partition::getName) - .sorted().collect(Collectors.toList()); + .sorted().collect(Collectors.toList()); Assert.assertEquals(ImmutableList.of(mvName), partitionNames); // test rewrite @@ -129,55 +131,56 @@ public void testCreateNonPartitionedMVForIceberg() throws Exception { public void testCreatePartitionedMVForIceberg() throws Exception { String mvName = "iceberg_parttbl_mv1"; starRocksAssert.useDatabase("test") - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_parttbl_mv1`\n" + - "PARTITION BY str2date(`date`, '%Y-%m-%d')\n" + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1` as a;"); - - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView partitionedMaterializedView = ((MaterializedView) testDb.getTable("iceberg_parttbl_mv1")); + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_parttbl_mv1`\n" + + "PARTITION BY str2date(`date`, '%Y-%m-%d')\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1` as a;"); + + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView partitionedMaterializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "iceberg_parttbl_mv1")); triggerRefreshMv(testDb, partitionedMaterializedView); Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(4, partitions.size()); MockIcebergMetadata mockIcebergMetadata = - (MockIcebergMetadata) connectContext.getGlobalStateMgr().getMetadataMgr(). - getOptionalMetadata(MockIcebergMetadata.MOCKED_ICEBERG_CATALOG_NAME).get(); + (MockIcebergMetadata) connectContext.getGlobalStateMgr().getMetadataMgr(). + getOptionalMetadata(MockIcebergMetadata.MOCKED_ICEBERG_CATALOG_NAME).get(); mockIcebergMetadata.updatePartitions("partitioned_db", "t1", - ImmutableList.of("date=2020-01-02")); + ImmutableList.of("date=2020-01-02")); // refresh only one partition Task task = TaskBuilder.buildMvTask(partitionedMaterializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); initAndExecuteTaskRun(taskRun); PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) - taskRun.getProcessor(); + taskRun.getProcessor(); MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); assertPlanContains(execPlan, "3: date >= '2020-01-02', 3: date < '2020-01-03'"); Map partitionVersionMap = partitionedMaterializedView.getPartitions().stream() - .collect(Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); + .collect(Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); Assert.assertEquals( - ImmutableMap.of("p20200104_20200105", 2L, - "p20200101_20200102", 2L, - "p20200103_20200104", 2L, - "p20200102_20200103", 3L), - ImmutableMap.copyOf(partitionVersionMap)); + ImmutableMap.of("p20200104_20200105", 2L, + "p20200101_20200102", 2L, + "p20200103_20200104", 2L, + "p20200102_20200103", 3L), + ImmutableMap.copyOf(partitionVersionMap)); // add new row and refresh again mockIcebergMetadata.updatePartitions("partitioned_db", "t1", - ImmutableList.of("date=2020-01-01")); + ImmutableList.of("date=2020-01-01")); taskRun = TaskRunBuilder.newBuilder(task).build(); initAndExecuteTaskRun(taskRun); processor = (PartitionBasedMvRefreshProcessor) - taskRun.getProcessor(); + taskRun.getProcessor(); mvContext = processor.getMvContext(); execPlan = mvContext.getExecPlan(); @@ -185,9 +188,9 @@ public void testCreatePartitionedMVForIceberg() throws Exception { // test rewrite starRocksAssert.query("SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1`") - .explainContains(mvName); + .explainContains(mvName); starRocksAssert.query("SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1` where date = '2020-01-01'") - .explainContains(mvName); + .explainContains(mvName); starRocksAssert.dropMaterializedView(mvName); } @@ -197,37 +200,39 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform() throws Exc { String mvName = "iceberg_year_mv1"; starRocksAssert.useDatabase("test") - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_year_mv1`\n" + - "PARTITION BY ts\n" + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_year` as a;"); - - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView partitionedMaterializedView = ((MaterializedView) testDb.getTable("iceberg_year_mv1")); + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_year_mv1`\n" + + "PARTITION BY ts\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_year` as a;"); + + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView partitionedMaterializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "iceberg_year_mv1")); triggerRefreshMv(testDb, partitionedMaterializedView); Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); List partitionNames = ImmutableList.of("p20190101000000", "p20200101000000", "p20210101000000", - "p20220101000000", "p20230101000000"); + "p20220101000000", "p20230101000000"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); MockIcebergMetadata mockIcebergMetadata = - (MockIcebergMetadata) connectContext.getGlobalStateMgr().getMetadataMgr(). - getOptionalMetadata(MockIcebergMetadata.MOCKED_ICEBERG_CATALOG_NAME).get(); + (MockIcebergMetadata) connectContext.getGlobalStateMgr().getMetadataMgr(). + getOptionalMetadata(MockIcebergMetadata.MOCKED_ICEBERG_CATALOG_NAME).get(); mockIcebergMetadata.updatePartitions("partitioned_transforms_db", "t0_year", - ImmutableList.of("ts_year=2020")); + ImmutableList.of("ts_year=2020")); // refresh only one partition Task task = TaskBuilder.buildMvTask(partitionedMaterializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); initAndExecuteTaskRun(taskRun); PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) - taskRun.getProcessor(); + taskRun.getProcessor(); MvTaskRunContext mvContext = processor.getMvContext(); ExecPlan execPlan = mvContext.getExecPlan(); @@ -235,91 +240,97 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform() throws Exc // test rewrite starRocksAssert.query("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_year`") - .explainContains(mvName); + .explainContains(mvName); starRocksAssert.dropMaterializedView(mvName); } // test partition by month(ts) { String mvName = "iceberg_month_mv1"; starRocksAssert.useDatabase("test") - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_month_mv1`\n" + - "PARTITION BY ts\n" + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_month` as a;"); - - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView partitionedMaterializedView = ((MaterializedView) testDb.getTable("iceberg_month_mv1")); + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_month_mv1`\n" + + "PARTITION BY ts\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_month` as a;"); + + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView partitionedMaterializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "iceberg_month_mv1")); triggerRefreshMv(testDb, partitionedMaterializedView); Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); List partitionNames = ImmutableList.of("p20220101000000", "p20220201000000", "p20220301000000", - "p20220401000000", "p20220501000000"); + "p20220401000000", "p20220501000000"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); // test rewrite starRocksAssert.query("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_month`") - .explainContains(mvName); + .explainContains(mvName); starRocksAssert.dropMaterializedView(mvName); } // test partition by day(ts) { String mvName = "iceberg_day_mv1"; starRocksAssert.useDatabase("test") - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_day_mv1`\n" + - "PARTITION BY ts\n" + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_day` as a;"); - - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView partitionedMaterializedView = ((MaterializedView) testDb.getTable("iceberg_day_mv1")); + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_day_mv1`\n" + + "PARTITION BY ts\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_day` as a;"); + + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView partitionedMaterializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "iceberg_day_mv1")); triggerRefreshMv(testDb, partitionedMaterializedView); Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); List partitionNames = ImmutableList.of("p20220101000000", "p20220102000000", "p20220103000000", - "p20220104000000", "p20220105000000"); + "p20220104000000", "p20220105000000"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); // test rewrite starRocksAssert.query("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_day`") - .explainContains(mvName); + .explainContains(mvName); starRocksAssert.dropMaterializedView(mvName); } // test partition by hour(ts) { String mvName = "iceberg_hour_mv1"; starRocksAssert.useDatabase("test") - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_hour_mv1`\n" + - "PARTITION BY ts\n" + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_hour` as a;"); - - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView partitionedMaterializedView = ((MaterializedView) testDb.getTable("iceberg_hour_mv1")); + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`iceberg_hour_mv1`\n" + + "PARTITION BY ts\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_hour` as a;"); + + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView partitionedMaterializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "iceberg_hour_mv1")); triggerRefreshMv(testDb, partitionedMaterializedView); Collection partitions = partitionedMaterializedView.getPartitions(); Assert.assertEquals(5, partitions.size()); List partitionNames = ImmutableList.of("p20220101000000", "p20220101010000", "p20220101020000", - "p20220101030000", "p20220101040000"); + "p20220101030000", "p20220101040000"); Assert.assertTrue(partitions.stream().map(Partition::getName).allMatch(partitionNames::contains)); // test rewrite starRocksAssert.query("SELECT id, data, ts FROM `iceberg0`.`partitioned_transforms_db`.`t0_hour`") - .explainContains(mvName); + .explainContains(mvName); starRocksAssert.dropMaterializedView(mvName); } } @@ -327,27 +338,27 @@ public void testCreatePartitionedMVForIcebergWithPartitionTransform() throws Exc @Test public void testRefreshWithCachePartitionTraits() { starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `test_mv1`\n" + - "PARTITION BY str2date(`date`, '%Y-%m-%d')\n" + - "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "AS SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1` as a;", - () -> { - MaterializedView mv = getMv("test", "test_mv1"); - PartitionBasedMvRefreshProcessor processor = refreshMV("test", mv); - RuntimeProfile runtimeProfile = processor.getRuntimeProfile(); - QueryMaterializationContext.QueryCacheStats queryCacheStats = getQueryCacheStats(runtimeProfile); - Assert.assertTrue(queryCacheStats != null); - queryCacheStats.getCounter().forEach((key, value) -> { - if (key.contains("cache_partitionNames")) { - Assert.assertEquals(1L, value.longValue()); - } else if (key.contains("cache_getPartitionKeyRange")) { - Assert.assertEquals(3L, value.longValue()); - } else { - Assert.assertEquals(1L, value.longValue()); - } + "PARTITION BY str2date(`date`, '%Y-%m-%d')\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "AS SELECT id, data, date FROM `iceberg0`.`partitioned_db`.`t1` as a;", + () -> { + MaterializedView mv = getMv("test", "test_mv1"); + PartitionBasedMvRefreshProcessor processor = refreshMV("test", mv); + RuntimeProfile runtimeProfile = processor.getRuntimeProfile(); + QueryMaterializationContext.QueryCacheStats queryCacheStats = getQueryCacheStats(runtimeProfile); + Assert.assertTrue(queryCacheStats != null); + queryCacheStats.getCounter().forEach((key, value) -> { + if (key.contains("cache_partitionNames")) { + Assert.assertEquals(1L, value.longValue()); + } else if (key.contains("cache_getPartitionKeyRange")) { + Assert.assertEquals(3L, value.longValue()); + } else { + Assert.assertEquals(1L, value.longValue()); + } + }); + Set partitionsToRefresh1 = getPartitionNamesToRefreshForMv(mv); + Assert.assertTrue(partitionsToRefresh1.isEmpty()); }); - Set partitionsToRefresh1 = getPartitionNamesToRefreshForMv(mv); - Assert.assertTrue(partitionsToRefresh1.isEmpty()); - }); } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java index 20bdf39a6bbe5..1e2118d3c9e41 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorJdbcTest.java @@ -54,57 +54,57 @@ public static void beforeClass() throws Exception { MVRefreshTestBase.beforeClass(); ConnectorPlanTestBase.mockCatalog(connectContext, MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME); starRocksAssert - .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`jdbc_parttbl_mv0`\n" + - "COMMENT \"MATERIALIZED_VIEW\"\n" + - "PARTITION BY (`d`)\n" + - "DISTRIBUTED BY HASH(`a`) BUCKETS 10\n" + - "REFRESH DEFERRED MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT `a`, `b`, `c`, `d` FROM `jdbc0`.`partitioned_db0`.`tbl0`;") - .withMaterializedView("create materialized view jdbc_parttbl_mv1 " + - "partition by ss " + - "distributed by hash(a) buckets 10 " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl1;") - .withMaterializedView("create materialized view jdbc_parttbl_mv2 " + - "partition by ss " + - "distributed by hash(a) buckets 10 " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl2;") - .withMaterializedView("create materialized view jdbc_parttbl_mv3 " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) buckets 10 " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select a, b, c, d from jdbc0.partitioned_db0.tbl1;") - .withMaterializedView("create materialized view jdbc_parttbl_mv5 " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) buckets 10 " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ") " + - "as select a, b, c, d from jdbc0.partitioned_db0.tbl3;") - .withMaterializedView("create materialized view jdbc_parttbl_mv6 " + - "partition by ss " + - "distributed by hash(a) buckets 10 " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"partition_refresh_number\" = \"1\"" + - ") " + - "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl1;"); + .withMaterializedView("CREATE MATERIALIZED VIEW `test`.`jdbc_parttbl_mv0`\n" + + "COMMENT \"MATERIALIZED_VIEW\"\n" + + "PARTITION BY (`d`)\n" + + "DISTRIBUTED BY HASH(`a`) BUCKETS 10\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT `a`, `b`, `c`, `d` FROM `jdbc0`.`partitioned_db0`.`tbl0`;") + .withMaterializedView("create materialized view jdbc_parttbl_mv1 " + + "partition by ss " + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl1;") + .withMaterializedView("create materialized view jdbc_parttbl_mv2 " + + "partition by ss " + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl2;") + .withMaterializedView("create materialized view jdbc_parttbl_mv3 " + + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select a, b, c, d from jdbc0.partitioned_db0.tbl1;") + .withMaterializedView("create materialized view jdbc_parttbl_mv5 " + + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select a, b, c, d from jdbc0.partitioned_db0.tbl3;") + .withMaterializedView("create materialized view jdbc_parttbl_mv6 " + + "partition by ss " + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"partition_refresh_number\" = \"1\"" + + ") " + + "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl1;"); } @AfterClass @@ -137,24 +137,24 @@ public void testJDBCProtocolType() { public void testPartitionJDBCSupported() throws Exception { // not supported Assert.assertThrows(AnalysisException.class, () -> - starRocksAssert.withMaterializedView("create materialized view mv_jdbc_postgres " + - "partition by d " + - "refresh deferred manual " + - "AS SELECT `a`, `b`, `c`, `d` FROM `jdbc_postgres`.`partitioned_db0`.`tbl0`;") + starRocksAssert.withMaterializedView("create materialized view mv_jdbc_postgres " + + "partition by d " + + "refresh deferred manual " + + "AS SELECT `a`, `b`, `c`, `d` FROM `jdbc_postgres`.`partitioned_db0`.`tbl0`;") ); // supported starRocksAssert.withMaterializedView("create materialized view mv_jdbc_mysql " + - "partition by d " + - "refresh deferred manual " + - "AS SELECT `a`, `b`, `c`, `d` FROM `jdbc0`.`partitioned_db0`.`tbl0`;"); + "partition by d " + + "refresh deferred manual " + + "AS SELECT `a`, `b`, `c`, `d` FROM `jdbc0`.`partitioned_db0`.`tbl0`;"); } @Test public void testRangePartitionChangeWithJDBCTable() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); MaterializedView materializedView = refreshMaterializedView("jdbc_parttbl_mv0", "20230801", "20230805"); @@ -169,8 +169,9 @@ public void testRangePartitionChangeWithJDBCTable() throws Exception { @NotNull private MaterializedView refreshMaterializedView(String materializedViewName, String start, String end) throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(materializedViewName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), materializedViewName)); refreshMVRange(materializedView.getName(), start, end, false); return materializedView; } @@ -179,15 +180,15 @@ private MaterializedView refreshMaterializedView(String materializedViewName, St public void testRangePartitionWithJDBCTableUseStr2Date() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); MaterializedView materializedView = refreshMaterializedView("jdbc_parttbl_mv1", "20230731", "20230805"); List partitions = materializedView.getPartitions().stream() - .map(Partition::getName).sorted().collect(Collectors.toList()); + .map(Partition::getName).sorted().collect(Collectors.toList()); Assert.assertEquals(ImmutableList.of("p00010101_20230801", "p20230801_20230802", - "p20230802_20230803", "p20230803_99991231"), - partitions); + "p20230802_20230803", "p20230803_99991231"), + partitions); } @Test @@ -203,74 +204,75 @@ public void testRangePartitionWithJDBCTableUseStr2DateForError() { public void testRangePartitionWithJDBCTableUseStr2Date2() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); MaterializedView materializedView = refreshMaterializedView("jdbc_parttbl_mv3", "20230731", "20230805"); List partitions = materializedView.getPartitions().stream() - .map(Partition::getName).sorted().collect(Collectors.toList()); + .map(Partition::getName).sorted().collect(Collectors.toList()); Assert.assertEquals(ImmutableList.of("p00010101_20230801", "p20230801_20230802", - "p20230802_20230803", "p20230803_99991231"), - partitions); + "p20230802_20230803", "p20230803_99991231"), + partitions); } @Test public void testStr2Date_DateTrunc() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); String mvName = "jdbc_parttbl_str2date"; starRocksAssert.withMaterializedView("create materialized view jdbc_parttbl_str2date " + - "partition by date_trunc('month', ss) " + - "distributed by hash(a) buckets 10 " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"partition_refresh_number\" = \"1\"" + - ") " + - "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl5;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by date_trunc('month', ss) " + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"partition_refresh_number\" = \"1\"" + + ") " + + "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl5;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // full refresh { starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p000101_202308", "p202308_202309"), partitions); } // partial range refresh 1 { Map partitionVersionMap = materializedView.getPartitions().stream().collect( - Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); + Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + - " partition start('2023-08-02') end('2023-09-01')" + - "force with sync mode"); + " partition start('2023-08-02') end('2023-09-01')" + + "force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p000101_202308", "p202308_202309"), partitions); Assert.assertEquals(partitionVersionMap.get("p202308_202309").longValue(), - materializedView.getPartition("p202308_202309").getVisibleVersion()); + materializedView.getPartition("p202308_202309").getVisibleVersion()); } // partial range refresh 2 { Map partitionVersionMap = materializedView.getPartitions().stream().collect( - Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); + Collectors.toMap(Partition::getName, Partition::getVisibleVersion)); starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + - " partition start('2023-07-01') end('2023-08-01')" + - "force with sync mode"); + " partition start('2023-07-01') end('2023-08-01')" + + "force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p000101_202308", "p202308_202309"), partitions); Assert.assertEquals(partitionVersionMap.get("p202308_202309").longValue(), - materializedView.getPartition("p202308_202309").getVisibleVersion()); + materializedView.getPartition("p202308_202309").getVisibleVersion()); } starRocksAssert.dropMaterializedView(mvName); @@ -280,40 +282,41 @@ public void testStr2Date_DateTrunc() throws Exception { public void testStr2Date_TTL() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); String mvName = "jdbc_parttbl_str2date"; starRocksAssert.withMaterializedView("create materialized view jdbc_parttbl_str2date " + - "partition by ss " + - "distributed by hash(a) buckets 10 " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"partition_refresh_number\" = \"1\"," + - "'partition_ttl_number'='2'" + - ") " + - "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl1;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by ss " + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"partition_refresh_number\" = \"1\"," + + "'partition_ttl_number'='2'" + + ") " + + "as select str2date(d,'%Y%m%d') ss, a, b, c from jdbc0.partitioned_db0.tbl1;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230802_20230803", "p20230803_99991231"), partitions); // modify TTL { starRocksAssert.getCtx().executeSql( - String.format("alter materialized view %s set ('partition_ttl_number'='1')", mvName)); + String.format("alter materialized view %s set ('partition_ttl_number'='1')", mvName)); starRocksAssert.getCtx().executeSql(String.format("refresh materialized view %s with sync mode", mvName)); GlobalStateMgr.getCurrentState().getDynamicPartitionScheduler().runOnceForTest(); partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230803_99991231"), partitions); } starRocksAssert.dropMaterializedView(mvName); @@ -323,36 +326,37 @@ public void testStr2Date_TTL() throws Exception { public void testRangePartitionWithJDBCTableUseStr2Date3() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); MaterializedView materializedView = refreshMaterializedView("jdbc_parttbl_mv5", "20230731", "20230805"); List partitions = materializedView.getPartitions().stream() - .map(Partition::getName).sorted().collect(Collectors.toList()); + .map(Partition::getName).sorted().collect(Collectors.toList()); Assert.assertEquals(ImmutableList.of("p00010101_20230801", "p20230801_20230802", - "p20230802_20230803", "p20230803_20230804"), - partitions); + "p20230802_20230803", "p20230803_20230804"), + partitions); } @Test public void testRefreshByParCreateOnlyNecessaryPar() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); // get base table partitions List baseParNames = mockedJDBCMetadata.listPartitionNames("partitioned_db0", "tbl1", TableVersionRange.empty()); Assert.assertEquals(4, baseParNames.size()); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("jdbc_parttbl_mv6")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "jdbc_parttbl_mv6")); HashMap taskRunProperties = new HashMap<>(); // check corner case: the first partition of base table is 0000 to 20230801 // p20230801 of mv should not be created refreshMVRange(materializedView.getName(), "20230801", "20230802", false); List partitions = materializedView.getPartitions().stream() - .map(Partition::getName).sorted().collect(Collectors.toList()); + .map(Partition::getName).sorted().collect(Collectors.toList()); Assert.assertEquals(ImmutableList.of("p20230801_20230802"), partitions); } @@ -360,31 +364,32 @@ public void testRefreshByParCreateOnlyNecessaryPar() throws Exception { public void testStr2DateMVRefresh_Rewrite() throws Exception { MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p00010101_20230801", "p20230801_20230802", "p20230802_20230803", - "p20230803_99991231"), partitions); + "p20230803_99991231"), partitions); starRocksAssert.dropMaterializedView(mvName); } diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java index c40d1d0853da7..5ab27a72662c2 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java @@ -108,13 +108,15 @@ public void testMVRefreshWithTheSameTables1() { String mvName = (String) obj; assertPlanWithoutPushdownBelowScan(mvName); }); - }; + } + ; }); } private void assertPlanWithoutPushdownBelowScan(String mvName) throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); Map testProperties = task.getProperties(); @@ -188,13 +190,15 @@ public void testMVRefreshWithTheSameTables22() { String mvName = (String) obj; assertPlanWithPushdownBelowScan(mvName); }); - }; + } + ; }); } private void assertPlanWithPushdownBelowScan(String mvName) throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); Map testProperties = task.getProperties(); @@ -268,7 +272,7 @@ public void testRefreshWithCachePartitionTraits() throws Exception { executeInsertSql(connectContext, "insert into tbl1 values(\"2022-02-20\", 2, 10)"); Partition p2 = table.getPartition("p2"); - while (p2.getVisibleVersion() != 3) { + while (p2.getVisibleVersion() != 3) { System.out.println("waiting for partition p2 to be visible:" + p2.getVisibleVersion()); Thread.sleep(1000); } @@ -318,7 +322,7 @@ public void testTaskRun() { "properties('replication_num' = '1', 'partition_refresh_number'='1')\n" + "as select k1, k2 from tbl6;"); String mvName = "mv_refresh_priority"; - Database testDb = GlobalStateMgr.getCurrentState().getDb(TEST_DB_NAME); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(TEST_DB_NAME); MaterializedView mv = ((MaterializedView) testDb.getTable(mvName)); TaskManager tm = GlobalStateMgr.getCurrentState().getTaskManager(); @@ -384,7 +388,7 @@ public void testRefreshPriority() { "refresh deferred manual\n" + "properties('replication_num' = '1', 'partition_refresh_number'='1')\n" + "as select k1, k2 from tbl6;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); MaterializedView mv = ((MaterializedView) testDb.getTable(mvName)); TaskManager tm = GlobalStateMgr.getCurrentState().getTaskManager(); long taskId = tm.getTask(TaskBuilder.getMvTaskName(mv.getId())).getId(); @@ -440,7 +444,7 @@ public void testMVRefreshProperties() { ")\n" + "as select k1, k2 from tbl6;", () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); MaterializedView mv = ((MaterializedView) testDb.getTable(mvName)); executeInsertSql(connectContext, "insert into tbl6 partition(p1) values('2022-01-02',2,10);"); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java index 1bd620fa1c63a..249275dd93e63 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapTest.java @@ -261,8 +261,9 @@ protected void assertPlanContains(ExecPlan execPlan, String... explain) throws E @Test public void testUnionAllMvWithPartition() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("union_all_mv")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "union_all_mv")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); try { @@ -287,8 +288,9 @@ public void testUnionAllMvWithPartition() { @Test public void testWithPartition() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); try { @@ -352,8 +354,9 @@ public void testWithPartition() { @Test public void testMvWithoutPartition() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_without_partition")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_without_partition")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); try { @@ -420,8 +423,9 @@ public void testRewriteNonPartitionedMVForOlapTable() throws Exception { @NotNull private MaterializedView refreshMaterializedView(String materializedViewName, String start, String end) throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(materializedViewName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), materializedViewName)); refreshMVRange(materializedView.getName(), start, end, false); return materializedView; } @@ -435,8 +439,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { taskRunCounter.incrementAndGet(); } }; - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_without_partition")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_without_partition")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -456,8 +461,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { @Test public void testMVClearQueryInfo() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_without_partition")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_without_partition")); new MockUp() { @Mock public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { @@ -488,7 +494,8 @@ private void testBaseTablePartitionInsertData(Database testDb, MaterializedView insertSql = "insert into tbl1 partition(p1) values('2022-01-01', 2, 10);"; executeInsertSql(connectContext, insertSql); - OlapTable tbl1 = ((OlapTable) testDb.getTable("tbl1")); + OlapTable tbl1 = + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1")); initAndExecuteTaskRun(taskRun); Map> baseTableVisibleVersionMap = materializedView.getRefreshScheme().getAsyncRefreshContext().getBaseTableVisibleVersionMap(); @@ -511,7 +518,8 @@ private void testBaseTablePartitionInsertData(Database testDb, MaterializedView private void testRefreshWithFailure(Database testDb, MaterializedView materializedView, TaskRun taskRun) throws Exception { - OlapTable tbl1 = ((OlapTable) testDb.getTable("tbl1")); + OlapTable tbl1 = + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1")); // insert new data into tbl1's p0 partition // update base table tbl1's p0 version to 3 String insertSql = "insert into tbl1 partition(p0) values('2021-12-01', 2, 10);"; @@ -548,12 +556,13 @@ private Map collectBaseTableSnapshotInfos(MaterializedV if (tableOptional.isEmpty() || !tableOptional.get().isOlapTable()) { continue; } - Database baseDb = GlobalStateMgr.getCurrentState().getDb(baseTableInfo.getDbId()); + Database baseDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(baseTableInfo.getDbId()); if (baseDb == null) { throw new SemanticException("Materialized view base db: " + baseTableInfo.getDbId() + " not exist."); } - OlapTable olapTable = (OlapTable) baseDb.getTable(baseTableInfo.getTableId()); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(baseDb.getId(), baseTableInfo.getTableId()); if (olapTable == null) { throw new SemanticException("Materialized view base table: " + baseTableInfo.getTableId() + " not exist."); @@ -590,7 +599,8 @@ private Map collectBaseTableSnapshotInfos(MaterializedV private void testBaseTablePartitionReplace(Database testDb, MaterializedView materializedView, TaskRun taskRun) throws Exception { // mv need refresh with base table partition p2, p2 replace with tp2 after collect and before insert overwrite - OlapTable tbl1 = ((OlapTable) testDb.getTable("tbl1")); + OlapTable tbl1 = + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1")); new MockUp() { @Mock public Map collectBaseTableSnapshotInfos( @@ -602,12 +612,13 @@ public Map collectBaseTableSnapshotInfos( if (!table.isOlapTable()) { continue; } - Database baseDb = GlobalStateMgr.getCurrentState().getDb(baseTableInfo.getDbId()); + Database baseDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(baseTableInfo.getDbId()); if (baseDb == null) { throw new SemanticException("Materialized view base db: " + baseTableInfo.getDbId() + " not exist."); } - OlapTable olapTable = (OlapTable) baseDb.getTable(baseTableInfo.getTableId()); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(baseDb.getId(), baseTableInfo.getTableId()); if (olapTable == null) { throw new SemanticException("Materialized view base table: " + baseTableInfo.getTableId() + " not exist."); @@ -654,7 +665,8 @@ public Map collectBaseTableSnapshotInfos( public void testBaseTableAddPartitionWhileSync(Database testDb, MaterializedView materializedView, TaskRun taskRun) throws Exception { // mv need refresh with base table partition p3, add partition p99 after collect and before insert overwrite - OlapTable tbl1 = ((OlapTable) testDb.getTable("tbl1")); + OlapTable tbl1 = + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1")); new MockUp() { @Mock public Map collectBaseTableSnapshotInfos(MaterializedView materializedView) { @@ -668,12 +680,13 @@ public Map collectBaseTableSnapshotInfos(MaterializedVi if (!tableOptional.get().isOlapTable()) { continue; } - Database baseDb = GlobalStateMgr.getCurrentState().getDb(baseTableInfo.getDbId()); + Database baseDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(baseTableInfo.getDbId()); if (baseDb == null) { throw new SemanticException("Materialized view base db: " + baseTableInfo.getDbId() + " not exist."); } - OlapTable olapTable = (OlapTable) baseDb.getTable(baseTableInfo.getTableId()); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(baseDb.getId(), baseTableInfo.getTableId()); if (olapTable == null) { throw new SemanticException("Materialized view base table: " + baseTableInfo.getTableId() + " not exist."); @@ -719,7 +732,8 @@ public void testBaseTableAddPartitionWhileRefresh(Database testDb, MaterializedV TaskRun taskRun) throws Exception { // mv need refresh with base table partition p3, add partition p99 after collect and before insert overwrite - OlapTable tbl1 = ((OlapTable) testDb.getTable("tbl1")); + OlapTable tbl1 = + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1")); new MockUp() { @Mock public void refreshMaterializedView(MvTaskRunContext mvContext, ExecPlan execPlan, @@ -759,7 +773,8 @@ public void refreshMaterializedView(MvTaskRunContext mvContext, ExecPlan execPla public void testBaseTableDropPartitionWhileSync(Database testDb, MaterializedView materializedView, TaskRun taskRun) throws Exception { // mv need refresh with base table partition p4, drop partition p4 after collect and before insert overwrite - OlapTable tbl1 = ((OlapTable) testDb.getTable("tbl1")); + OlapTable tbl1 = + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1")); new MockUp() { @Mock private Map collectBaseTableSnapshotInfos(MaterializedView materializedView) { @@ -770,12 +785,13 @@ private Map collectBaseTableSnapshotInfos(MaterializedV if (!table.isOlapTable()) { continue; } - Database baseDb = GlobalStateMgr.getCurrentState().getDb(baseTableInfo.getDbId()); + Database baseDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(baseTableInfo.getDbId()); if (baseDb == null) { throw new SemanticException("Materialized view base db: " + baseTableInfo.getDbId() + " not exist."); } - OlapTable olapTable = (OlapTable) baseDb.getTable(baseTableInfo.getTableId()); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(baseDb.getId(), baseTableInfo.getTableId()); if (olapTable == null) { throw new SemanticException("Materialized view base table: " + baseTableInfo.getTableId() + " not exist."); @@ -812,7 +828,8 @@ public void testBaseTableDropPartitionWhileRefresh(Database testDb, Materialized TaskRun taskRun) throws Exception { // drop partition p4 after collect and before insert overwrite - OlapTable tbl1 = ((OlapTable) testDb.getTable("tbl1")); + OlapTable tbl1 = + ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1")); new MockUp() { @Mock public void refreshMaterializedView(MvTaskRunContext mvContext, ExecPlan execPlan, @@ -869,13 +886,14 @@ public void testFilterPartitionByRefreshNumber() throws Exception { // PARTITION p4 values [('2022-04-01'),('2022-05-01')) String mvName = "mv_with_test_refresh"; - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.withMaterializedView("create materialized view test.mv_with_test_refresh\n" + "partition by k1\n" + "distributed by hash(k2) buckets 10\n" + "refresh deferred manual\n" + "as select k1, k2, sum(v1) as total_sum from base group by k1, k2;"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); initAndExecuteTaskRun(taskRun); @@ -904,7 +922,7 @@ public void testFilterPartitionByRefreshNumberAndDescending() throws Exception { // PARTITION p3 values [('2022-03-01'),('2022-04-01')) // PARTITION p4 values [('2022-04-01'),('2022-05-01')) - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String mvName = "mv_reverse_refresh"; starRocksAssert.withMaterializedView("create materialized view test.mv_reverse_refresh\n" + "partition by k1\n" + @@ -912,7 +930,8 @@ public void testFilterPartitionByRefreshNumberAndDescending() throws Exception { "refresh deferred manual\n" + "as select k1, k2, sum(v1) as total_sum from base group by k1, k2;"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); initAndExecuteTaskRun(taskRun); @@ -995,8 +1014,9 @@ public void testRefreshMaterializedViewDefaultConfig1() throws Exception { "'storage_medium' = 'SSD'" + ")\n" + "as select k1, k2 from tbl1;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_config1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_config1")); String insertSql = "insert into tbl1 partition(p3) values('2022-03-01', 3, 10);"; executeInsertSql(connectContext, insertSql); @@ -1042,8 +1062,9 @@ public void testRefreshMaterializedViewDefaultConfig2() throws Exception { "'session.enable_spill' = 'false'" + ")\n" + "as select k1, k2 from tbl1;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_config2")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_config2")); String insertSql = "insert into tbl1 partition(p3) values('2022-03-01', 3, 10);"; executeInsertSql(connectContext, insertSql); @@ -1067,8 +1088,9 @@ public void testSyncPartitionWithSsdStorage() throws Exception { "properties('replication_num' = '1',\n" + "'storage_medium' = 'SSD')\n" + "as select tbl1.k1, tbl2.k2 from tbl1 join tbl2 on tbl1.k2 = tbl2.k2;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_with_ssd")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_with_ssd")); refreshMVRange(materializedView.getName(), false); refreshMVRange(materializedView.getName(), false); @@ -1085,15 +1107,16 @@ public void testSyncPartitionWithSsdStorageAndCooldownTime() throws Exception { "'storage_medium' = 'SSD',\n" + "'storage_cooldown_time' = '2222-04-21 20:45:11')\n" + "as select tbl1.k1, tbl2.k2 from tbl1 join tbl2 on tbl1.k2 = tbl2.k2;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_use_ssd_and_cooldown")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_use_ssd_and_cooldown")); refreshMVRange(materializedView.getName(), false); refreshMVRange(materializedView.getName(), false); } @Test public void testMVOnListPartitionTables1() throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); String createSQL = "CREATE TABLE test.list_partition_tbl1 (\n" + " id BIGINT,\n" + " age SMALLINT,\n" + @@ -1117,7 +1140,8 @@ public void testMVOnListPartitionTables1() throws Exception { "distributed by hash(dt, province) buckets 10 " + "as select dt, province, avg(age) from list_partition_tbl1 group by dt, province;"; starRocksAssert.withMaterializedView(sql); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("list_partition_mv1")); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "list_partition_mv1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); // run 1 @@ -1178,8 +1202,9 @@ public void testPartitionPruneNonRefBaseTable1() throws Exception { ")\n" + "AS SELECT t1.k1, sum(t1.k2) as sum1, avg(t2.k2) as avg1 FROM tbl4 as t1 join " + "tbl5 t2 on t1.k1=t2.dt group by t1.k1"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("partition_prune_non_ref_tables1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "partition_prune_non_ref_tables1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -1242,8 +1267,9 @@ public void testPartitionPruneNonRefBaseTable2() throws Exception { ")\n" + "AS SELECT t1.k1 as k11, sum(t1.k2) as sum1, avg(t2.k2) as avg1 FROM tbl4 as t1 join " + "tbl5 t2 on t1.k1=t2.dt group by t1.k1"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("partition_prune_non_ref_tables2")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "partition_prune_non_ref_tables2")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -1306,8 +1332,9 @@ public void testPartitionPruneNonRefBaseTable3() throws Exception { ")\n" + "AS SELECT t1.k1, sum(t1.k2) as sum1, avg(t2.k2) as avg1 FROM tbl4 as t1 join " + "tbl5 t2 on t1.k1=t2.dt where t1.k1>'2022-01-01' and t1.k2>0 group by t1.k1"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("partition_prune_non_ref_tables1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "partition_prune_non_ref_tables1")); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); @@ -1365,9 +1392,10 @@ public void testMVDropBaseVersionMetaOfOlapTable() throws Exception { "refresh async \n" + "as select k1, k2, sum(v1) as total from tbl1 group by k1, k2;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - Table tbl1 = testDb.getTable("tbl1"); - MaterializedView mv = ((MaterializedView) testDb.getTable("test_drop_partition_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table tbl1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "tbl1"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_drop_partition_mv1")); Map> versionMap = mv.getRefreshScheme().getAsyncRefreshContext().getBaseTableVisibleVersionMap(); Map> mvPartitionNameRefBaseTablePartitionMap = @@ -1453,16 +1481,20 @@ public void testFilterPartitionByJoinPredicate_RefreshPartitionNum() { "PROPERTIES('partition_refresh_number' = '1')" + "refresh deferred manual\n" + "as select a.k1, b.k2 from tt1 a join tt2 b on a.k1=b.k1;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_with_join0")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_with_join0")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); Map testProperties = task.getProperties(); testProperties.put(TaskRun.IS_TEST, "true"); - OlapTable tbl1 = (OlapTable) testDb.getTable("tt1"); - OlapTable tbl2 = (OlapTable) testDb.getTable("tt2"); + OlapTable tbl1 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tt1"); + OlapTable tbl2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tt2"); TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); @@ -1472,9 +1504,11 @@ public void testFilterPartitionByJoinPredicate_RefreshPartitionNum() { { taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - MvTaskRunContext mvContext = ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); + MvTaskRunContext mvContext = + ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); Assert.assertTrue(mvContext.hasNextBatchPartition()); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); System.out.println(processor.getMVTaskRunExtraMessage()); Assert.assertEquals(Sets.newHashSet("p1"), processor.getMVTaskRunExtraMessage().getMvPartitionsToRefresh()); @@ -1496,9 +1530,11 @@ public void testFilterPartitionByJoinPredicate_RefreshPartitionNum() { { taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - MvTaskRunContext mvContext = ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); + MvTaskRunContext mvContext = + ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); Assert.assertTrue(!mvContext.hasNextBatchPartition()); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); System.out.println(processor.getMVTaskRunExtraMessage()); Assert.assertEquals(Sets.newHashSet("p2"), processor.getMVTaskRunExtraMessage().getMvPartitionsToRefresh()); @@ -1557,18 +1593,24 @@ public void testFilterPartitionByJoinPredicateWithNonPartitionTable() { "as select a.k1, b.k2 from tt1 a join tt2 b on a.k1=b.k1;", (name) -> { String mvName = (String) name; - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); - OlapTable tbl1 = (OlapTable) testDb.getTable("tt1"); - OlapTable tbl2 = (OlapTable) testDb.getTable("tt2"); + OlapTable tbl1 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tt1"); + OlapTable tbl2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tt2"); TaskRun taskRun = buildMVTaskRun(materializedView, TEST_DB_NAME); taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - executeInsertSql(connectContext, "insert into tt1 partition(p1) values('2022-01-02', 3, 10);"); - executeInsertSql(connectContext, "insert into tt1 partition(p2) values('2022-02-02', 3, 10);"); + executeInsertSql(connectContext, + "insert into tt1 partition(p1) values('2022-01-02', 3, 10);"); + executeInsertSql(connectContext, + "insert into tt1 partition(p2) values('2022-02-02', 3, 10);"); executeInsertSql(connectContext, "insert into tt2 values('2022-02-02', 3, 10);"); List tt1Partitions = ImmutableList.of("p0", "p1", "p2"); @@ -1589,12 +1631,14 @@ public void testFilterPartitionByJoinPredicateWithNonPartitionTable() { MaterializedView.AsyncRefreshContext asyncRefreshContext = materializedView.getRefreshScheme().getAsyncRefreshContext(); - Map> baseTableVisibleVersionMap = + Map> + baseTableVisibleVersionMap = asyncRefreshContext.getBaseTableVisibleVersionMap(); System.out.println(baseTableVisibleVersionMap); Assert.assertTrue(baseTableVisibleVersionMap.containsKey(tbl1.getId())); - Assert.assertTrue(baseTableVisibleVersionMap.get(tbl1.getId()).containsKey(tt1Partition)); + Assert.assertTrue( + baseTableVisibleVersionMap.get(tbl1.getId()).containsKey(tt1Partition)); Assert.assertTrue(baseTableVisibleVersionMap.containsKey(tbl2.getId())); taskRun = processor.getNextTaskRun(); Assert.assertTrue(isEnd ? taskRun == null : taskRun != null); @@ -1637,14 +1681,18 @@ public void testFilterPartitionByJoinPredicateWithNonPartition_RefreshPartitionN "PROPERTIES('partition_refresh_number' = '1')" + "refresh deferred manual\n" + "as select a.k1, b.k2 from tt1 a join tt2 b on a.k1=b.k1;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("mv_with_join0")); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_with_join0")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); TaskRun taskRun = buildMVTaskRun(materializedView, TEST_DB_NAME); - OlapTable tbl1 = (OlapTable) testDb.getTable("tt1"); - OlapTable tbl2 = (OlapTable) testDb.getTable("tt2"); + OlapTable tbl1 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tt1"); + OlapTable tbl2 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "tt2"); taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); @@ -1654,9 +1702,11 @@ public void testFilterPartitionByJoinPredicateWithNonPartition_RefreshPartitionN { taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - MvTaskRunContext mvContext = ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); + MvTaskRunContext mvContext = + ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); Assert.assertTrue(mvContext.hasNextBatchPartition()); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); System.out.println(processor.getMVTaskRunExtraMessage()); Assert.assertEquals(Sets.newHashSet("p0"), processor.getMVTaskRunExtraMessage().getMvPartitionsToRefresh()); @@ -1679,9 +1729,11 @@ public void testFilterPartitionByJoinPredicateWithNonPartition_RefreshPartitionN { taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); taskRun.executeTaskRun(); - MvTaskRunContext mvContext = ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); + MvTaskRunContext mvContext = + ((PartitionBasedMvRefreshProcessor) taskRun.getProcessor()).getMvContext(); Assert.assertTrue(!mvContext.hasNextBatchPartition()); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); + PartitionBasedMvRefreshProcessor processor = + (PartitionBasedMvRefreshProcessor) taskRun.getProcessor(); System.out.println(processor.getMVTaskRunExtraMessage()); Assert.assertEquals(Sets.newHashSet("p1"), processor.getMVTaskRunExtraMessage().getMvPartitionsToRefresh()); @@ -1737,8 +1789,11 @@ public void testMVPartitionMappingWithManyToMany() { "properties('replication_num' = '1', 'partition_refresh_number'='1')\n" + "as select k1, k2, v1 from mock_tbl;", (mvName) -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb(TEST_DB_NAME); - MaterializedView materializedView = ((MaterializedView) testDb.getTable((String) mvName)); + Database testDb = + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(TEST_DB_NAME); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), (String) mvName)); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); // initial refresh @@ -1760,7 +1815,10 @@ public void testMVPartitionMappingWithManyToMany() { Map snapshotInfoMap = processor.getSnapshotBaseTables(); Assert.assertEquals(1, snapshotInfoMap.size()); TableSnapshotInfo tableSnapshotInfo = - snapshotInfoMap.get(testDb.getTable("mock_tbl").getId()); + snapshotInfoMap.get( + GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mock_tbl") + .getId()); Assert.assertEquals(Sets.newHashSet("p0", "p1", "p2"), tableSnapshotInfo.getRefreshedPartitionInfos().keySet()); @@ -1790,7 +1848,10 @@ public void testMVPartitionMappingWithManyToMany() { Map snapshotInfoMap = processor.getSnapshotBaseTables(); Assert.assertEquals(1, snapshotInfoMap.size()); TableSnapshotInfo tableSnapshotInfo = - snapshotInfoMap.get(testDb.getTable("mock_tbl").getId()); + snapshotInfoMap.get( + GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mock_tbl") + .getId()); System.out.println(processor.getMVTaskRunExtraMessage()); Assert.assertEquals(Sets.newHashSet("p0", "p1", "p2", "p3"), tableSnapshotInfo.getRefreshedPartitionInfos().keySet()); @@ -1831,8 +1892,11 @@ public void testMVPartitionMappingWithOneToMany() { "properties('replication_num' = '1', 'partition_refresh_number'='1')\n" + "as select k1, k2, v1 from mock_tbl;", (mvName) -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb(TEST_DB_NAME); - MaterializedView materializedView = ((MaterializedView) testDb.getTable((String) mvName)); + Database testDb = + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(TEST_DB_NAME); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), (String) mvName)); Assert.assertEquals(1, materializedView.getPartitionExprMaps().size()); // initial refresh @@ -1854,13 +1918,17 @@ public void testMVPartitionMappingWithOneToMany() { Map snapshotInfoMap = processor.getSnapshotBaseTables(); Assert.assertEquals(1, snapshotInfoMap.size()); TableSnapshotInfo tableSnapshotInfo = - snapshotInfoMap.get(testDb.getTable("mock_tbl").getId()); + snapshotInfoMap.get( + GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mock_tbl") + .getId()); Assert.assertEquals(Sets.newHashSet("p0"), tableSnapshotInfo.getRefreshedPartitionInfos().keySet()); MVTaskRunExtraMessage extraMessage = processor.getMVTaskRunExtraMessage(); System.out.println(processor.getMVTaskRunExtraMessage()); - Assert.assertTrue(extraMessage.getMvPartitionsToRefresh().contains("p20210723_20210724")); + Assert.assertTrue( + extraMessage.getMvPartitionsToRefresh().contains("p20210723_20210724")); Assert.assertEquals(Sets.newHashSet("p0"), extraMessage.getBasePartitionsToRefreshMap().get("mock_tbl")); Assert.assertTrue(processor.getNextTaskRun() == null); @@ -1880,14 +1948,18 @@ public void testMVPartitionMappingWithOneToMany() { Map snapshotInfoMap = processor.getSnapshotBaseTables(); Assert.assertEquals(1, snapshotInfoMap.size()); TableSnapshotInfo tableSnapshotInfo = - snapshotInfoMap.get(testDb.getTable("mock_tbl").getId()); + snapshotInfoMap.get( + GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mock_tbl") + .getId()); System.out.println(processor.getMVTaskRunExtraMessage()); Assert.assertEquals(Sets.newHashSet("p1", "p2"), tableSnapshotInfo.getRefreshedPartitionInfos().keySet()); MVTaskRunExtraMessage extraMessage = processor.getMVTaskRunExtraMessage(); System.out.println(processor.getMVTaskRunExtraMessage()); - Assert.assertTrue(extraMessage.getMvPartitionsToRefresh().contains("p20210811_20210812")); + Assert.assertTrue( + extraMessage.getMvPartitionsToRefresh().contains("p20210811_20210812")); Assert.assertEquals(Sets.newHashSet("p1", "p2"), extraMessage.getBasePartitionsToRefreshMap().get("mock_tbl")); Assert.assertTrue(processor.getNextTaskRun() == null); @@ -1930,8 +2002,11 @@ public void testShowMaterializedViewsWithNonForce() { "as select k1, k2 from mockTbl;", () -> { String mvName = "mock_mv0"; - Database testDb = GlobalStateMgr.getCurrentState().getDb(TEST_DB_NAME); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(TEST_DB_NAME); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); TaskManager tm = GlobalStateMgr.getCurrentState().getTaskManager(); executeInsertSql(connectContext, mTable.getGenerateDataSQL()); @@ -1962,7 +2037,8 @@ public void testShowMaterializedViewsWithNonForce() { materializedView.getName()); System.out.println(status); status.setLastJobTaskRunStatus(taskRunStatuses); - ShowMaterializedViewStatus.RefreshJobStatus refreshJobStatus = status.getRefreshJobStatus(); + ShowMaterializedViewStatus.RefreshJobStatus refreshJobStatus = + status.getRefreshJobStatus(); System.out.println(refreshJobStatus); Assert.assertEquals(refreshJobStatus.isForce(), false); Assert.assertEquals(refreshJobStatus.isRefreshFinished(), true); @@ -2015,8 +2091,11 @@ public void testShowMaterializedViewsWithPartialRefresh() { "as select k1, k2 from mockTbl;", () -> { String mvName = "mock_mv0"; - Database testDb = GlobalStateMgr.getCurrentState().getDb(TEST_DB_NAME); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(TEST_DB_NAME); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); TaskManager tm = GlobalStateMgr.getCurrentState().getTaskManager(); executeInsertSql(connectContext, mTable.getGenerateDataSQL()); @@ -2051,7 +2130,8 @@ public void testShowMaterializedViewsWithPartialRefresh() { new ShowMaterializedViewStatus(materializedView.getId(), TEST_DB_NAME, materializedView.getName()); status.setLastJobTaskRunStatus(taskNameJobStatusMap.get(mvTaskName)); - ShowMaterializedViewStatus.RefreshJobStatus refreshJobStatus = status.getRefreshJobStatus(); + ShowMaterializedViewStatus.RefreshJobStatus refreshJobStatus = + status.getRefreshJobStatus(); System.out.println(refreshJobStatus); Assert.assertEquals(refreshJobStatus.isForce(), false); Assert.assertEquals(refreshJobStatus.isRefreshFinished(), true); @@ -2102,8 +2182,11 @@ public void testShowMaterializedViewsWithForce() { "as select k1, k2 from mockTbl;", () -> { String mvName = "mock_mv0"; - Database testDb = GlobalStateMgr.getCurrentState().getDb(TEST_DB_NAME); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = + GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(TEST_DB_NAME); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); TaskManager tm = GlobalStateMgr.getCurrentState().getTaskManager(); // refresh materialized view(force) @@ -2127,7 +2210,8 @@ public void testShowMaterializedViewsWithForce() { new ShowMaterializedViewStatus(materializedView.getId(), TEST_DB_NAME, materializedView.getName()); status.setLastJobTaskRunStatus(taskNameJobStatusMap.get(mvTaskName)); - ShowMaterializedViewStatus.RefreshJobStatus refreshJobStatus = status.getRefreshJobStatus(); + ShowMaterializedViewStatus.RefreshJobStatus refreshJobStatus = + status.getRefreshJobStatus(); System.out.println(refreshJobStatus); Assert.assertEquals(refreshJobStatus.isForce(), true); Assert.assertEquals(refreshJobStatus.isRefreshFinished(), true); @@ -2183,10 +2267,11 @@ public void testMVRefreshStatus() { "refresh deferred manual\n" + "as select a.k1, b.k2 from tt1 a join tt2 b on a.k1=b.k1;", () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); MaterializedView materializedView = - ((MaterializedView) testDb.getTable("mv_with_join0")); + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_with_join0")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); Map testProperties = task.getProperties(); @@ -2204,7 +2289,8 @@ public void testMVRefreshStatus() { String jobID = ""; { TaskRunStatus taskRunStatus = - taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); + taskRun.initStatus(UUIDUtil.genUUID().toString(), + System.currentTimeMillis()); taskRun.executeTaskRun(); taskRunStatus.setState(Constants.TaskRunState.SUCCESS); @@ -2244,7 +2330,8 @@ public void testMVRefreshStatus() { { TaskRunStatus taskRunStatus = - taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis()); + taskRun.initStatus(UUIDUtil.genUUID().toString(), + System.currentTimeMillis()); taskRun.executeTaskRun(); taskRunStatus.setState(Constants.TaskRunState.SUCCESS); @@ -2325,10 +2412,11 @@ public void testMVRefreshWithFailedStatus() { "refresh deferred manual\n" + "as select a.k1, b.k2 from tt1 a join tt2 b on a.k1=b.k1;", () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); MaterializedView materializedView = - ((MaterializedView) testDb.getTable("mv_with_join0")); + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "mv_with_join0")); Assert.assertEquals(2, materializedView.getPartitionExprMaps().size()); Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); Map testProperties = task.getProperties(); @@ -2390,8 +2478,10 @@ public void testRefreshWithTraceProfile() { "as select k1, k2 from tbl1;", () -> { Config.enable_mv_refresh_query_rewrite = false; - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("test_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_mv1")); String insertSql = "insert into tbl1 partition(p3) values('2022-03-01', 3, 10);"; executeInsertSql(connectContext, insertSql); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorPaimonTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorPaimonTest.java index 1f9a85ec83949..14f18af232047 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorPaimonTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorPaimonTest.java @@ -74,10 +74,11 @@ public void testcreateUnpartitionedPmnMaterializeView() { ")\n" + "AS SELECT pk, d FROM `paimon0`.`pmn_db1`.`unpartitioned_table` as a;", () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); MaterializedView unpartitionedMaterializedView = - ((MaterializedView) testDb.getTable("paimon_parttbl_mv2")); + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "paimon_parttbl_mv2")); triggerRefreshMv(testDb, unpartitionedMaterializedView); Collection partitions = unpartitionedMaterializedView.getPartitions(); @@ -104,9 +105,10 @@ public void testCreatePartitionedPmnMaterializeView() { ")\n" + "AS SELECT pk, pt,d FROM `paimon0`.`pmn_db1`.`partitioned_table` as a;", () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); MaterializedView partitionedMaterializedView = - ((MaterializedView) testDb.getTable("paimon_parttbl_mv1")); + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "paimon_parttbl_mv1")); triggerRefreshMv(testDb, partitionedMaterializedView); Collection partitions = partitionedMaterializedView.getPartitions(); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshTest.java index 294bf26870f3b..3a37479666c23 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshTest.java @@ -49,62 +49,62 @@ public static void beforeClass() throws Exception { MVRefreshTestBase.beforeClass(); ConnectorPlanTestBase.mockAllCatalogs(connectContext, temp.newFolder().toURI().toString()); starRocksAssert - .withTable("CREATE TABLE `t1` (\n" + - " `k1` date not null, \n" + - " `k2` datetime not null, \n" + - " `k3` char(20), \n" + - " `k4` varchar(20), \n" + - " `k5` boolean, \n" + - " `k6` tinyint, \n" + - " `k7` smallint, \n" + - " `k8` int, \n" + - " `k9` bigint, \n" + - " `k10` largeint, \n" + - " `k11` float, \n" + - " `k12` double, \n" + - " `k13` decimal(27,9) ) \n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`) \n" + - "PARTITION BY RANGE(`k1`) \n" + - "(\n" + - "PARTITION p20201022 VALUES [(\"2020-10-22\"), (\"2020-10-23\")), \n" + - "PARTITION p20201023 VALUES [(\"2020-10-23\"), (\"2020-10-24\")), \n" + - "PARTITION p20201024 VALUES [(\"2020-10-24\"), (\"2020-10-25\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3;") - .withTable("CREATE TABLE `t2` (\n" + - " `k1` date not null, \n" + - " `k2` datetime not null, \n" + - " `k3` char(20), \n" + - " `k4` varchar(20), \n" + - " `k5` boolean, \n" + - " `k6` tinyint, \n" + - " `k7` smallint, \n" + - " `k8` int, \n" + - " `k9` bigint, \n" + - " `k10` largeint, \n" + - " `k11` float, \n" + - " `k12` double, \n" + - " `k13` decimal(27,9) ) \n" + - "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`) \n" + - "PARTITION BY RANGE(`k1`) \n" + - "(\n" + - "PARTITION p20201010 VALUES [(\"2020-10-10\"), (\"2020-10-11\")), \n" + - "PARTITION p20201011 VALUES [(\"2020-10-11\"), (\"2020-10-12\")), \n" + - "PARTITION p20201012 VALUES [(\"2020-10-12\"), (\"2020-10-13\")), \n" + - "PARTITION p20201021 VALUES [(\"2020-10-21\"), (\"2020-10-22\")), \n" + - "PARTITION p20201022 VALUES [(\"2020-10-22\"), (\"2020-10-23\"))\n" + - ")\n" + - "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3;"); + .withTable("CREATE TABLE `t1` (\n" + + " `k1` date not null, \n" + + " `k2` datetime not null, \n" + + " `k3` char(20), \n" + + " `k4` varchar(20), \n" + + " `k5` boolean, \n" + + " `k6` tinyint, \n" + + " `k7` smallint, \n" + + " `k8` int, \n" + + " `k9` bigint, \n" + + " `k10` largeint, \n" + + " `k11` float, \n" + + " `k12` double, \n" + + " `k13` decimal(27,9) ) \n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`) \n" + + "PARTITION BY RANGE(`k1`) \n" + + "(\n" + + "PARTITION p20201022 VALUES [(\"2020-10-22\"), (\"2020-10-23\")), \n" + + "PARTITION p20201023 VALUES [(\"2020-10-23\"), (\"2020-10-24\")), \n" + + "PARTITION p20201024 VALUES [(\"2020-10-24\"), (\"2020-10-25\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3;") + .withTable("CREATE TABLE `t2` (\n" + + " `k1` date not null, \n" + + " `k2` datetime not null, \n" + + " `k3` char(20), \n" + + " `k4` varchar(20), \n" + + " `k5` boolean, \n" + + " `k6` tinyint, \n" + + " `k7` smallint, \n" + + " `k8` int, \n" + + " `k9` bigint, \n" + + " `k10` largeint, \n" + + " `k11` float, \n" + + " `k12` double, \n" + + " `k13` decimal(27,9) ) \n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`) \n" + + "PARTITION BY RANGE(`k1`) \n" + + "(\n" + + "PARTITION p20201010 VALUES [(\"2020-10-10\"), (\"2020-10-11\")), \n" + + "PARTITION p20201011 VALUES [(\"2020-10-11\"), (\"2020-10-12\")), \n" + + "PARTITION p20201012 VALUES [(\"2020-10-12\"), (\"2020-10-13\")), \n" + + "PARTITION p20201021 VALUES [(\"2020-10-21\"), (\"2020-10-22\")), \n" + + "PARTITION p20201022 VALUES [(\"2020-10-22\"), (\"2020-10-23\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3;"); executeInsertSql(connectContext, "INSERT INTO t1 VALUES\n" + - " ('2020-10-22','2020-10-23 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + - " ('2020-10-23','2020-10-24 12:12:12','k3','k4',0,0,2,3,4,5,1.1,1.12,2.889),\n" + - " ('2020-10-24','2020-10-25 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889);"); + " ('2020-10-22','2020-10-23 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + + " ('2020-10-23','2020-10-24 12:12:12','k3','k4',0,0,2,3,4,5,1.1,1.12,2.889),\n" + + " ('2020-10-24','2020-10-25 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889);"); executeInsertSql(connectContext, "INSERT INTO t2 VALUES\n" + - " ('2020-10-10','2020-10-23 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + - " ('2020-10-11','2020-10-24 12:12:12','k3','k4',0,0,2,3,4,5,1.1,1.12,2.889),\n" + - " ('2020-10-12','2020-10-25 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + - " ('2020-10-21','2020-10-24 12:12:12','k3','k4',0,0,2,3,4,5,1.1,1.12,2.889),\n" + - " ('2020-10-22','2020-10-25 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889)"); + " ('2020-10-10','2020-10-23 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + + " ('2020-10-11','2020-10-24 12:12:12','k3','k4',0,0,2,3,4,5,1.1,1.12,2.889),\n" + + " ('2020-10-12','2020-10-25 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889),\n" + + " ('2020-10-21','2020-10-24 12:12:12','k3','k4',0,0,2,3,4,5,1.1,1.12,2.889),\n" + + " ('2020-10-22','2020-10-25 12:12:12','k3','k4',0,1,2,3,4,5,1.1,1.12,2.889)"); } @AfterClass @@ -127,7 +127,7 @@ protected void assertPlanContains(ExecPlan execPlan, String... explain) throws E for (String expected : explain) { Assert.assertTrue("expected is: " + expected + " but plan is \n" + explainString, - StringUtils.containsIgnoreCase(explainString.toLowerCase(), expected)); + StringUtils.containsIgnoreCase(explainString.toLowerCase(), expected)); } } @@ -156,49 +156,51 @@ private void testRefreshUnionAllWithDefaultRefreshNumber(String mvSql, Assert.assertTrue(t1PartitionNums.size() == t2PartitionNums.size()); int mvRefreshTimes = t1PartitionNums.size(); starRocksAssert.withMaterializedView(mvSql, - () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable("test_mv0")); - Task task = TaskBuilder.buildMvTask(mv, testDb.getFullName()); + () -> { + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_mv0")); + Task task = TaskBuilder.buildMvTask(mv, testDb.getFullName()); - TaskRun taskRun = null; - for (int i = 0; i < mvRefreshTimes; i++) { - if (i == 0) { - taskRun = TaskRunBuilder.newBuilder(task).build(); + TaskRun taskRun = null; + for (int i = 0; i < mvRefreshTimes; i++) { + if (i == 0) { + taskRun = TaskRunBuilder.newBuilder(task).build(); + } + System.out.println("start to execute task run:" + i); + Assert.assertTrue(taskRun != null); + initAndExecuteTaskRun(taskRun); + PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) + taskRun.getProcessor(); + MvTaskRunContext mvContext = processor.getMvContext(); + ExecPlan execPlan = mvContext.getExecPlan(); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + Assert.assertTrue(plan != null); + taskRun = processor.getNextTaskRun(); + PlanTestBase.assertContains(plan, String.format(" TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=%s/3", t1PartitionNums.get(i))); + PlanTestBase.assertContains(plan, String.format(" TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=%s/5", t2PartitionNums.get(i))); + if (i == mvRefreshTimes - 1) { + Assert.assertTrue(taskRun == null); + } } - System.out.println("start to execute task run:" + i); - Assert.assertTrue(taskRun != null); - initAndExecuteTaskRun(taskRun); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) - taskRun.getProcessor(); - MvTaskRunContext mvContext = processor.getMvContext(); - ExecPlan execPlan = mvContext.getExecPlan(); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - Assert.assertTrue(plan != null); - taskRun = processor.getNextTaskRun(); - PlanTestBase.assertContains(plan, String.format(" TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=%s/3", t1PartitionNums.get(i))); - PlanTestBase.assertContains(plan, String.format(" TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=%s/5", t2PartitionNums.get(i))); - if (i == mvRefreshTimes - 1) { - Assert.assertTrue(taskRun == null); - } - } - }); + }); } + @Test public void testUnionAllMvWithPartition1() { String sql = "create materialized view test_mv0 \n" + - "partition by k1 \n" + - "distributed by random \n" + - "refresh async \n" + - "properties(" + - "\"partition_refresh_number\" = \"1\"" + - ")" + - "as " + - " select * from t1 union all select * from t2;"; + "partition by k1 \n" + + "distributed by random \n" + + "refresh async \n" + + "properties(" + + "\"partition_refresh_number\" = \"1\"" + + ")" + + "as " + + " select * from t1 union all select * from t2;"; List t1PartitionNums = ImmutableList.of(0, 0, 0, 0, 1, 1, 1); List t2PartitionNums = ImmutableList.of(1, 1, 1, 1, 1, 0, 0); testRefreshUnionAllWithDefaultRefreshNumber(sql, t1PartitionNums, t2PartitionNums); @@ -207,14 +209,14 @@ public void testUnionAllMvWithPartition1() { @Test public void testUnionAllMvWithPartition2() { String sql = "create materialized view test_mv0 \n" + - "partition by k1 \n" + - "distributed by random \n" + - "refresh async \n" + - "properties(" + - "\"partition_refresh_number\" = \"1\"" + - ")" + - "as " + - " select * from t2 union all select * from t1;"; + "partition by k1 \n" + + "distributed by random \n" + + "refresh async \n" + + "properties(" + + "\"partition_refresh_number\" = \"1\"" + + ")" + + "as " + + " select * from t2 union all select * from t1;"; List t1PartitionNums = ImmutableList.of(0, 0, 0, 0, 1, 1, 1); List t2PartitionNums = ImmutableList.of(1, 1, 1, 1, 1, 0, 0); testRefreshUnionAllWithDefaultRefreshNumber(sql, t1PartitionNums, t2PartitionNums); @@ -223,121 +225,123 @@ public void testUnionAllMvWithPartition2() { @Test public void testUnionAllMvWithPartitionForceMode() { String sql = "create materialized view test_mv0 \n" + - "partition by k1 \n" + - "distributed by random \n" + - "refresh async \n" + - "as " + - " select * from t1 union all select * from t2;"; + "partition by k1 \n" + + "distributed by random \n" + + "refresh async \n" + + "as " + + " select * from t1 union all select * from t2;"; starRocksAssert.withMaterializedView(sql, - () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable("test_mv0")); - Task task = TaskBuilder.buildMvTask(mv, testDb.getFullName()); + () -> { + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_mv0")); + Task task = TaskBuilder.buildMvTask(mv, testDb.getFullName()); - TaskRun taskRun = null; - int refreshTimes = 1; - for (int i = 0; i < refreshTimes; i++) { - if (i == 0) { - taskRun = TaskRunBuilder.newBuilder(task).build(); - } - System.out.println("start to execute task run:" + i); - Assert.assertTrue(taskRun != null); - initAndExecuteTaskRun(taskRun); - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) - taskRun.getProcessor(); - MvTaskRunContext mvContext = processor.getMvContext(); - ExecPlan execPlan = mvContext.getExecPlan(); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - taskRun = processor.getNextTaskRun(); + TaskRun taskRun = null; + int refreshTimes = 1; + for (int i = 0; i < refreshTimes; i++) { + if (i == 0) { + taskRun = TaskRunBuilder.newBuilder(task).build(); + } + System.out.println("start to execute task run:" + i); + Assert.assertTrue(taskRun != null); + initAndExecuteTaskRun(taskRun); + PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) + taskRun.getProcessor(); + MvTaskRunContext mvContext = processor.getMvContext(); + ExecPlan execPlan = mvContext.getExecPlan(); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + taskRun = processor.getNextTaskRun(); - PlanTestBase.assertContains(plan, " TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=3/3"); - PlanTestBase.assertContains(plan, " TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=5/5"); - if (i == refreshTimes - 1) { - Assert.assertTrue(taskRun == null); + PlanTestBase.assertContains(plan, " TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=3/3"); + PlanTestBase.assertContains(plan, " TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=5/5"); + if (i == refreshTimes - 1) { + Assert.assertTrue(taskRun == null); + } } - } - }); + }); } @Test public void testUnionAllMvWithPartitionWithPartitionStartAndEnd() { String sql = "create materialized view test_mv0 \n" + - "partition by k1 \n" + - "distributed by random \n" + - "refresh async \n" + - "properties(" + - "\"partition_refresh_number\" = \"1\"" + - ")" + - "as " + - " select * from t2 union all select * from t1;"; + "partition by k1 \n" + + "distributed by random \n" + + "refresh async \n" + + "properties(" + + "\"partition_refresh_number\" = \"1\"" + + ")" + + "as " + + " select * from t2 union all select * from t1;"; starRocksAssert.withMaterializedView(sql, - () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv = ((MaterializedView) testDb.getTable("test_mv0")); + () -> { + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_mv0")); - Task task = TaskBuilder.buildMvTask(mv, testDb.getFullName()); - int mvRefreshTimes = 3; - List t1PartitionNums = ImmutableList.of(0, 0, 1); - List t2PartitionNums = ImmutableList.of(1, 1, 1); - TaskRun taskRun = null; - for (int i = 0; i < mvRefreshTimes; i++) { - System.out.println("start to execute task run:" + i); - if (i == 0) { - taskRun = TaskRunBuilder.newBuilder(task).build(); - initAndExecuteTaskRun(taskRun, "2020-10-12", "2020-10-23"); - } else { - ExecuteOption executeOption = taskRun.getExecuteOption(); - String partitionStart = executeOption.getTaskRunProperties().get(TaskRun.PARTITION_START); - String partitionEnd = executeOption.getTaskRunProperties().get(TaskRun.PARTITION_END); - initAndExecuteTaskRun(taskRun, partitionStart, partitionEnd); - } - PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) - taskRun.getProcessor(); - MvTaskRunContext mvContext = processor.getMvContext(); - ExecPlan execPlan = mvContext.getExecPlan(); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - Assert.assertTrue(plan != null); - taskRun = processor.getNextTaskRun(); - PlanTestBase.assertContains(plan, String.format(" TABLE: t1\n" + - " PREAGGREGATION: ON\n" + - " partitions=%s/3", t1PartitionNums.get(i))); - PlanTestBase.assertContains(plan, String.format(" TABLE: t2\n" + - " PREAGGREGATION: ON\n" + - " partitions=%s/5", t2PartitionNums.get(i))); - if (i == mvRefreshTimes - 1) { - Assert.assertTrue(taskRun == null); + Task task = TaskBuilder.buildMvTask(mv, testDb.getFullName()); + int mvRefreshTimes = 3; + List t1PartitionNums = ImmutableList.of(0, 0, 1); + List t2PartitionNums = ImmutableList.of(1, 1, 1); + TaskRun taskRun = null; + for (int i = 0; i < mvRefreshTimes; i++) { + System.out.println("start to execute task run:" + i); + if (i == 0) { + taskRun = TaskRunBuilder.newBuilder(task).build(); + initAndExecuteTaskRun(taskRun, "2020-10-12", "2020-10-23"); + } else { + ExecuteOption executeOption = taskRun.getExecuteOption(); + String partitionStart = executeOption.getTaskRunProperties().get(TaskRun.PARTITION_START); + String partitionEnd = executeOption.getTaskRunProperties().get(TaskRun.PARTITION_END); + initAndExecuteTaskRun(taskRun, partitionStart, partitionEnd); + } + PartitionBasedMvRefreshProcessor processor = (PartitionBasedMvRefreshProcessor) + taskRun.getProcessor(); + MvTaskRunContext mvContext = processor.getMvContext(); + ExecPlan execPlan = mvContext.getExecPlan(); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + Assert.assertTrue(plan != null); + taskRun = processor.getNextTaskRun(); + PlanTestBase.assertContains(plan, String.format(" TABLE: t1\n" + + " PREAGGREGATION: ON\n" + + " partitions=%s/3", t1PartitionNums.get(i))); + PlanTestBase.assertContains(plan, String.format(" TABLE: t2\n" + + " PREAGGREGATION: ON\n" + + " partitions=%s/5", t2PartitionNums.get(i))); + if (i == mvRefreshTimes - 1) { + Assert.assertTrue(taskRun == null); + } } - } - }); + }); } @Test public void testJoinMV_SlotRef() throws Exception { starRocksAssert.withTable("CREATE TABLE join_base_t1 (dt1 date, int1 int)\n" + - " PARTITION BY RANGE(dt1)\n" + - " (\n" + - " PARTITION p1 VALUES LESS THAN (\"2020-07-01\"),\n" + - " PARTITION p2 VALUES LESS THAN (\"2020-08-01\"),\n" + - " PARTITION p3 VALUES LESS THAN (\"2020-09-01\")\n" + - " );"); + " PARTITION BY RANGE(dt1)\n" + + " (\n" + + " PARTITION p1 VALUES LESS THAN (\"2020-07-01\"),\n" + + " PARTITION p2 VALUES LESS THAN (\"2020-08-01\"),\n" + + " PARTITION p3 VALUES LESS THAN (\"2020-09-01\")\n" + + " );"); starRocksAssert.withTable("CREATE TABLE join_base_t2 (dt2 date, int2 int)\n" + - " PARTITION BY RANGE(dt2)\n" + - " (\n" + - " PARTITION p4 VALUES LESS THAN (\"2020-07-01\"),\n" + - " PARTITION p5 VALUES LESS THAN (\"2020-08-01\"),\n" + - " PARTITION p6 VALUES LESS THAN (\"2020-09-01\")\n" + - " );"); + " PARTITION BY RANGE(dt2)\n" + + " (\n" + + " PARTITION p4 VALUES LESS THAN (\"2020-07-01\"),\n" + + " PARTITION p5 VALUES LESS THAN (\"2020-08-01\"),\n" + + " PARTITION p6 VALUES LESS THAN (\"2020-09-01\")\n" + + " );"); starRocksAssert.withRefreshedMaterializedView("CREATE MATERIALIZED VIEW join_mv1 " + - "PARTITION BY dt1 " + - "REFRESH MANUAL " + - "PROPERTIES (\"partition_refresh_number\"=\"3\") AS " + - "SELECT dt1,dt2,sum(int1) " + - "FROM join_base_t1 t1 " + - "JOIN join_base_t2 t2 ON t1.dt1=t2.dt2 GROUP BY dt1,dt2;"); + "PARTITION BY dt1 " + + "REFRESH MANUAL " + + "PROPERTIES (\"partition_refresh_number\"=\"3\") AS " + + "SELECT dt1,dt2,sum(int1) " + + "FROM join_base_t1 t1 " + + "JOIN join_base_t2 t2 ON t1.dt1=t2.dt2 GROUP BY dt1,dt2;"); MaterializedView mv = starRocksAssert.getMv("test", "join_mv1"); Assert.assertEquals(Sets.newHashSet("p1", "p2", "p3"), mv.getPartitionNames()); @@ -349,25 +353,25 @@ public void testJoinMV_SlotRef() throws Exception { @Test public void testJoinMV_ListPartition() throws Exception { starRocksAssert.withTable("CREATE TABLE join_base_t1 (dt1 date, int1 int)\n" + - "PARTITION BY RANGE(dt1)\n" + - "(\n" + - " PARTITION p202006 VALUES LESS THAN (\"2020-07-01\"),\n" + - " PARTITION p202007 VALUES LESS THAN (\"2020-08-01\"),\n" + - " PARTITION p202008 VALUES LESS THAN (\"2020-09-01\")\n" + - ")"); + "PARTITION BY RANGE(dt1)\n" + + "(\n" + + " PARTITION p202006 VALUES LESS THAN (\"2020-07-01\"),\n" + + " PARTITION p202007 VALUES LESS THAN (\"2020-08-01\"),\n" + + " PARTITION p202008 VALUES LESS THAN (\"2020-09-01\")\n" + + ")"); starRocksAssert.withTable("CREATE TABLE join_base_t2 (dt2 date not null, int2 int)\n" + - "PARTITION BY LIST(dt2)\n" + - "(\n" + - " PARTITION p202006 VALUES in (\"2020-06-23\"),\n" + - " PARTITION p202007 VALUES in (\"2020-07-23\"),\n" + - " PARTITION p202008 VALUES in (\"2020-08-23\")\n" + - ");"); + "PARTITION BY LIST(dt2)\n" + + "(\n" + + " PARTITION p202006 VALUES in (\"2020-06-23\"),\n" + + " PARTITION p202007 VALUES in (\"2020-07-23\"),\n" + + " PARTITION p202008 VALUES in (\"2020-08-23\")\n" + + ");"); Exception e = Assert.assertThrows(DmlException.class, () -> - starRocksAssert.withRefreshedMaterializedView("CREATE MATERIALIZED VIEW join_mv1 " + - "PARTITION BY dt1 REFRESH MANUAL PROPERTIES (\"partition_refresh_number\"=\"3\") AS \n" + - "SELECT dt1,dt2,sum(int1) " + - "FROM join_base_t1 t1 " + - "JOIN join_base_t2 t2 ON t1.dt1=t2.dt2 GROUP BY dt1,dt2") + starRocksAssert.withRefreshedMaterializedView("CREATE MATERIALIZED VIEW join_mv1 " + + "PARTITION BY dt1 REFRESH MANUAL PROPERTIES (\"partition_refresh_number\"=\"3\") AS \n" + + "SELECT dt1,dt2,sum(int1) " + + "FROM join_base_t1 t1 " + + "JOIN join_base_t2 t2 ON t1.dt1=t2.dt2 GROUP BY dt1,dt2") ); // TODO(fix me): throw a better stack System.out.println(e.getMessage()); diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/mv/MVMaintenanceJobTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/mv/MVMaintenanceJobTest.java index 8855947b8cf84..d59ed7d462035 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/mv/MVMaintenanceJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/mv/MVMaintenanceJobTest.java @@ -126,7 +126,7 @@ public void buildPhysicalTopology() throws Exception { pair.first); String currentDb = connectContext.getDatabase(); - long dbId = GlobalStateMgr.getCurrentState().getDb(currentDb).getId(); + long dbId = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(currentDb).getId(); MaterializedView view = new MaterializedView(); view.setDbId(dbId); view.setId(1024); diff --git a/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java b/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java index f9ba111658d56..8e9712043cf28 100644 --- a/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/server/ConcurrentDDLTest.java @@ -90,9 +90,9 @@ public void testConcurrentCreatingColocateTables() throws InterruptedException { try { System.out.println("start to create table"); starRocksAssert.withTable("create table test.test_tbl_" + Thread.currentThread().getId() + - " (id int) duplicate key (id)" + - " distributed by hash(id) buckets 5183 " + - "properties(\"replication_num\"=\"1\", \"colocate_with\"=\"test_cg_001\");"); + " (id int) duplicate key (id)" + + " distributed by hash(id) buckets 5183 " + + "properties(\"replication_num\"=\"1\", \"colocate_with\"=\"test_cg_001\");"); System.out.println("end to create table"); } catch (Exception e) { throw new RuntimeException(e); @@ -111,19 +111,20 @@ public void testConcurrentCreatingColocateTables() throws InterruptedException { thread.join(); } - Database db = GlobalStateMgr.getServingState().getDb("test"); - Table table = db.getTable("test_tbl_" + threadIds.get(0)); + Database db = GlobalStateMgr.getServingState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "test_tbl_" + threadIds.get(0)); List> bucketSeq = GlobalStateMgr.getCurrentState().getColocateTableIndex().getBackendsPerBucketSeq( - GlobalStateMgr.getCurrentState().getColocateTableIndex().getGroup(table.getId())); + GlobalStateMgr.getCurrentState().getColocateTableIndex().getGroup(table.getId())); // check all created colocate tables has same tablet distribution as the bucket seq in colocate group for (long threadId : threadIds) { - table = db.getTable("test_tbl_" + threadId); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "test_tbl_" + threadId); List tablets = table.getPartitions().stream().findFirst().get().getBaseIndex().getTabletIdsInOrder(); List backendIdList = tablets.stream() - .map(id -> GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getReplicasByTabletId(id)) - .map(replicaList -> replicaList.get(0).getBackendId()) - .collect(Collectors.toList()); + .map(id -> GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getReplicasByTabletId(id)) + .map(replicaList -> replicaList.get(0).getBackendId()) + .collect(Collectors.toList()); Assert.assertEquals(bucketSeq, backendIdList.stream().map(Arrays::asList).collect(Collectors.toList())); } } @@ -131,13 +132,13 @@ public void testConcurrentCreatingColocateTables() throws InterruptedException { @Test public void testConcurrentlyDropDbAndCreateTable() throws Exception { final String createTableSqlFormat = - "CREATE TABLE IF NOT EXISTS concurrent_test_db.test_tbl_RRR(k1 int, k2 int, k3 int)" + - " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"; + "CREATE TABLE IF NOT EXISTS concurrent_test_db.test_tbl_RRR(k1 int, k2 int, k3 int)" + + " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"; final String createViewSqlFormat = "CREATE VIEW IF NOT EXISTS concurrent_test_db.test_view_RRR" + - " as select k1,k2 from concurrent_test_db.base_t1;"; + " as select k1,k2 from concurrent_test_db.base_t1;"; final String createMVSqlFormat = "CREATE MATERIALIZED VIEW IF NOT EXISTS" + - " concurrent_test_db.test_mv_RRR DISTRIBUTED BY HASH(`k2`) REFRESH MANUAL" + - " as select k2,k3 from concurrent_test_db.base_t1;"; + " concurrent_test_db.test_mv_RRR DISTRIBUTED BY HASH(`k2`) REFRESH MANUAL" + + " as select k2,k3 from concurrent_test_db.base_t1;"; final int NUM_ROUND = 1; @@ -155,16 +156,16 @@ public void testConcurrentlyDropDbAndCreateTable() throws Exception { System.out.println("creating table and db time: " + times); starRocksAssert.withDatabase("concurrent_test_db"); starRocksAssert.withTable( - "CREATE TABLE IF NOT EXISTS concurrent_test_db.base_t1(k1 int, k2 int, k3 int)" + - " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); + "CREATE TABLE IF NOT EXISTS concurrent_test_db.base_t1(k1 int, k2 int, k3 int)" + + " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); int time = 300 + random.nextInt(100); // sleep random time before dropping database Thread.sleep(time); System.out.println("dropping table and db"); - Database db = GlobalStateMgr.getCurrentState().getDb("concurrent_test_db"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("concurrent_test_db"); ShowTableStmt showTableStmt = - (ShowTableStmt) UtFrameUtils.parseStmtWithNewParser( - "show tables from concurrent_test_db", connectContext); + (ShowTableStmt) UtFrameUtils.parseStmtWithNewParser( + "show tables from concurrent_test_db", connectContext); starRocksAssert.dropDatabase("concurrent_test_db"); System.out.println("concurrent_test_db dropped"); } catch (Exception e) { @@ -251,9 +252,9 @@ public void testConcurrentCreateSameTable() throws InterruptedException { System.out.println("start to create table same_tbl"); try { starRocksAssert.withTable("create table test.same_tbl " + - " (id int) duplicate key (id)" + - " distributed by hash(id) buckets 5183 " + - "properties(\"replication_num\"=\"1\", \"colocate_with\"=\"test_cg_001\");"); + " (id int) duplicate key (id)" + + " distributed by hash(id) buckets 5183 " + + "properties(\"replication_num\"=\"1\", \"colocate_with\"=\"test_cg_001\");"); } catch (Exception e) { errorCount.incrementAndGet(); } diff --git a/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java b/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java index 50e0fe0b2f9f6..ca7fb572536f7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/server/LocalMetaStoreTest.java @@ -80,17 +80,17 @@ public static void beforeClass() throws Exception { starRocksAssert = new StarRocksAssert(connectContext); starRocksAssert.withDatabase("test").useDatabase("test") - .withTable( - "CREATE TABLE test.t1(k1 int, k2 int, k3 int)" + - " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); + .withTable( + "CREATE TABLE test.t1(k1 int, k2 int, k3 int)" + + " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); UtFrameUtils.PseudoImage.setUpImageVersion(); } @Test public void testGetNewPartitionsFromPartitions() throws DdlException { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - Table table = db.getTable("t1"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); Assert.assertTrue(table instanceof OlapTable); OlapTable olapTable = (OlapTable) table; Partition sourcePartition = olapTable.getPartition("t1"); @@ -102,8 +102,8 @@ public void testGetNewPartitionsFromPartitions() throws DdlException { Assert.assertEquals(olapTable.getName(), copiedTable.getName()); Set tabletIdSet = Sets.newHashSet(); List newPartitions = localMetastore.getNewPartitionsFromPartitions(db, - olapTable, sourcePartitionIds, origPartitions, copiedTable, "_100", tabletIdSet, tmpPartitionIds, - null, WarehouseManager.DEFAULT_WAREHOUSE_ID); + olapTable, sourcePartitionIds, origPartitions, copiedTable, "_100", tabletIdSet, tmpPartitionIds, + null, WarehouseManager.DEFAULT_WAREHOUSE_ID); Assert.assertEquals(sourcePartitionIds.size(), newPartitions.size()); Assert.assertEquals(1, newPartitions.size()); Partition newPartition = newPartitions.get(0); @@ -112,8 +112,8 @@ public void testGetNewPartitionsFromPartitions() throws DdlException { PartitionInfo partitionInfo = olapTable.getPartitionInfo(); partitionInfo.addPartition(newPartition.getId(), partitionInfo.getDataProperty(sourcePartition.getId()), - partitionInfo.getReplicationNum(sourcePartition.getId()), - partitionInfo.getIsInMemory(sourcePartition.getId())); + partitionInfo.getReplicationNum(sourcePartition.getId()), + partitionInfo.getIsInMemory(sourcePartition.getId())); olapTable.replacePartition("t1", "t1_100"); Assert.assertEquals(newPartition.getId(), olapTable.getPartition("t1").getId()); @@ -122,15 +122,15 @@ public void testGetNewPartitionsFromPartitions() throws DdlException { @Test public void testGetPartitionIdToStorageMediumMap() throws Exception { starRocksAssert.withMaterializedView( - "CREATE MATERIALIZED VIEW test.mv1\n" + - "distributed by hash(k1) buckets 3\n" + - "refresh async\n" + - "properties(\n" + - "'replication_num' = '1'\n" + - ")\n" + - "as\n" + - "select k1,k2 from test.t1;"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + "CREATE MATERIALIZED VIEW test.mv1\n" + + "distributed by hash(k1) buckets 3\n" + + "refresh async\n" + + "properties(\n" + + "'replication_num' = '1'\n" + + ")\n" + + "as\n" + + "select k1,k2 from test.t1;"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); new MockUp() { @Mock public DataProperty getDataProperty(long partitionId) { @@ -141,7 +141,8 @@ public DataProperty getDataProperty(long partitionId) { @Mock public void logModifyPartition(ModifyPartitionInfo info) { Assert.assertNotNull(info); - Assert.assertTrue(db.getTable(info.getTableId()).isOlapTableOrMaterializedView()); + Assert.assertTrue(GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getId(), info.getTableId()) + .isOlapTableOrMaterializedView()); Assert.assertEquals(TStorageMedium.HDD, info.getDataProperty().getStorageMedium()); Assert.assertEquals(DataProperty.MAX_COOLDOWN_TIME_MS, info.getDataProperty().getCooldownTimeMs()); } @@ -156,8 +157,8 @@ public void logModifyPartition(ModifyPartitionInfo info) { @Test public void testLoadClusterV2() throws Exception { LocalMetastore localMetaStore = new LocalMetastore(GlobalStateMgr.getCurrentState(), - GlobalStateMgr.getCurrentState().getRecycleBin(), - GlobalStateMgr.getCurrentState().getColocateTableIndex()); + GlobalStateMgr.getCurrentState().getRecycleBin(), + GlobalStateMgr.getCurrentState().getColocateTableIndex()); UtFrameUtils.PseudoImage image = new UtFrameUtils.PseudoImage(); localMetaStore.save(image.getImageWriter()); @@ -174,16 +175,16 @@ public void testLoadClusterV2() throws Exception { @Test public void testReplayAddSubPartition() throws DdlException { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t1"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); Partition p = table.getPartitions().stream().findFirst().get(); int schemaHash = table.getSchemaHashByIndexId(p.getBaseIndex().getId()); MaterializedIndex index = new MaterializedIndex(); TabletMeta tabletMeta = new TabletMeta(db.getId(), table.getId(), p.getId(), - index.getId(), schemaHash, table.getPartitionInfo().getDataProperty(p.getId()).getStorageMedium()); + index.getId(), schemaHash, table.getPartitionInfo().getDataProperty(p.getId()).getStorageMedium()); index.addTablet(new LocalTablet(0), tabletMeta); PhysicalPartitionPersistInfoV2 info = new PhysicalPartitionPersistInfoV2( - db.getId(), table.getId(), p.getId(), new PhysicalPartitionImpl(123, "", p.getId(), 0, index)); + db.getId(), table.getId(), p.getId(), new PhysicalPartitionImpl(123, "", p.getId(), 0, index)); LocalMetastore localMetastore = connectContext.getGlobalStateMgr().getLocalMetastore(); localMetastore.replayAddSubPartition(info); @@ -191,8 +192,8 @@ public void testReplayAddSubPartition() throws DdlException { @Test public void testReplayTruncateTable() throws DdlException { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t1"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); Partition p = table.getPartitions().stream().findFirst().get(); TruncateTableInfo info = new TruncateTableInfo(db.getId(), table.getId(), Lists.newArrayList(p), false); @@ -202,8 +203,8 @@ public void testReplayTruncateTable() throws DdlException { @Test public void testModifyAutomaticBucketSize() throws DdlException { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t1"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); Locker locker = new Locker(); locker.lockDatabase(db, LockType.WRITE); @@ -221,8 +222,8 @@ public void testModifyAutomaticBucketSize() throws DdlException { @Test public void testCreateTableIfNotExists() throws Exception { // create table if not exists, if the table already exists, do nothing - Database db = connectContext.getGlobalStateMgr().getDb("test"); - Table table = db.getTable("t1"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); Assert.assertTrue(table instanceof OlapTable); LocalMetastore localMetastore = connectContext.getGlobalStateMgr().getLocalMetastore(); @@ -239,20 +240,20 @@ public void testCreateTableIfNotExists() throws Exception { starRocksAssert = new StarRocksAssert(connectContext); // with IF NOT EXIST starRocksAssert.useDatabase("test").withTable( - "CREATE TABLE IF NOT EXISTS test.t1(k1 int, k2 int, k3 int)" + - " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); + "CREATE TABLE IF NOT EXISTS test.t1(k1 int, k2 int, k3 int)" + + " distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); // w/o IF NOT EXIST Assert.assertThrows(AnalysisException.class, () -> - starRocksAssert.useDatabase("test").withTable( - "CREATE TABLE test.t1(k1 int, k2 int, k3 int)" + - " distributed by hash(k1) buckets 3 properties('replication_num' = '1');")); + starRocksAssert.useDatabase("test").withTable( + "CREATE TABLE test.t1(k1 int, k2 int, k3 int)" + + " distributed by hash(k1) buckets 3 properties('replication_num' = '1');")); } @Test public void testAlterTableProperties() throws Exception { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t1"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.PROPERTIES_DATACACHE_PARTITION_DURATION, "abcd"); @@ -277,7 +278,7 @@ public void testRenameColumnException() throws Exception { } Database db = localMetastore.getDb("test"); - Table table = db.getTable("t1"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); try { localMetastore.renameColumn(new Database(1, "_statistics_"), table, null); Assert.fail("should not happen"); diff --git a/fe/fe-core/src/test/java/com/starrocks/server/MVRepairHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/server/MVRepairHandlerTest.java index 589ef9dd0341a..1954d202be409 100644 --- a/fe/fe-core/src/test/java/com/starrocks/server/MVRepairHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/server/MVRepairHandlerTest.java @@ -53,7 +53,7 @@ public static void beforeClass() throws Exception { "distributed by hash(k1) buckets 3 refresh async as select k1 from test.t1"); database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); - table = database.getTable("t1"); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "t1"); Assert.assertTrue(table instanceof OlapTable); OlapTable olapTable = (OlapTable) table; partition = olapTable.getPartition("t1"); diff --git a/fe/fe-core/src/test/java/com/starrocks/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/com/starrocks/service/FrontendServiceImplTest.java index 7812c2299e649..0d6ada7448397 100644 --- a/fe/fe-core/src/test/java/com/starrocks/service/FrontendServiceImplTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/service/FrontendServiceImplTest.java @@ -110,7 +110,7 @@ public class FrontendServiceImplTest { ExecuteEnv exeEnv; private TUpdateResourceUsageRequest genUpdateResourceUsageRequest( - long backendId, int numRunningQueries, long memLimitBytes, long memUsedBytes, int cpuUsedPermille) { + long backendId, int numRunningQueries, long memLimitBytes, long memUsedBytes, int cpuUsedPermille) { TResourceUsage usage = new TResourceUsage(); usage.setNum_running_queries(numRunningQueries); usage.setMem_limit_bytes(memLimitBytes); @@ -129,7 +129,7 @@ public void testUpdateImmutablePartitionException() throws TException { new MockUp() { @Mock public synchronized TImmutablePartitionResult updateImmutablePartitionInternal( - TImmutablePartitionRequest request) { + TImmutablePartitionRequest request) { throw new RuntimeException("test"); } }; @@ -157,132 +157,132 @@ public static void beforeClass() throws Exception { starRocksAssert = new StarRocksAssert(connectContext); starRocksAssert.withDatabase("test").useDatabase("test") - .withTable("CREATE TABLE site_access_auto (\n" + - " event_day DATETIME NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "DISTRIBUTED BY RANDOM\n" + - "PROPERTIES(\n" + - " \"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_exception (\n" + - " event_day DATETIME NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "DISTRIBUTED BY RANDOM\n" + - "PROPERTIES(\n" + - " \"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_empty (\n" + - " event_day DATETIME NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - " \"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_border (\n" + - " event_day DATETIME NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - " \"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_hour (\n" + - " event_day DATETIME,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('hour', event_day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_day (\n" + - " event_day DATE,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day) (\n" + - "START (\"2020-06-01\") END (\"2022-06-05\") EVERY (INTERVAL 1 day)\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_month (\n" + - " event_day DATE,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('month', event_day) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");") - .withTable("CREATE TABLE site_access_slice (\n" + - " event_day datetime,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY time_slice(event_day, interval 5 day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\"replication_num\" = \"1\");") - .withTable("CREATE TABLE site_access_list (\n" + - " event_day DATE not null,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY (event_day) (\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\"\n" + - ");") - .withView("create view v as select * from site_access_empty") - .withView("create view v1 as select current_role()") - .withView("create view v2 as select current_user()") - .withView("create view v3 as select database()") - .withView("create view v4 as select user()") - .withView("create view v5 as select CONNECTION_ID()") - .withView("create view v6 as select CATALOG()") - - .withMaterializedView("create materialized view mv refresh async as select * from site_access_empty"); + .withTable("CREATE TABLE site_access_auto (\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "DISTRIBUTED BY RANDOM\n" + + "PROPERTIES(\n" + + " \"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_exception (\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "DISTRIBUTED BY RANDOM\n" + + "PROPERTIES(\n" + + " \"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_empty (\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + " \"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_border (\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + " \"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_hour (\n" + + " event_day DATETIME,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('hour', event_day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_day (\n" + + " event_day DATE,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day) (\n" + + "START (\"2020-06-01\") END (\"2022-06-05\") EVERY (INTERVAL 1 day)\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_month (\n" + + " event_day DATE,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('month', event_day) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");") + .withTable("CREATE TABLE site_access_slice (\n" + + " event_day datetime,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY time_slice(event_day, interval 5 day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\"replication_num\" = \"1\");") + .withTable("CREATE TABLE site_access_list (\n" + + " event_day DATE not null,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY (event_day) (\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");") + .withView("create view v as select * from site_access_empty") + .withView("create view v1 as select current_role()") + .withView("create view v2 as select current_user()") + .withView("create view v3 as select database()") + .withView("create view v4 as select user()") + .withView("create view v5 as select CONNECTION_ID()") + .withView("create view v6 as select CATALOG()") + + .withMaterializedView("create materialized view mv refresh async as select * from site_access_empty"); } @AfterClass @@ -302,13 +302,14 @@ public static void tearDown() throws Exception { @Test public void testImmutablePartitionException() throws TException { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("site_access_exception"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "site_access_exception"); List partitionIds = Lists.newArrayList(); FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv); TImmutablePartitionRequest request = new TImmutablePartitionRequest(); TImmutablePartitionResult partition = impl.updateImmutablePartition(request); - Table t = db.getTable("v"); + Table t = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "v"); Assert.assertEquals(partition.getStatus().getStatus_code(), TStatusCode.RUNTIME_ERROR); @@ -334,7 +335,7 @@ public void testImmutablePartitionException() throws TException { Assert.assertEquals(partition.getStatus().getStatus_code(), TStatusCode.OK); partitionIds = table.getPhysicalPartitions().stream() - .map(PhysicalPartition::getId).collect(Collectors.toList()); + .map(PhysicalPartition::getId).collect(Collectors.toList()); request.setPartition_ids(partitionIds); partition = impl.updateImmutablePartition(request); Assert.assertEquals(partition.getStatus().getStatus_code(), TStatusCode.OK); @@ -342,10 +343,11 @@ public void testImmutablePartitionException() throws TException { @Test public void testImmutablePartitionApi() throws TException { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) db.getTable("site_access_auto"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(db.getFullName(), "site_access_auto"); List partitionIds = table.getPhysicalPartitions().stream() - .map(PhysicalPartition::getId).collect(Collectors.toList()); + .map(PhysicalPartition::getId).collect(Collectors.toList()); FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv); TImmutablePartitionRequest request = new TImmutablePartitionRequest(); request.setDb_id(db.getId()); @@ -361,7 +363,7 @@ public void testImmutablePartitionApi() throws TException { Assert.assertEquals(2, table.getPhysicalPartitions().size()); partitionIds = table.getPhysicalPartitions().stream() - .map(PhysicalPartition::getId).collect(Collectors.toList()); + .map(PhysicalPartition::getId).collect(Collectors.toList()); request.setPartition_ids(partitionIds); partition = impl.updateImmutablePartition(request); Assert.assertEquals(partition.getStatus().getStatus_code(), TStatusCode.OK); @@ -377,8 +379,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_day"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_day"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1990-04-24"); @@ -408,8 +410,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_day"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_day"); ((OlapTable) table).setState(OlapTable.OlapTableState.SCHEMA_CHANGE); List> partitionValues = Lists.newArrayList(); @@ -437,8 +439,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_day"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_day"); ((OlapTable) table).setState(OlapTable.OlapTableState.ROLLUP); List> partitionValues = Lists.newArrayList(); @@ -457,12 +459,10 @@ public TransactionState getTransactionState(long dbId, long transactionId) { ((OlapTable) table).setState(OlapTable.OlapTableState.NORMAL); } - - @Test public void testCreatePartitionExceedLimit() throws TException { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_day"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_day"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1990-04-24"); @@ -511,8 +511,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_slice"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_slice"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1990-04-24"); @@ -542,8 +542,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_day"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_day"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1990-04-24"); @@ -579,8 +579,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_month"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_month"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1990-04-24"); @@ -622,8 +622,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_border"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_border"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("NULL"); @@ -654,8 +654,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { @Test public void testAutomaticPartitionLimitExceed() throws TException { Config.max_partition_number_per_table = 1; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_slice"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_slice"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1991-04-24"); @@ -685,8 +685,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_month"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_month"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1999-04-29"); @@ -750,19 +750,19 @@ public void testListTableStatus() throws TException { @Test public void testListViewStatusWithBaseTableDropped() throws Exception { starRocksAssert.useDatabase("test") - .withTable("CREATE TABLE site_access_empty_for_view (\n" + - " event_day DATETIME NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('day', event_day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id)\n" + - "PROPERTIES(\n" + - " \"replication_num\" = \"1\"\n" + - ");"); + .withTable("CREATE TABLE site_access_empty_for_view (\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('day', event_day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id)\n" + + "PROPERTIES(\n" + + " \"replication_num\" = \"1\"\n" + + ");"); starRocksAssert.withView("create view test.view11 as select * from test.site_access_empty_for_view"); // drop the base table referenced by test.view11 starRocksAssert.dropTable("test.site_access_empty_for_view"); @@ -782,8 +782,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_hour"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_hour"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1990-04-24 12:34:56"); @@ -806,8 +806,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { @Test public void testCreateEmptyPartition() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_empty"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_empty"); Collection partitions = table.getPartitions(); Assert.assertEquals(1, partitions.size()); String name = partitions.iterator().next().getName(); @@ -818,125 +818,125 @@ public void testCreateEmptyPartition() { @Test(expected = AnalysisException.class) public void testCreateCeilForbidAutomaticTable() throws Exception { starRocksAssert.withDatabase("test2").useDatabase("test2") - .withTable("CREATE TABLE site_access_ceil (\n" + - " event_day datetime,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY time_slice(event_day, interval 1 day, CEIL) \n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\"replication_num\" = \"1\");"); + .withTable("CREATE TABLE site_access_ceil (\n" + + " event_day datetime,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY time_slice(event_day, interval 1 day, CEIL) \n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\"replication_num\" = \"1\");"); } @Test(expected = AnalysisException.class) public void testCreateTimeSliceForbidAutomaticTable() throws Exception { starRocksAssert.withDatabase("test2").useDatabase("test2") - .withTable("CREATE TABLE site_access_time_slice_hour_date (\n" + - " event_day date,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY time_slice(event_day, interval 1 hour) \n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\"replication_num\" = \"1\");"); + .withTable("CREATE TABLE site_access_time_slice_hour_date (\n" + + " event_day date,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY time_slice(event_day, interval 1 hour) \n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\"replication_num\" = \"1\");"); } @Test(expected = AnalysisException.class) public void testCreateDateTruncForbidAutomaticTable() throws Exception { starRocksAssert.withDatabase("test2").useDatabase("test2") - .withTable("CREATE TABLE site_access_date_trunc_hour_date (\n" + - " event_day DATE,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ")\n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('hour', event_day)\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\"replication_num\" = \"1\");"); + .withTable("CREATE TABLE site_access_date_trunc_hour_date (\n" + + " event_day DATE,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('hour', event_day)\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\"replication_num\" = \"1\");"); } @Test(expected = AnalysisException.class) public void testUnsupportedAutomaticTableGranularityDoesNotMatch() throws Exception { starRocksAssert.withDatabase("test2").useDatabase("test2") - .withTable("CREATE TABLE site_access_granularity_does_not_match(\n" + - " event_day DATE NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ") \n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('month', event_day)(\n" + - " START (\"2023-05-01\") END (\"2023-05-03\") EVERY (INTERVAL 1 day)\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\n" + - " \"partition_live_number\" = \"3\",\n" + - " \"replication_num\" = \"1\"\n" + - ");"); + .withTable("CREATE TABLE site_access_granularity_does_not_match(\n" + + " event_day DATE NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ") \n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('month', event_day)(\n" + + " START (\"2023-05-01\") END (\"2023-05-03\") EVERY (INTERVAL 1 day)\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\n" + + " \"partition_live_number\" = \"3\",\n" + + " \"replication_num\" = \"1\"\n" + + ");"); } @Test(expected = AnalysisException.class) public void testUnsupportedAutomaticTableGranularityDoesNotMatch2() throws Exception { starRocksAssert.withDatabase("test2").useDatabase("test2") - .withTable("CREATE TABLE site_access_granularity_does_not_match2(\n" + - " event_day DATE NOT NULL,\n" + - " site_id INT DEFAULT '10',\n" + - " city_code VARCHAR(100),\n" + - " user_name VARCHAR(32) DEFAULT '',\n" + - " pv BIGINT DEFAULT '0'\n" + - ") \n" + - "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + - "PARTITION BY date_trunc('month', event_day)(\n" + - " START (\"2022-05-01\") END (\"2022-05-03\") EVERY (INTERVAL 1 day),\n" + - " START (\"2023-05-01\") END (\"2023-05-03\") EVERY (INTERVAL 1 day)\n" + - ")\n" + - "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + - "PROPERTIES(\n" + - " \"partition_live_number\" = \"3\",\n" + - " \"replication_num\" = \"1\"\n" + - ");"); + .withTable("CREATE TABLE site_access_granularity_does_not_match2(\n" + + " event_day DATE NOT NULL,\n" + + " site_id INT DEFAULT '10',\n" + + " city_code VARCHAR(100),\n" + + " user_name VARCHAR(32) DEFAULT '',\n" + + " pv BIGINT DEFAULT '0'\n" + + ") \n" + + "DUPLICATE KEY(event_day, site_id, city_code, user_name)\n" + + "PARTITION BY date_trunc('month', event_day)(\n" + + " START (\"2022-05-01\") END (\"2022-05-03\") EVERY (INTERVAL 1 day),\n" + + " START (\"2023-05-01\") END (\"2023-05-03\") EVERY (INTERVAL 1 day)\n" + + ")\n" + + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 32\n" + + "PROPERTIES(\n" + + " \"partition_live_number\" = \"3\",\n" + + " \"replication_num\" = \"1\"\n" + + ");"); } @Test public void testGetTablesInfo() throws Exception { starRocksAssert.withDatabase("test_table").useDatabase("test_table") - .withTable("CREATE TABLE `t1` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `v1` int(11) NULL COMMENT \"\",\n" + - " `v2` int(11) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"LZ4\"\n" + - ")") - .withTable("CREATE TABLE `t2` (\n" + - " `k1` date NULL COMMENT \"\",\n" + - " `v1` int(11) NULL COMMENT \"\",\n" + - " `v2` int(11) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`k1`)\n" + - "COMMENT \"OLAP\"\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"LZ4\"\n" + - ")"); + .withTable("CREATE TABLE `t1` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `v1` int(11) NULL COMMENT \"\",\n" + + " `v2` int(11) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"LZ4\"\n" + + ")") + .withTable("CREATE TABLE `t2` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `v1` int(11) NULL COMMENT \"\",\n" + + " `v2` int(11) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT \"OLAP\"\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"LZ4\"\n" + + ")"); ConnectContext ctx = starRocksAssert.getCtx(); String createUserSql = "create user test1"; @@ -963,19 +963,19 @@ public void testGetTablesInfo() throws Exception { @Test public void testDefaultValueMeta() throws Exception { starRocksAssert.withDatabase("test_table").useDatabase("test_table") - .withTable("CREATE TABLE `test_default_value` (\n" + - " `id` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT \"\",\n" + - " `value` int(11) NULL DEFAULT \"2\" COMMENT \"\"\n" + - ") ENGINE=OLAP \n" + - "DUPLICATE KEY(`id`, `value`)\n" + - "DISTRIBUTED BY RANDOM\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"LZ4\"\n" + - ");"); + .withTable("CREATE TABLE `test_default_value` (\n" + + " `id` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT \"\",\n" + + " `value` int(11) NULL DEFAULT \"2\" COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`id`, `value`)\n" + + "DISTRIBUTED BY RANDOM\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"LZ4\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String createUserSql = "create user test2"; @@ -993,8 +993,8 @@ public void testDefaultValueMeta() throws Exception { TDescribeTableResult response = impl.describeTable(request); List columnDefList = response.getColumns(); List testDefaultValue = columnDefList.stream() - .filter(u -> u.getColumnDesc().getTableName().equalsIgnoreCase("test_default_value")) - .collect(Collectors.toList()); + .filter(u -> u.getColumnDesc().getTableName().equalsIgnoreCase("test_default_value")) + .collect(Collectors.toList()); Assert.assertEquals(2, testDefaultValue.size()); Assert.assertEquals("CURRENT_TIMESTAMP", testDefaultValue.get(0).getColumnDesc().getColumnDefault()); Assert.assertEquals("2", testDefaultValue.get(1).getColumnDesc().getColumnDefault()); @@ -1003,18 +1003,18 @@ public void testDefaultValueMeta() throws Exception { @Test public void testGetSpecialColumn() throws Exception { starRocksAssert.withDatabase("test_table").useDatabase("test_table") - .withTable("CREATE TABLE `ye$test` (\n" + - "event_day DATE,\n" + - "department_id int(11) NOT NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "PRIMARY KEY(event_day, department_id)\n" + - "DISTRIBUTED BY HASH(department_id) BUCKETS 1\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"storage_format\" = \"DEFAULT\",\n" + - "\"enable_persistent_index\" = \"false\"\n" + - ");"); + .withTable("CREATE TABLE `ye$test` (\n" + + "event_day DATE,\n" + + "department_id int(11) NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "PRIMARY KEY(event_day, department_id)\n" + + "DISTRIBUTED BY HASH(department_id) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"DEFAULT\",\n" + + "\"enable_persistent_index\" = \"false\"\n" + + ");"); ConnectContext ctx = starRocksAssert.getCtx(); String createUserSql = "create user test3"; @@ -1038,19 +1038,19 @@ public void testGetSpecialColumn() throws Exception { @Test public void testGetSpecialColumnForSyncMv() throws Exception { starRocksAssert.withDatabase("test_table").useDatabase("test_table") - .withTable("CREATE TABLE `base1` (\n" + - "event_day DATE,\n" + - "department_id int(11) NOT NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(event_day, department_id)\n" + - "DISTRIBUTED BY HASH(department_id) BUCKETS 1\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"storage_format\" = \"DEFAULT\",\n" + - "\"enable_persistent_index\" = \"false\"\n" + - ");") - .withMaterializedView("create materialized view test_table.mv$test as select event_day from base1"); + .withTable("CREATE TABLE `base1` (\n" + + "event_day DATE,\n" + + "department_id int(11) NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(event_day, department_id)\n" + + "DISTRIBUTED BY HASH(department_id) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"DEFAULT\",\n" + + "\"enable_persistent_index\" = \"false\"\n" + + ");") + .withMaterializedView("create materialized view test_table.mv$test as select event_day from base1"); ConnectContext ctx = starRocksAssert.getCtx(); String createUserSql = "create user test4"; @@ -1074,14 +1074,14 @@ public void testGetSpecialColumnForSyncMv() throws Exception { @Test public void testGetLoadTxnStatus() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_day"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_day"); UUID uuid = UUID.randomUUID(); TUniqueId requestId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); long transactionId = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr().beginTransaction(db.getId(), - Lists.newArrayList(table.getId()), "1jdc689-xd232", requestId, - new TxnCoordinator(TxnSourceType.BE, "1.1.1.1"), - TransactionState.LoadJobSourceType.BACKEND_STREAMING, -1, 600); + Lists.newArrayList(table.getId()), "1jdc689-xd232", requestId, + new TxnCoordinator(TxnSourceType.BE, "1.1.1.1"), + TransactionState.LoadJobSourceType.BACKEND_STREAMING, -1, 600); FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv); TGetLoadTxnStatusRequest request = new TGetLoadTxnStatusRequest(); request.setDb("non-exist-db"); @@ -1123,9 +1123,9 @@ public void testStreamLoadPutColumnMapException() { List errMsg = status.getError_msgs(); Assert.assertEquals(1, errMsg.size()); Assert.assertEquals( - "Expr 'str_to_date(`col1`)' analyze error: No matching function with signature: str_to_date(varchar), " + - "derived column is 'event_day'", - errMsg.get(0)); + "Expr 'str_to_date(`col1`)' analyze error: No matching function with signature: str_to_date(varchar), " + + "derived column is 'event_day'", + errMsg.get(0)); } @Test @@ -1227,8 +1227,8 @@ public TransactionState getTransactionState(long dbId, long transactionId) { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = db.getTable("site_access_list"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "site_access_list"); List> partitionValues = Lists.newArrayList(); List values = Lists.newArrayList(); values.add("1990-04-24"); @@ -1244,25 +1244,26 @@ public TransactionState getTransactionState(long dbId, long transactionId) { TCreatePartitionResult partition = impl.createPartition(request); GlobalStateMgr currentState = GlobalStateMgr.getCurrentState(); - Database testDb = currentState.getDb("test"); - OlapTable olapTable = (OlapTable) testDb.getTable("site_access_list"); + Database testDb = currentState.getLocalMetastore().getDb("test"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "site_access_list"); PartitionInfo partitionInfo = olapTable.getPartitionInfo(); DistributionInfo defaultDistributionInfo = olapTable.getDefaultDistributionInfo(); List partitionDescs = Lists.newArrayList(); Partition p19910425 = olapTable.getPartition("p19900425"); partitionDescs.add(new ListPartitionDesc(Lists.newArrayList("p19900425"), - Lists.newArrayList(new SingleItemListPartitionDesc(true, "p19900425", - Lists.newArrayList("1990-04-25"), Maps.newHashMap())))); + Lists.newArrayList(new SingleItemListPartitionDesc(true, "p19900425", + Lists.newArrayList("1990-04-25"), Maps.newHashMap())))); AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDescs.get(0), - defaultDistributionInfo.toDistributionDesc(table.getIdToColumn()), Maps.newHashMap(), false); + defaultDistributionInfo.toDistributionDesc(table.getIdToColumn()), Maps.newHashMap(), false); List partitionList = Lists.newArrayList(); partitionList.add(p19910425); currentState.getLocalMetastore().addListPartitionLog(testDb, olapTable, partitionDescs, - addPartitionClause.isTempPartition(), partitionInfo, partitionList, Sets.newSet("p19900425")); + addPartitionClause.isTempPartition(), partitionInfo, partitionList, Sets.newSet("p19900425")); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeCreateAnalyzeJobTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeCreateAnalyzeJobTest.java index c1655a24dbbe8..4acb50aedd68a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeCreateAnalyzeJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeCreateAnalyzeJobTest.java @@ -18,6 +18,7 @@ import com.starrocks.catalog.Database; import com.starrocks.catalog.Table; import com.starrocks.qe.DDLStmtExecutor; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.CreateAnalyzeJobStmt; import com.starrocks.sql.plan.ConnectorPlanTestBase; import com.starrocks.statistic.StatsConstants; @@ -65,7 +66,7 @@ public void testAllTable() throws Exception { String sql = "create analyze full database db"; CreateAnalyzeJobStmt analyzeStmt = (CreateAnalyzeJobStmt) analyzeSuccess(sql); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("db"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("db"); Assert.assertEquals(db.getId(), analyzeStmt.getDbId()); Assert.assertEquals(StatsConstants.DEFAULT_ALL_ID, analyzeStmt.getTableId()); Assert.assertTrue(analyzeStmt.getColumnNames().isEmpty()); @@ -76,9 +77,9 @@ public void testColumn() throws Exception { String sql = "create analyze table db.tbl(kk1, kk2)"; CreateAnalyzeJobStmt analyzeStmt = (CreateAnalyzeJobStmt) analyzeSuccess(sql); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("db"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("db"); Assert.assertEquals(db.getId(), analyzeStmt.getDbId()); - Table table = db.getTable("tbl"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl"); Assert.assertEquals(table.getId(), analyzeStmt.getTableId()); Assert.assertEquals(2, analyzeStmt.getColumnNames().size()); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeInsertTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeInsertTest.java index b93c2a9d17846..76a01970038bc 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeInsertTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeInsertTest.java @@ -101,7 +101,7 @@ public void testInsert() { @Test public void testInsertOverwriteWhenSchemaChange() throws Exception { OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState() - .getDb("test").getTable("t0"); + .getLocalMetastore().getDb("test").getTable("t0"); table.setState(OlapTable.OlapTableState.SCHEMA_CHANGE); analyzeFail("insert overwrite t0 select * from t0;", "table state is SCHEMA_CHANGE, please wait to insert overwrite until table state is normal"); @@ -114,11 +114,15 @@ public void testInsertIcebergUnpartitionedTable(@Mocked IcebergTable icebergTabl "Unknown catalog 'err_catalog'"); MetadataMgr metadata = AnalyzeTestUtil.getConnectContext().getGlobalStateMgr().getMetadataMgr(); - new MockUp() { + + new MockUp() { @Mock - public Database getDatabase(String catalogName, String tableName) { + public Database getDb(String catalogName, String dbName) { return new Database(); } + }; + + new MockUp() { @Mock public Table getSessionAwareTable(ConnectContext context, Database database, TableName tableName) { return null; @@ -135,6 +139,10 @@ public Table getSessionAwareTable(ConnectContext context, Database database, Tab }; new Expectations(metadata) { { + metadata.getDb(anyString, anyString); + minTimes = 0; + result = new Database(); + icebergTable.supportInsert(); result = true; minTimes = 0; @@ -145,6 +153,10 @@ public Table getSessionAwareTable(ConnectContext context, Database database, Tab new Expectations(metadata) { { + metadata.getDb(anyString, anyString); + minTimes = 0; + result = new Database(); + icebergTable.getBaseSchema(); result = ImmutableList.of(new Column("c1", Type.INT)); minTimes = 0; @@ -157,12 +169,14 @@ public Table getSessionAwareTable(ConnectContext context, Database database, Tab public void testPartitionedIcebergTable(@Mocked IcebergTable icebergTable) { MetadataMgr metadata = AnalyzeTestUtil.getConnectContext().getGlobalStateMgr().getMetadataMgr(); - new MockUp() { + new MockUp() { @Mock - public Database getDatabase(String catalogName, String databaseName) { + public Database getDb(String catalogName, String dbName) { return new Database(); } + }; + new MockUp() { @Mock public Table getSessionAwareTable(ConnectContext context, Database database, TableName tableName) { return icebergTable; @@ -171,6 +185,10 @@ public Table getSessionAwareTable(ConnectContext context, Database database, Tab new Expectations(metadata) { { + metadata.getDb(anyString, anyString); + minTimes = 0; + result = new Database(); + icebergTable.supportInsert(); result = true; minTimes = 0; @@ -241,11 +259,14 @@ public Table getSessionAwareTable(ConnectContext context, Database database, Tab @Test public void testInsertHiveNonManagedTable(@Mocked HiveTable hiveTable) { - new MockUp() { + new MockUp() { @Mock - public Database getDatabase(String catalogName, String databaseName) { - return null; + public Database getDb(String catalogName, String dbName) { + return new Database(); } + }; + + new MockUp() { @Mock public Table getSessionAwareTable(ConnectContext conntext, Database database, TableName tableName) { return hiveTable; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java index c462783e5ad08..7f783507de7ac 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.sql.analyzer; import com.google.common.collect.Lists; @@ -160,7 +159,7 @@ public void testStructColumns() { } @Test - public void testAnalyzeHiveTable() { + public void testAnalyzeHiveTable() { String sql = "analyze table hive0.tpch.customer"; AnalyzeStmt analyzeStmt = (AnalyzeStmt) analyzeSuccess(sql); @@ -209,8 +208,8 @@ public void testProperties() { public void testShow() throws MetaNotFoundException { String sql = "show analyze"; ShowAnalyzeJobStmt showAnalyzeJobStmt = (ShowAnalyzeJobStmt) analyzeSuccess(sql); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = testDb.getTable("t0"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "t0"); NativeAnalyzeJob nativeAnalyzeJob = new NativeAnalyzeJob(testDb.getId(), table.getId(), Lists.newArrayList(), Lists.newArrayList(), @@ -237,7 +236,7 @@ public void testShow() throws MetaNotFoundException { AnalyzeStatus analyzeStatus = new NativeAnalyzeStatus(-1, testDb.getId(), table.getId(), Lists.newArrayList(), StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.ONCE, Maps.newHashMap(), LocalDateTime.of( - 2020, 1, 1, 1, 1)); + 2020, 1, 1, 1, 1)); analyzeStatus.setEndTime(LocalDateTime.of(2020, 1, 1, 1, 1)); analyzeStatus.setStatus(StatsConstants.ScheduleStatus.FAILED); analyzeStatus.setReason("Test Failed"); @@ -264,7 +263,7 @@ public void testShow() throws MetaNotFoundException { ShowHistogramStatsMetaStmt showHistogramStatsMetaStmt = (ShowHistogramStatsMetaStmt) analyzeSuccess(sql); HistogramStatsMeta histogramStatsMeta = new HistogramStatsMeta(testDb.getId(), table.getId(), "v1", StatsConstants.AnalyzeType.HISTOGRAM, LocalDateTime.of( - 2020, 1, 1, 1, 1), + 2020, 1, 1, 1, 1), Maps.newHashMap()); Assert.assertEquals("[test, t0, v1, HISTOGRAM, 2020-01-01 01:01:00, {}]", ShowHistogramStatsMetaStmt.showHistogramStatsMeta(getConnectContext(), histogramStatsMeta).toString()); @@ -272,8 +271,8 @@ public void testShow() throws MetaNotFoundException { @Test public void testStatisticsSqlBuilder() throws Exception { - Database database = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable table = (OlapTable) database.getTable("t0"); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "t0"); System.out.println(table.getPartitions()); Partition partition = (new ArrayList<>(table.getPartitions())).get(0); @@ -292,9 +291,9 @@ public void testStatisticsSqlBuilder() throws Exception { Lists.newArrayList("v1", "v2"), Lists.newArrayList(v1.getType(), v2.getType()))); Assert.assertEquals(String.format( - "SELECT cast(1 as INT), update_time, db_id, table_id, column_name, row_count, " + - "data_size, distinct_count, null_count, max, min " + - "FROM table_statistic_v1 WHERE db_id = %d and table_id = %d and column_name in ('v1', 'v2')", + "SELECT cast(1 as INT), update_time, db_id, table_id, column_name, row_count, " + + "data_size, distinct_count, null_count, max, min " + + "FROM table_statistic_v1 WHERE db_id = %d and table_id = %d and column_name in ('v1', 'v2')", database.getId(), table.getId()), StatisticSQLBuilder.buildQuerySampleStatisticsSQL(database.getId(), table.getId(), Lists.newArrayList("v1", "v2"))); @@ -322,7 +321,7 @@ public void testHistogram() { @Test public void testHistogramSampleRatio() { OlapTable t0 = (OlapTable) starRocksAssert.getCtx().getGlobalStateMgr() - .getDb("db").getTable("tbl"); + .getLocalMetastore().getDb("db").getTable("tbl"); for (Partition partition : t0.getAllPartitions()) { partition.getBaseIndex().setRowCount(10000); } @@ -388,38 +387,39 @@ public void testDropAnalyzeTest() { @Test public void testAnalyzeStatus() throws MetaNotFoundException { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = testDb.getTable("t0"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "t0"); AnalyzeStatus analyzeStatus = new NativeAnalyzeStatus(-1, testDb.getId(), table.getId(), Lists.newArrayList(), StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.ONCE, Maps.newHashMap(), LocalDateTime.of( - 2020, 1, 1, 1, 1)); + 2020, 1, 1, 1, 1)); analyzeStatus.setEndTime(LocalDateTime.of(2020, 1, 1, 1, 1)); analyzeStatus.setStatus(StatsConstants.ScheduleStatus.RUNNING); Assert.assertEquals( "[-1, default_catalog.test, t0, ALL, FULL, ONCE, RUNNING (0%), 2020-01-01 01:01:00, 2020-01-01 01:01:00," + - " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); + " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); analyzeStatus.setProgress(50); Assert.assertEquals( "[-1, default_catalog.test, t0, ALL, FULL, ONCE, RUNNING (50%), 2020-01-01 01:01:00, 2020-01-01 01:01:00," + - " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); + " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); analyzeStatus.setStatus(StatsConstants.ScheduleStatus.FINISH); Assert.assertEquals( "[-1, default_catalog.test, t0, ALL, FULL, ONCE, SUCCESS, 2020-01-01 01:01:00, 2020-01-01 01:01:00," + - " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); + " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); analyzeStatus.setStatus(StatsConstants.ScheduleStatus.FAILED); Assert.assertEquals( "[-1, default_catalog.test, t0, ALL, FULL, ONCE, FAILED, 2020-01-01 01:01:00, 2020-01-01 01:01:00," + - " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); + " {}, ]", ShowAnalyzeStatusStmt.showAnalyzeStatus(getConnectContext(), analyzeStatus).toString()); } @Test public void testObjectColumns() { - Database database = GlobalStateMgr.getCurrentState().getDb("db"); - OlapTable table = (OlapTable) database.getTable("tb2"); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "tb2"); Column kk1 = table.getColumn("kk1"); Column kk2 = table.getColumn("kk2"); @@ -447,7 +447,7 @@ public void testQueryDict() throws Exception { String tblName = "insert"; String sql = "select cast(" + 1 + " as Int), " + "cast(" + 2 + " as bigint), " + - "dict_merge(" + StatisticUtils.quoting(column) + ") as _dict_merge_" + column + + "dict_merge(" + StatisticUtils.quoting(column) + ") as _dict_merge_" + column + " from " + StatisticUtils.quoting(catalogName, dbName, tblName) + " [_META_]"; QueryStatement stmt = (QueryStatement) UtFrameUtils.parseStmtWithNewParserNotIncludeAnalyzer(sql, getConnectContext()); Assert.assertEquals("select.insert", diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/CTASAnalyzerTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/CTASAnalyzerTest.java index a06675f96db48..1ba79c8113a46 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/CTASAnalyzerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/CTASAnalyzerTest.java @@ -215,7 +215,7 @@ public void testSelectColumn() throws Exception { String sql = "create table t2 as select k1 as a,k2 as b from duplicate_table_with_null t2;"; StatisticStorage storage = new CachedStatisticStorage(); - Table table = ctx.getGlobalStateMgr().getDb("ctas") + Table table = ctx.getGlobalStateMgr().getLocalMetastore().getDb("ctas") .getTable("duplicate_table_with_null"); ColumnStatistic k1cs = new ColumnStatistic(1.5928416E9, 1.5982848E9, 1.5256461111280627E-4, 4.0, 64.0); @@ -367,7 +367,7 @@ public void testCTASView() throws Exception { @Test public void testCTASReplicaNum() throws Exception { ConnectContext ctx = starRocksAssert.getCtx(); - Table table = ctx.getGlobalStateMgr().getDb("ctas") + Table table = ctx.getGlobalStateMgr().getLocalMetastore().getDb("ctas") .getTable("duplicate_table_with_null"); OlapTable olapTable = (OlapTable) table; olapTable.setReplicationNum((short) 3); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java index ce47de0779810..ecd2557440f8a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzerTest.java @@ -30,6 +30,7 @@ import com.starrocks.common.Pair; import com.starrocks.qe.ShowExecutor; import com.starrocks.qe.ShowResultSet; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.ShowStmt; import com.starrocks.sql.plan.ConnectorPlanTestBase; import com.starrocks.utframe.StarRocksAssert; @@ -240,8 +241,8 @@ public void testCreateIcebergTable() throws Exception { @Test public void testRefreshMaterializedView() throws Exception { analyzeSuccess("refresh materialized view mv"); - Database testDb = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table = testDb.getTable("mv"); + Database testDb = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "mv"); Assert.assertNotNull(table); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/PrivilegeCheckerTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/PrivilegeCheckerTest.java index cfc616929b16a..2153e5194b13d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/PrivilegeCheckerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/PrivilegeCheckerTest.java @@ -787,7 +787,7 @@ public void testShowAnalyzeJobStatement() throws Exception { grantRevokeSqlAsRoot("revoke SELECT,INSERT on db3.tbl1 from test"); GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db1 = globalStateMgr.getDb("db1"); + Database db1 = globalStateMgr.getLocalMetastore().getDb("db1"); nativeAnalyzeJob = new NativeAnalyzeJob(db1.getId(), -1, Lists.newArrayList(), Lists.newArrayList(), StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.ONCE, Maps.newHashMap(), StatsConstants.ScheduleStatus.FINISH, LocalDateTime.MIN); @@ -832,7 +832,7 @@ public void testShowAnalyzeJobStatement() throws Exception { new StmtExecutor(ctx, new KillAnalyzeStmt(0L)).checkPrivilegeForKillAnalyzeStmt(ctx, nativeAnalyzeJob.getId()); grantRevokeSqlAsRoot("revoke SELECT,INSERT on db1.tbl1 from test"); - Database db2 = globalStateMgr.getDb("db2"); + Database db2 = globalStateMgr.getLocalMetastore().getDb("db2"); tbl1 = db2.getTable("tbl1"); nativeAnalyzeJob = new NativeAnalyzeJob(db2.getId(), tbl1.getId(), Lists.newArrayList(), Lists.newArrayList(), StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.ONCE, Maps.newHashMap(), @@ -854,7 +854,7 @@ public void testShowAnalyzeStatusStatement() throws Exception { AnalyzeMgr analyzeManager = GlobalStateMgr.getCurrentState().getAnalyzeMgr(); GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db1 = globalStateMgr.getDb("db1"); + Database db1 = globalStateMgr.getLocalMetastore().getDb("db1"); Table tbl1 = db1.getTable("tbl1"); AnalyzeStatus analyzeStatus = new NativeAnalyzeStatus(1, db1.getId(), tbl1.getId(), @@ -898,7 +898,7 @@ public void testShowAnalyzeStatusStatement() throws Exception { // can show result for stats on table that user has any privilege on Assert.assertNotNull(showResult); - Database db2 = globalStateMgr.getDb("db2"); + Database db2 = globalStateMgr.getLocalMetastore().getDb("db2"); tbl1 = db2.getTable("tbl1"); analyzeStatus = new NativeAnalyzeStatus(1, db2.getId(), tbl1.getId(), Lists.newArrayList(), StatsConstants.AnalyzeType.FULL, @@ -1674,7 +1674,7 @@ public void testDatabaseStmt() throws Exception { "Access denied"); // Test `use database` : check any privilege on any function in db - Database db1 = GlobalStateMgr.getCurrentState().getDb("db1"); + Database db1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1"); FunctionName fn = FunctionName.createFnName("db1.my_udf_json_get"); Function function = new Function(fn, Arrays.asList(Type.STRING, Type.STRING), Type.STRING, false); try { @@ -1861,7 +1861,7 @@ public void testShowTabletStmt() throws Exception { List replicas = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getReplicasByTabletId(tabletId); Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1"); - Table table = db.getTable("tbl1"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); ReplicasProcNode replicasProcNode = new ReplicasProcNode(db, (OlapTable) table, tabletId, replicas); try { replicasProcNode.fetchResult(); @@ -1892,7 +1892,7 @@ public void testCheckPrivForCurrUserInTabletCtx() throws Exception { // test user has `OPERATE` privilege grantRevokeSqlAsRoot("grant OPERATE on SYSTEM to test"); - Table table = db.getTable("tbl1"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tbl1"); tabletSchedCtx = new TabletSchedCtx(TabletSchedCtx.Type.REPAIR, db.getId(), table.getId(), 3, 4, 1000, System.currentTimeMillis()); result = tabletSchedCtx.checkPrivForCurrUser(testUser); @@ -2787,7 +2787,7 @@ public void analyze(CreateFunctionStmt stmt, ConnectContext context) { @Test public void testFunc() throws Exception { - Database db1 = GlobalStateMgr.getCurrentState().getDb("db1"); + Database db1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1"); FunctionName fn = FunctionName.createFnName("db1.my_udf_json_get"); Function function = new Function(fn, Arrays.asList(Type.STRING, Type.STRING), Type.STRING, false); try { @@ -2900,7 +2900,7 @@ public void testDropGlobalFunc() throws Exception { @Test public void testShowFunc() throws Exception { - Database db1 = GlobalStateMgr.getCurrentState().getDb("db1"); + Database db1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("db1"); FunctionName fn = FunctionName.createFnName("db1.my_udf_json_get"); Function function = new Function(fn, Arrays.asList(Type.STRING, Type.STRING), Type.STRING, false); try { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/common/MetaUtilTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/common/MetaUtilTest.java index a809711bdea84..97c1c8eff79e3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/common/MetaUtilTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/common/MetaUtilTest.java @@ -72,8 +72,8 @@ public static void beforeClass() throws Exception { @Test public void testIsPartitionExist() { - Database database = GlobalStateMgr.getCurrentState().getDb("test"); - Table table = database.getTable("t0"); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "t0"); List partitionList = new ArrayList<>(table.getPartitions()); Assert.assertFalse(MetaUtils.isPartitionExist(GlobalStateMgr.getCurrentState(), -1, -1, -1)); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/common/SqlWithIdUtilsTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/common/SqlWithIdUtilsTest.java index 49071af24f49a..655aa919a1fc4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/common/SqlWithIdUtilsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/common/SqlWithIdUtilsTest.java @@ -75,7 +75,7 @@ public static void beforeClass() throws Exception { @Test public void testDecodeAndEncode() { - Database test = GlobalStateMgr.getCurrentState().getDb("test"); + Database test = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Table tbl1 = test.getTable("tbl1"); Table tbl2 = test.getTable("tbl2"); System.out.println(test.getId() + " " + tbl1.getId() + " " + tbl2.getId()); @@ -128,7 +128,7 @@ public void testDecodeAndEncodeHasAlias() { @Test public void testDecodeAndEncodeNoDataBase() throws Exception { String sql = "select tbl1.k1, tbl2.k2 from test.tbl1 join test.tbl2 on tbl1.k1 = tbl2.k1"; - Database test = GlobalStateMgr.getCurrentState().getDb("test"); + Database test = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); try { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, connectContext); String encode = SqlWithIdUtils.encode(statementBase, connectContext); @@ -171,7 +171,7 @@ public void testDecodeAndEncodeNoDataBase() throws Exception { @Test public void testDecodeAndEncodeNoTable() throws Exception { String sql = "select tbl1.k1, tbl2.k2 from test.tbl1 join test.tbl2 on tbl1.k1 = tbl2.k1"; - Database test = GlobalStateMgr.getCurrentState().getDb("test"); + Database test = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); Table tbl1 = test.getTable("tbl1"); Table tbl2 = test.getTable("tbl2"); try { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java index d1bc9e1d2e75b..6697579d58ec6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java @@ -261,7 +261,7 @@ public void compoundAnd3() { public void unknownStats1() { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 10); GlobalStateMgr.getCurrentState().getStatisticStorage().addColumnStatistic(t0, "v1", new ColumnStatistic(1, 1, 0, 1, 1)); @@ -289,7 +289,7 @@ public void unknownStats1() { @Test public void unknownStats2() { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); OptExpression opt = new OptExpression( new LogicalOlapScanOperator(t1, Maps.newHashMap(), Maps.newHashMap(), null, -1, null)); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/operator/ColumnFilterConverterTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/operator/ColumnFilterConverterTest.java index 952af4ae70f83..cc58fe577b6d5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/operator/ColumnFilterConverterTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/operator/ColumnFilterConverterTest.java @@ -264,7 +264,7 @@ public void testConvertPredicate() { argument.add(constantOperator); ScalarOperator predicate = new BinaryPredicateOperator(BinaryType.EQ, argument); - Table table = GlobalStateMgr.getCurrentState().getDb("test").getTable("bill_detail"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("bill_detail"); ExpressionRangePartitionInfoV2 partitionInfo = (ExpressionRangePartitionInfoV2) ((OlapTable) table).getPartitionInfo(); ScalarOperator afterConvert = ColumnFilterConverter.convertPredicate(predicate, partitionInfo, table.getIdToColumn()); Assert.assertEquals(2921712368984L, ((ConstantOperator) afterConvert.getChild(1)).getValue()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rewrite/PredicateReorderRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rewrite/PredicateReorderRuleTest.java index 91bb240937640..417718b36ac8f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rewrite/PredicateReorderRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rewrite/PredicateReorderRuleTest.java @@ -139,8 +139,8 @@ public static void beforeClass() throws Exception { GlobalStateMgr catalog = GlobalStateMgr.getCurrentState(); CachedStatisticStorage cachedStatisticStorage = new CachedStatisticStorage(); - OlapTable t0 = (OlapTable) catalog.getDb("test").getTable("t0"); - OlapTable t1 = (OlapTable) catalog.getDb("test").getTable("t1"); + OlapTable t0 = (OlapTable) catalog.getLocalMetastore().getDb("test").getTable("t0"); + OlapTable t1 = (OlapTable) catalog.getLocalMetastore().getDb("test").getTable("t1"); cachedStatisticStorage.addColumnStatistic(t0, v1.getName(), statistics.getColumnStatistic(v1)); cachedStatisticStorage.addColumnStatistic(t0, v2.getName(), statistics.getColumnStatistic(v2)); cachedStatisticStorage.addColumnStatistic(t1, v1.getName(), statistics.getColumnStatistic(v1)); @@ -156,7 +156,7 @@ public void testOlapScanPredicateReorder() { sessionVariable.enablePredicateReorder(); - OlapTable t0 = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("t0"); PredicateReorderRule predicateReorderRule = new PredicateReorderRule(sessionVariable); @@ -230,8 +230,8 @@ public void testOlapScanPredicateReorder() { public void testHashJoinPredicateReorder() { sessionVariable.enablePredicateReorder(); - OlapTable t0 = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test").getTable("t0"); - OlapTable t1 = (OlapTable) GlobalStateMgr.getCurrentState().getDb("test").getTable("t1"); + OlapTable t0 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("t0"); + OlapTable t1 = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("t1"); PredicateReorderRule predicateReorderRule = new PredicateReorderRule(sessionVariable); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRuleTest.java index af6063a0a121c..f925f8164f84b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/mv/MaterializedViewRuleTest.java @@ -47,10 +47,10 @@ public class MaterializedViewRuleTest extends PlanTestBase { public static void beforeClass() throws Exception { PlanTestBase.beforeClass(); starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW lo_count_mv as " + - "select LO_ORDERDATE,count(LO_LINENUMBER) from lineorder_flat_for_mv group by LO_ORDERDATE;"); + "select LO_ORDERDATE,count(LO_LINENUMBER) from lineorder_flat_for_mv group by LO_ORDERDATE;"); starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW lo_count_key_mv as " + - "select LO_ORDERDATE, LO_ORDERKEY, count(LO_LINENUMBER) from lineorder_flat_for_mv" + - " group by LO_ORDERDATE, LO_ORDERKEY;"); + "select LO_ORDERDATE, LO_ORDERKEY, count(LO_LINENUMBER) from lineorder_flat_for_mv" + + " group by LO_ORDERDATE, LO_ORDERKEY;"); } @Test @@ -63,8 +63,9 @@ public void testMaterializedViewWithCountSelection() throws Exception { OlapScanNode olapScanNode = (OlapScanNode) plan.getScanNodes().get(0); Long selectedIndexid = olapScanNode.getSelectedIndexId(); GlobalStateMgr globalStateMgr = starRocksAssert.getCtx().getGlobalStateMgr(); - Database database = globalStateMgr.getDb("test"); - Table table = database.getTable("lineorder_flat_for_mv"); + Database database = globalStateMgr.getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(database.getFullName(), "lineorder_flat_for_mv"); Assert.assertTrue(table instanceof OlapTable); OlapTable baseTable = (OlapTable) table; Assert.assertEquals(baseTable.getIndexIdByName("lo_count_mv"), selectedIndexid); @@ -73,13 +74,14 @@ public void testMaterializedViewWithCountSelection() throws Exception { @Test public void testKeyColumnsMatch() throws Exception { GlobalStateMgr globalStateMgr = starRocksAssert.getCtx().getGlobalStateMgr(); - Database database = globalStateMgr.getDb("test"); - Table table = database.getTable("lineorder_flat_for_mv"); + Database database = globalStateMgr.getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(database.getFullName(), "lineorder_flat_for_mv"); OlapTable baseTable = (OlapTable) table; String sql = "select LO_ORDERDATE, sum(case when LO_ORDERKEY=0 then 0 else 1 end) as test, " + - "sum(case when LO_ORDERKEY=1 then 1 else 0 end) as nontest " + - " from lineorder_flat_for_mv group by LO_ORDERDATE;"; + "sum(case when LO_ORDERKEY=1 then 1 else 0 end) as nontest " + + " from lineorder_flat_for_mv group by LO_ORDERDATE;"; ExecPlan plan = getExecPlan(sql); OlapScanNode olapScanNode = (OlapScanNode) plan.getScanNodes().get(0); Long selectedIndexid = olapScanNode.getSelectedIndexId(); @@ -96,7 +98,7 @@ public void testCount1Rewrite() { arguments.add(queryColumnRef); CallOperator aggCall = new CallOperator(FunctionSet.COUNT, Type.BIGINT, arguments); MaterializedViewRule.RewriteContext rewriteContext = - new MaterializedViewRule.RewriteContext(aggCall, queryColumnRef, mvColumnRef, mvColumn); + new MaterializedViewRule.RewriteContext(aggCall, queryColumnRef, mvColumnRef, mvColumn); ColumnRefOperator dsColumnRef = tmpRefFactory.create("ds", Type.INT, false); List groupKeys = Lists.newArrayList(); groupKeys.add(dsColumnRef); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVRewriteWithSchemaChangeTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVRewriteWithSchemaChangeTest.java index b63192c5ca313..6875211d87059 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVRewriteWithSchemaChangeTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVRewriteWithSchemaChangeTest.java @@ -70,7 +70,6 @@ public void testSyncMVRewrite_PartitionPrune() throws Exception { PlanTestBase.assertContains(plan, "sync_mv1"); } - @Test public void testMVCacheInvalidAndReValid() throws Exception { starRocksAssert.withTable("\n" + @@ -125,8 +124,9 @@ public void testMVCacheInvalidAndReValid() throws Exception { waitForSchemaChangeAlterJobFinish(); // check mv invalid - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView mv1 = ((MaterializedView) testDb.getTable("test_cache_mv1")); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView mv1 = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_cache_mv1")); Assert.assertFalse(mv1.isActive()); try { cluster.runSql("test", "alter materialized view test_cache_mv1 active;"); @@ -273,7 +273,6 @@ public void testMVWithViewAndSubQuery1() throws Exception { dropMv("test", "join_mv_1"); } - { starRocksAssert.withMTableNames(cluster, List.of("depts", "emps_par"), () -> { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java index a5dbbe89b0444..212959465d8cc 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java @@ -45,15 +45,15 @@ public static void beforeClass() throws Exception { public void testStr2DateMVRefreshRewriteSingleTable_UnionRewrite() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select a, b, d, bitmap_union(to_bitmap(t1.c))" + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " group by a, b, d;"); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select a, b, d, bitmap_union(to_bitmap(t1.c))" + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " group by a, b, d;"); testSingleTableWithMVRewrite(mvName); @@ -65,16 +65,16 @@ public void testStr2DateMVRefreshRewriteSingleTable_UnionRewrite() throws Except public void testStr2DateMVRefreshRewriteSingleTableWithView() throws Exception { String mvName = "test_mv1"; starRocksAssert.withView("CREATE VIEW view1 as select a, b, d, bitmap_union(to_bitmap(t1.c))\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " group by a, b, d;"); + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " group by a, b, d;"); starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select * from view1"); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select * from view1"); testSingleTableWithMVRewrite(mvName); @@ -84,24 +84,25 @@ public void testStr2DateMVRefreshRewriteSingleTableWithView() throws Exception { } private void testSingleTableWithMVRewrite(String mvName) throws Exception { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "test_mv1"); } @@ -109,86 +110,86 @@ private void testSingleTableWithMVRewrite(String mvName) throws Exception { connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select a, b, d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " where d='2023-08-01'" + - " group by a, b, d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " where d='2023-08-01'" + + " group by a, b, d;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select a, b, d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " where t1.d >= '2023-08-01' \n" + - " group by a, b, d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " where t1.d >= '2023-08-01' \n" + + " group by a, b, d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); PlanTestBase.assertContains(plan, "IcebergScanNode\n" + - " TABLE: part_tbl1\n" + - " PREDICATES: 13: d != '2023-08-01', 13: d >= '2023-08-01'"); + " TABLE: part_tbl1\n" + + " PREDICATES: 13: d != '2023-08-01', 13: d >= '2023-08-01'"); } { String query = "select a, b, d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " group by a, b, d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " group by a, b, d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); PlanTestBase.assertContains(plan, "IcebergScanNode\n" + - " TABLE: part_tbl1\n" + - " PREDICATES: 13: d != '2023-08-01'"); + " TABLE: part_tbl1\n" + + " PREDICATES: 13: d != '2023-08-01'"); } starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-02') " + - "end ('2023-08-03') force with sync mode"); + "end ('2023-08-03') force with sync mode"); partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802", "p20230802_20230803"), partitions); { String query = "select a, b, d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " where t1.d >= '2023-08-01' \n" + - " group by a, b, d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " where t1.d >= '2023-08-01' \n" + + " group by a, b, d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/2"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/2"); PlanTestBase.assertContains(plan, "IcebergScanNode\n" + - " TABLE: part_tbl1\n" + - " PREDICATES: 13: d != '2023-08-01', 13: d != '2023-08-02'"); + " TABLE: part_tbl1\n" + + " PREDICATES: 13: d != '2023-08-01', 13: d != '2023-08-02'"); } { String query = "select a, b, d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " group by a, b, d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " group by a, b, d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/2"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/2"); PlanTestBase.assertContains(plan, "IcebergScanNode\n" + - " TABLE: part_tbl1\n" + - " PREDICATES: 13: d != '2023-08-01', 13: d != '2023-08-02'"); + " TABLE: part_tbl1\n" + + " PREDICATES: 13: d != '2023-08-01', 13: d != '2023-08-02'"); } } @@ -196,67 +197,68 @@ private void testSingleTableWithMVRewrite(String mvName) throws Exception { public void testStr2DateMVRefreshRewrite_InnerJoin_FullRefresh() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802", "p20230802_20230803", "p20230803_20230804"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='20230801';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d >= '20230801'\n" + - " partitions=3/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d >= '20230801'\n" + + " partitions=3/3\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d < '20230802';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d < '20230802';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); @@ -266,64 +268,65 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_FullRefresh() throws Exceptio public void testStr2DateMVRefreshRewrite_InnerJoin_PartialRefresh() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='20230801';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d < '20230802';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d < '20230802';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "12:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); @@ -333,67 +336,68 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_PartialRefresh() throws Excep public void testStr2DateMVRefreshRewrite_LeftJoin_FullRefresh() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802", "p20230802_20230803", "p20230803_20230804"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='20230801';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d >= '20230801'\n" + - " partitions=3/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d >= '20230801'\n" + + " partitions=3/3\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d < '20230802';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d < '20230802';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); @@ -403,89 +407,90 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_FullRefresh() throws Exception public void testStr2DateMVRefreshRewrite_LeftJoin_PartialRefresh() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='20230801';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d < '20230802';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d < '20230802';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "12:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d < '2023-08-02';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d < '2023-08-02';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "12:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); } @@ -494,138 +499,139 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_PartialRefresh() throws Except public void testStr2DateMVRefreshRewrite_InnerJoin_PartitionPrune1() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802", "p20230802_20230803", "p20230803_20230804"), partitions); { String query = "select count(*) from " + mvName + - " where d='2023-08-01';"; + " where d='2023-08-01';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } { String query = "select count(*) from " + mvName + - " where d in ('2023-08-01');"; + " where d in ('2023-08-01');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); } { String query = "select count(*) from " + mvName + - " where d in ('2023-08-01', '2023-08-02');"; + " where d in ('2023-08-01', '2023-08-02');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02')\n" + - " partitions=2/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02')\n" + + " partitions=2/3\n" + + " rollup: test_mv1"); } { String query = "select count(*) from " + mvName + - " where d in ('2023-08-01', '2023-08-02', '2023-08-03');"; + " where d in ('2023-08-01', '2023-08-02', '2023-08-03');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " partitions=3/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " partitions=3/3"); } { String query = "select count(*) from " + mvName + - " where d not in ('2023-08-01', '2023-08-02');"; + " where d not in ('2023-08-01', '2023-08-02');"; String plan = getFragmentPlan(query); // TODO: no partition prune PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 4: d NOT IN ('2023-08-01', '2023-08-02')\n" + - " partitions=3/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 4: d NOT IN ('2023-08-01', '2023-08-02')\n" + + " partitions=3/3\n" + + " rollup: test_mv1"); } { String query = "select count(*) from " + mvName + - " where d >= '2023-08-01' and d < '2023-08-02';"; + " where d >= '2023-08-01' and d < '2023-08-02';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } { String query = "select count(*) from " + mvName + - " where cast(d as date) >= '2023-08-01'"; + " where cast(d as date) >= '2023-08-01'"; String plan = getFragmentPlan(query); // TODO: no partition prune PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: CAST(4: d AS DATE) >= '2023-08-01'\n" + - " partitions=3/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: CAST(4: d AS DATE) >= '2023-08-01'\n" + + " partitions=3/3"); } connectContext.getSessionVariable().setRangePrunerPredicateMaxLen(0); { String query = "select count(*) from " + mvName + - " where d in ('2023-08-01');"; + " where d in ('2023-08-01');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); } { String query = "select count(*) from " + mvName + - " where d in ('2023-08-01', '2023-08-02');"; + " where d in ('2023-08-01', '2023-08-02');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02')\n" + - " partitions=2/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02')\n" + + " partitions=2/3\n" + + " rollup: test_mv1"); } { String query = "select count(*) from " + mvName + - " where d in ('2023-08-01', '2023-08-02', '2023-08-03');"; + " where d in ('2023-08-01', '2023-08-02', '2023-08-03');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " partitions=3/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 4: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " partitions=3/3"); } starRocksAssert.dropMaterializedView(mvName); @@ -635,165 +641,166 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_PartitionPrune1() throws Exce public void testStr2DateMVRefreshRewrite_InnerJoin_PartitionPrune2() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802", "p20230802_20230803", "p20230803_20230804"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d='2023-08-01';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d='2023-08-01';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01');"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02');"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02')\n" + - " partitions=2/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02')\n" + + " partitions=2/3"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03');"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " partitions=3/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " partitions=3/3"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d not in ('2023-08-01', '2023-08-02');"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d not in ('2023-08-01', '2023-08-02');"; String plan = getFragmentPlan(query); // TODO: no partition prune PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d != '2023-08-01', 16: d != '2023-08-02'\n" + - " partitions=3/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d != '2023-08-01', 16: d != '2023-08-02'\n" + + " partitions=3/3"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d >= '2023-08-01' and t1.d < '2023-08-02';"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d >= '2023-08-01' and t1.d < '2023-08-02';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where cast(t1.d as date) >= '2023-08-01'"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where cast(t1.d as date) >= '2023-08-01'"; String plan = getFragmentPlan(query); // TODO: no partition prune PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: CAST(16: d AS DATE) >= '2023-08-01'\n" + - " partitions=3/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: CAST(16: d AS DATE) >= '2023-08-01'\n" + + " partitions=3/3"); } connectContext.getSessionVariable().setRangePrunerPredicateMaxLen(0); { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01');"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/3"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02');"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02')\n" + - " partitions=2/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02')\n" + + " partitions=2/3"); } { String query = "select t1.a, t2.b, t3.c, t1.d \n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03');"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03');"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " partitions=3/3"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d IN ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " partitions=3/3"); } starRocksAssert.dropMaterializedView(mvName); @@ -803,89 +810,90 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_PartitionPrune2() throws Exce public void testStr2DateMVRefreshRewrite_InnerJoin_ExtraCompensate() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(b) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.d, t2.b, t3.c;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(b) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.d, t2.b, t3.c, count(t1.a) " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.d, t2.b, t3.c;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null and t2.b in ('xxx') " + - " and t3.c in ('xxx') " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null and t2.b in ('xxx') " + + " and t3.c in ('xxx') " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); @@ -895,50 +903,51 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_ExtraCompensate() throws Exce public void testStr2DateMVRefreshRewrite_LeftJoin_OnPredicates_ExtraCompensate() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(b) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.d, t2.b, t3.c;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(b) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.d, t2.b, t3.c, count(t1.a) " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.d, t2.b, t3.c;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); @@ -946,11 +955,11 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_OnPredicates_ExtraCompensate() } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); PlanTestBase.assertNotContains(plan, "test_mv1"); @@ -958,45 +967,45 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_OnPredicates_ExtraCompensate() { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d in ('2023-08-01', '2023-08-02') " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d in ('2023-08-01', '2023-08-02') " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); PlanTestBase.assertNotContains(plan, "test_mv1"); } starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-02') " + - "end ('2023-08-03') force with sync mode"); + "end ('2023-08-03') force with sync mode"); partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802", "p20230802_20230803"), partitions); { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "3:IcebergScanNode\n" + - " TABLE: part_tbl1\n" + - " PREDICATES: 19: d >= '2023-08-01'"); + " TABLE: part_tbl1\n" + + " PREDICATES: 19: d >= '2023-08-01'"); } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d in ('2023-08-01', '2023-08-02') " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d in ('2023-08-01', '2023-08-02') " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); @@ -1009,50 +1018,51 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_OnPredicates_ExtraCompensate() public void testStr2DateMVRefreshRewrite_InnerJoin_OnPredicates_ExtraCompensate() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(b) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.d, t2.b, t3.c;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(b) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.d, t2.b, t3.c, count(t1.a) " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.d, t2.b, t3.c;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b is not null " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); @@ -1060,11 +1070,11 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_OnPredicates_ExtraCompensate( } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "test_mv1"); @@ -1072,11 +1082,11 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_OnPredicates_ExtraCompensate( { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d in ('2023-08-01', '2023-08-02') " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d and t2.b != '' " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d in ('2023-08-01', '2023-08-02') " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "test_mv1"); @@ -1089,90 +1099,91 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_OnPredicates_ExtraCompensate( public void testStr2DateMVRefreshRewrite_LeftJoin_ExtraCompensate() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(b) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.d, t2.b, t3.c;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(b) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.d, t2.b, t3.c, count(t1.a) " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.d, t2.b, t3.c;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.d, t2.b, t3.c, count(t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null and t2.b in ('xxx') " + - " and t3.c in ('xxx') " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' and t2.b != '' and t2.b is not null and t2.b in ('xxx') " + + " and t3.c in ('xxx') " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); starRocksAssert.dropMaterializedView(mvName); @@ -1182,154 +1193,155 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_ExtraCompensate() throws Excep public void testStr2DateMVRefreshRewriteInnerJoinAggregatePartialRefreshUnion() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t1.d, count(t1.c)" + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.a, t2.b, t1.d;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t1.d, count(t1.c)" + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.a, t2.b, t1.d;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d ='2023-08-01'\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d ='2023-08-01'\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d >= '2023-08-01' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d >= '2023-08-01' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d >= '2023-08-01' and t1.d <= '2023-08-02' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d >= '2023-08-01' and t1.d <= '2023-08-02' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d = '2023-08-01' or t1.d = '2023-08-02' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d = '2023-08-01' or t1.d = '2023-08-02' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " group by t1.a, t2.b, t1.d;"; connectContext.getSessionVariable().setRangePrunerPredicateMaxLen(0); String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); } @@ -1338,153 +1350,154 @@ public void testStr2DateMVRefreshRewriteInnerJoinAggregatePartialRefreshUnion() public void testStr2DateMVRefreshRewriteLeftJoinAggregatePartialRefreshUnion() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t1.d, count(t1.c)" + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.a, t2.b, t1.d;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t1.d, count(t1.c)" + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.a, t2.b, t1.d;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d ='2023-08-01'\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d ='2023-08-01'\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d >= '2023-08-01' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d >= '2023-08-01' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d >= '2023-08-01' and t1.d <= '2023-08-02' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d >= '2023-08-01' and t1.d <= '2023-08-02' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d = '2023-08-01' or t1.d = '2023-08-02' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d = '2023-08-01' or t1.d = '2023-08-02' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " group by t1.a, t2.b, t1.d;"; connectContext.getSessionVariable().setRangePrunerPredicateMaxLen(0); String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); } @@ -1493,36 +1506,37 @@ public void testStr2DateMVRefreshRewriteLeftJoinAggregatePartialRefreshUnion() t public void testStr2DateMVRefreshRewriteLeftJoinBitmapAggregatePartialRefreshUnion() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t1.d, bitmap_union(to_bitmap(t1.c))" + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.a, t2.b, t1.d;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t1.d, bitmap_union(to_bitmap(t1.c))" + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.a, t2.b, t1.d;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "test_mv1"); } @@ -1530,132 +1544,132 @@ public void testStr2DateMVRefreshRewriteLeftJoinBitmapAggregatePartialRefreshUni connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); PlanTestBase.assertContains(plan, "5:IcebergScanNode\n" + - " TABLE: part_tbl3\n" + - " PREDICATES: 27: d != '2023-08-01'"); + " TABLE: part_tbl3\n" + + " PREDICATES: 27: d != '2023-08-01'"); } { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d ='2023-08-01'\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d ='2023-08-01'\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d >= '2023-08-01' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d >= '2023-08-01' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d >= '2023-08-01' and t1.d <= '2023-08-02' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d >= '2023-08-01' and t1.d <= '2023-08-02' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d = '2023-08-01' or t1.d = '2023-08-02' \n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d = '2023-08-01' or t1.d = '2023-08-02' \n" + + " group by t1.a, t2.b, t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t1.d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + - " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + - " group by t1.a, t2.b, t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d \n" + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d \n" + + " where t1.d in ('2023-08-01', '2023-08-02', '2023-08-03')\n" + + " group by t1.a, t2.b, t1.d;"; connectContext.getSessionVariable().setRangePrunerPredicateMaxLen(0); String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "14:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); starRocksAssert.dropMaterializedView(mvName); @@ -1665,105 +1679,106 @@ public void testStr2DateMVRefreshRewriteLeftJoinBitmapAggregatePartialRefreshUni public void testStr2DateMVRefreshRewriteWithBitmapHash_LeftJoin() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(b) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.d, t2.b, t3.c, bitmap_union(bitmap_hash(t1.a)) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.d, t2.b, t3.c;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(b) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.d, t2.b, t3.c, bitmap_union(bitmap_hash(t1.a)) " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.d, t2.b, t3.c;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); { String query = "select t1.d, t2.b, t3.c, bitmap_union(bitmap_hash(t1.a)) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, t2.b, t3.c, bitmap_union_count(bitmap_hash(t1.a)) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select t1.d, t2.b, t3.c, count(distinct t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, t2.b, t3.c, count(distinct t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, count(distinct t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " left join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " left join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "15:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } starRocksAssert.dropMaterializedView(mvName); connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); @@ -1773,104 +1788,105 @@ public void testStr2DateMVRefreshRewriteWithBitmapHash_LeftJoin() throws Excepti public void testStr2DateMVRefreshRewriteWithBitmapHash_InnerJoin() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(b) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.d, t2.b, t3.c, bitmap_union(bitmap_hash(t1.a)) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.d, t2.b, t3.c;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(b) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.d, t2.b, t3.c, bitmap_union(bitmap_hash(t1.a)) " + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.d, t2.b, t3.c;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.d, t2.b, t3.c, bitmap_union(bitmap_hash(t1.a)) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, t2.b, t3.c, bitmap_union_count(bitmap_hash(t1.a)) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select t1.d, t2.b, t3.c, count(distinct t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, t2.b, t3.c, count(distinct t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d, t2.b, t3.c;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d, t2.b, t3.c;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "13:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } { String query = "select t1.d, count(distinct t1.a) " + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='2023-08-01' " + - " group by t1.d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='2023-08-01' " + + " group by t1.d;"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "UNION"); PlanTestBase.assertContains(plan, "15:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } connectContext.getSessionVariable().setMaterializedViewRewriteMode("default"); starRocksAssert.dropMaterializedView(mvName); @@ -1880,51 +1896,52 @@ public void testStr2DateMVRefreshRewriteWithBitmapHash_InnerJoin() throws Except public void testStr2DateMVRefreshRewriteSingleTableWithDateTruc_Day() throws Exception { String mvName = "test_mv1"; starRocksAssert.withView("CREATE VIEW view1 as select " + - " a, b, " + - " date_trunc('day', str2date(d,'%Y-%m-%d')) as dt, " + - " bitmap_union(to_bitmap(t1.c))\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " group by a, b, dt;"); + " a, b, " + + " date_trunc('day', str2date(d,'%Y-%m-%d')) as dt, " + + " bitmap_union(to_bitmap(t1.c))\n" + + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " group by a, b, dt;"); starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by dt " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select * from view1"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by dt " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select * from view1"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select a, b, date_trunc('day', str2date(d,'%Y-%m-%d')) as dt, " + - " count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " where d='2023-08-01'" + - " group by a, b, dt;"; + " count(distinct t1.c)\n" + + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " where d='2023-08-01'" + + " group by a, b, dt;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=1/1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=1/1"); } // TODO: Support date_trunc('day', str2date(t1.d, ''%Y-%m-%d'')) to str2date(d, '%Y-%m-%d') { String query = "select a, b, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " where date_trunc('day', str2date(t1.d, '%Y-%m-%d')) >= '2023-08-01' \n" + - " group by a, b;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " where date_trunc('day', str2date(t1.d, '%Y-%m-%d')) >= '2023-08-01' \n" + + " group by a, b;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, mvName); } @@ -1938,53 +1955,54 @@ public void testStr2DateMVRefreshRewriteSingleTableWithDateTruc_Day() throws Exc public void testStr2DateMVRefreshRewriteSingleTableWithDateTruc_Month() throws Exception { String mvName = "test_mv1"; starRocksAssert.withView("CREATE VIEW view1 as select a, b, " + - " date_trunc('month', str2date(d,'%Y-%m-%d')) as dt, " + - " bitmap_union(to_bitmap(t1.c))\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " group by a, b, d;"); + " date_trunc('month', str2date(d,'%Y-%m-%d')) as dt, " + + " bitmap_union(to_bitmap(t1.c))\n" + + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " group by a, b, d;"); starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by dt " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as select * from view1"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by dt " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as select * from view1"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('2023-08-01') " + - "end ('2023-08-02') force with sync mode"); + "end ('2023-08-02') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p202308_202309"), partitions); connectContext.getSessionVariable().setMaterializedViewRewriteMode("force"); { String query = "select " + - " a, b, date_trunc('month', str2date(d,'%Y-%m-%d')) as dt, " + - " count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " where date_trunc('month', str2date(d,'%Y-%m-%d')) ='2023-08-01'" + - " group by a, b, dt;"; + " a, b, date_trunc('month', str2date(d,'%Y-%m-%d')) as dt, " + + " count(distinct t1.c)\n" + + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " where date_trunc('month', str2date(d,'%Y-%m-%d')) ='2023-08-01'" + + " group by a, b, dt;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "UNION"); UtFrameUtils.matchPlanWithoutId(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 9: dt = '2023-08-01'\n" + - " partitions=1/1\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 9: dt = '2023-08-01'\n" + + " partitions=1/1\n" + + " rollup: test_mv1"); } // TODO: Support date_trunc('day', str2date(t1.d, ''%Y-%m-%d'')) to str2date(d, '%Y-%m-%d') { String query = "select a, b, d, count(distinct t1.c)\n" + - " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + - " where t1.d >= '2023-08-01' \n" + - " group by a, b, d;"; + " from iceberg0.partitioned_db.part_tbl1 as t1 \n" + + " where t1.d >= '2023-08-01' \n" + + " group by a, b, d;"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, mvName); } @@ -1998,26 +2016,26 @@ public void testStr2DateMVRefreshRewriteSingleTableWithDateTruc_Month() throws E public void testViewBasedRewrite() throws Exception { connectContext.getSessionVariable().setEnableViewBasedMvRewrite(true); String view = "create view iceberg_table_view " + - " as select t1.a, t2.b, t1.d, count(t1.c) as cnt" + - " from iceberg0.partitioned_db.part_tbl1 as t1 " + - " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.a, t2.b, t1.d;"; + " as select t1.a, t2.b, t1.d, count(t1.c) as cnt" + + " from iceberg0.partitioned_db.part_tbl1 as t1 " + + " inner join iceberg0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join iceberg0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.a, t2.b, t1.d;"; starRocksAssert.withView(view); String mvName = "iceberg_mv_1"; String mv = "create materialized view iceberg_mv_1 " + - "partition by str2date(d,'%Y-%m-%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'" + - ") " + - "as " + - "select a, b, d, cnt " + - "from iceberg_table_view"; + "partition by str2date(d,'%Y-%m-%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'" + + ") " + + "as " + + "select a, b, d, cnt " + + "from iceberg_table_view"; starRocksAssert.withMaterializedView(mv); starRocksAssert.getCtx().executeSql("refresh materialized view iceberg_mv_1" + - " partition start('2023-08-01') end('2023-08-02') with sync mode"); + " partition start('2023-08-01') end('2023-08-02') with sync mode"); { String query = "select a, b, d, cnt from iceberg_table_view"; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteJDBCTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteJDBCTest.java index aad6b687924b9..57209b0b56fd7 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteJDBCTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteJDBCTest.java @@ -41,7 +41,7 @@ public static void beforeClass() throws Exception { ConnectorPlanTestBase.mockCatalog(connectContext, MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME); MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr(); MockedJDBCMetadata mockedJDBCMetadata = - (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); + (MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get(); mockedJDBCMetadata.initPartitions(); } @@ -49,69 +49,70 @@ public static void beforeClass() throws Exception { public void testStr2DateMVRefreshRewrite_InnerJoin_FullRefresh() throws Exception { String mvName = "test_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'," + - "'query_rewrite_consistency' = 'loose'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'," + + "'query_rewrite_consistency' = 'loose'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p00010101_20230801", "p20230801_20230802", - "p20230802_20230803", "p20230803_99991231"), partitions); + "p20230802_20230803", "p20230803_99991231"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='20230801';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d = '20230801'\n" + - " partitions=1/4"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d = '20230801'\n" + + " partitions=1/4"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='20230801';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d >= '20230801'\n" + - " partitions=3/4\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d >= '20230801'\n" + + " partitions=3/4\n" + + " rollup: test_mv1"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d < '20230802';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d < '20230802';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv1\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/4\n" + - " rollup: test_mv1"); + " TABLE: test_mv1\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/4\n" + + " rollup: test_mv1"); } starRocksAssert.dropMaterializedView(mvName); @@ -121,34 +122,35 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_FullRefresh() throws Exceptio public void testStr2DateMVRefreshRewrite_InnerJoin_PartialRefresh() throws Exception { String mvName = "test_mv2"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"query_rewrite_consistency\" = \"disable\",\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"query_rewrite_consistency\" = \"disable\",\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('20230801') " + - "end ('20230802') force with sync mode"); + "end ('20230802') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='20230801';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "test_mv2"); } @@ -156,74 +158,74 @@ public void testStr2DateMVRefreshRewrite_InnerJoin_PartialRefresh() throws Excep starRocksAssert.dropMaterializedView(mvName); } - @Test public void testStr2DateMVRefreshRewrite_LeftJoin_FullRefresh() throws Exception { String mvName = "test_mv3"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "'replication_num' = '1'," + - "'query_rewrite_consistency' = 'loose'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "'replication_num' = '1'," + + "'query_rewrite_consistency' = 'loose'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p00010101_20230801", "p20230801_20230802", "p20230802_20230803", - "p20230803_99991231"), partitions); + "p20230803_99991231"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='20230801';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv3\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d = '20230801'\n" + - " partitions=1/4"); + " TABLE: test_mv3\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d = '20230801'\n" + + " partitions=1/4"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d>='20230801';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d>='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv3\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 16: d >= '20230801'\n" + - " partitions=3/4\n" + - " rollup: test_mv3"); + " TABLE: test_mv3\n" + + " PREAGGREGATION: ON\n" + + " PREDICATES: 16: d >= '20230801'\n" + + " partitions=3/4\n" + + " rollup: test_mv3"); } { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d < '20230802';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d < '20230802';"; String plan = getFragmentPlan(query); PlanTestBase.assertContains(plan, "0:OlapScanNode\n" + - " TABLE: test_mv3\n" + - " PREAGGREGATION: ON\n" + - " partitions=2/4\n" + - " rollup: test_mv3"); + " TABLE: test_mv3\n" + + " PREAGGREGATION: ON\n" + + " partitions=2/4\n" + + " rollup: test_mv3"); } starRocksAssert.dropMaterializedView(mvName); @@ -233,34 +235,35 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_FullRefresh() throws Exception public void testStr2DateMVRefreshRewrite_LeftJoin_PartialRefresh() throws Exception { String mvName = "test_mv4"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"query_rewrite_consistency\" = \"disable\",\n" + - "'replication_num' = '1'" + - ") " + - "as select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable(mvName)); + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"query_rewrite_consistency\" = \"disable\",\n" + + "'replication_num' = '1'" + + ") " + + "as select t1.a, t2.b, t3.c, t1.d " + + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d ;"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), mvName)); // initial create starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " partition start('20230801') " + - "end ('20230802') force with sync mode"); + "end ('20230802') force with sync mode"); List partitions = - materializedView.getPartitions().stream().map(Partition::getName).sorted() - .collect(Collectors.toList()); + materializedView.getPartitions().stream().map(Partition::getName).sorted() + .collect(Collectors.toList()); Assert.assertEquals(Arrays.asList("p20230801_20230802"), partitions); { String query = "select t1.a, t2.b, t3.c, t1.d " + - " from jdbc0.partitioned_db0.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + - " where t1.d='20230801';"; + " from jdbc0.partitioned_db0.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db0.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db0.part_tbl3 t3 on t1.d=t3.d " + + " where t1.d='20230801';"; String plan = getFragmentPlan(query); PlanTestBase.assertNotContains(plan, "test_mv4"); } @@ -271,24 +274,24 @@ public void testStr2DateMVRefreshRewrite_LeftJoin_PartialRefresh() throws Except @Test public void testViewBasedMvRewriteOnJdbcTable() throws Exception { String view = "create view jdbc_table_view " + - " as select t1.a, t2.b, t1.d, count(t1.c) as cnt" + - " from jdbc0.partitioned_db.part_tbl1 as t1 " + - " inner join jdbc0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + - " inner join jdbc0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + - " group by t1.a, t2.b, t1.d;"; + " as select t1.a, t2.b, t1.d, count(t1.c) as cnt" + + " from jdbc0.partitioned_db.part_tbl1 as t1 " + + " inner join jdbc0.partitioned_db.part_tbl2 t2 on t1.d=t2.d " + + " inner join jdbc0.partitioned_db.part_tbl3 t3 on t1.d=t3.d " + + " group by t1.a, t2.b, t1.d;"; starRocksAssert.withView(view); String mvName = "jdbc_view_mv1"; starRocksAssert.withMaterializedView("create materialized view " + mvName + " " + - "partition by str2date(d,'%Y%m%d') " + - "distributed by hash(a) " + - "REFRESH DEFERRED MANUAL " + - "PROPERTIES (\n" + - "\"query_rewrite_consistency\" = \"loose\",\n" + - "'replication_num' = '1'" + - ") " + - " as select a, b, d, cnt" + - " from jdbc_table_view"); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + "partition by str2date(d,'%Y%m%d') " + + "distributed by hash(a) " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"query_rewrite_consistency\" = \"loose\",\n" + + "'replication_num' = '1'" + + ") " + + " as select a, b, d, cnt" + + " from jdbc_table_view"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); starRocksAssert.getCtx().executeSql("refresh materialized view " + mvName + " with sync mode"); { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshTest.java index 0d60bc2a927cb..af21791c251cb 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshTest.java @@ -31,7 +31,7 @@ import org.junit.BeforeClass; import org.junit.Test; -public class MvRefreshTest extends MVRefreshTestBase { +public class MvRefreshTest extends MVRefreshTestBase { @BeforeClass public static void beforeClass() throws Exception { PseudoCluster.getOrCreateWithRandomPort(true, 1); @@ -45,143 +45,149 @@ public static void beforeClass() throws Exception { @Test public void testMvWithComplexNameRefresh() throws Exception { createAndRefreshMv("create materialized view `cc`" + - " partition by id_date" + - " distributed by hash(`t1a`)" + - " as" + - " select t1a, id_date, t1b from table_with_partition"); + " partition by id_date" + + " distributed by hash(`t1a`)" + + " as" + + " select t1a, id_date, t1b from table_with_partition"); createAndRefreshMv("create materialized view `luchen_order_8e2c65ba_1c30`" + - " partition by id_date" + - " distributed by hash(`t1a`)" + - " as" + - " select t1a, id_date, t1b from table_with_partition"); + " partition by id_date" + + " distributed by hash(`t1a`)" + + " as" + + " select t1a, id_date, t1b from table_with_partition"); } @Test public void testMVRefreshWithQueryRewrite1() throws Exception { createAndRefreshMv("create materialized view `test_mv1` \n" + - " partition by id_date\n" + - " distributed by hash(`t1a`)\n" + - " refresh deferred manual \n" + - " as\n" + - " select t1a, id_date, t1b, sum(t1c) as sum1 from table_with_partition group by t1a, id_date, t1b"); + " partition by id_date\n" + + " distributed by hash(`t1a`)\n" + + " refresh deferred manual \n" + + " as\n" + + " select t1a, id_date, t1b, sum(t1c) as sum1 from table_with_partition group by t1a, id_date, t1b"); starRocksAssert.withMaterializedView("create materialized view `test_mv2` \n" + - " partition by id_date\n" + - " distributed by hash(`t1a`)\n" + - " refresh deferred manual \n" + - " as\n" + - " select t1a, id_date, sum(t1c) as sum1 from table_with_partition group by t1a, id_date", - () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("test_mv2")); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + " partition by id_date\n" + + " distributed by hash(`t1a`)\n" + + " refresh deferred manual \n" + + " as\n" + + " select t1a, id_date, sum(t1c) as sum1 from table_with_partition group by t1a, id_date", + () -> { + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_mv2")); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - // If the base table has been refreshed and test_mv1 has been refreshed, test_mv2 refresh can use - // test_mv1. - { - ExecPlan execPlan = getMVRefreshExecPlan(taskRun); - Assert.assertNotNull("exec plan should not null", execPlan); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, "test_mv1"); - } + // If the base table has been refreshed and test_mv1 has been refreshed, test_mv2 refresh can use + // test_mv1. + { + ExecPlan execPlan = getMVRefreshExecPlan(taskRun); + Assert.assertNotNull("exec plan should not null", execPlan); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, "test_mv1"); + } - // If the base table has changed and test_mv1 has not been refreshed, test_mv2 refresh should - // not use test_mv1. - { - executeInsertSql(connectContext, "insert into table_with_partition partition(p1991)" + - " values(\"varchar12\", '1991-03-01', 2, 1, 1)"); - ExecPlan execPlan = getMVRefreshExecPlan(taskRun); - Assert.assertNotNull("exec plan should not null", execPlan); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertNotContains(plan, "test_mv1"); - } - }); + // If the base table has changed and test_mv1 has not been refreshed, test_mv2 refresh should + // not use test_mv1. + { + executeInsertSql(connectContext, "insert into table_with_partition partition(p1991)" + + " values(\"varchar12\", '1991-03-01', 2, 1, 1)"); + ExecPlan execPlan = getMVRefreshExecPlan(taskRun); + Assert.assertNotNull("exec plan should not null", execPlan); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertNotContains(plan, "test_mv1"); + } + }); starRocksAssert.dropMaterializedView("test_mv1"); } @Test public void testMVRefreshWithQueryRewrite2() throws Exception { createAndRefreshMv("create materialized view `test_mv1` \n" + - " partition by id_date\n" + - " distributed by hash(`t1a`)\n" + - " refresh deferred manual \n" + - " as\n" + - " select t1a, id_date, t1b, sum(t1c) as sum1 from table_with_partition group by t1a, id_date, t1b"); + " partition by id_date\n" + + " distributed by hash(`t1a`)\n" + + " refresh deferred manual \n" + + " as\n" + + " select t1a, id_date, t1b, sum(t1c) as sum1 from table_with_partition group by t1a, id_date, t1b"); starRocksAssert.withMaterializedView("create materialized view `test_mv2` \n" + - "partition by id_date\n" + - "distributed by hash(`t1a`)\n" + - "refresh deferred manual \n" + - "PROPERTIES (\n" + - " 'replication_num' = '1', \n" + - " 'session.enable_materialized_view_rewrite' = 'false' \n" + - ")\n" + - " as\n" + - " select t1a, id_date, sum(t1c) as sum1 from table_with_partition group by t1a, id_date", - () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("test_mv2")); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + "partition by id_date\n" + + "distributed by hash(`t1a`)\n" + + "refresh deferred manual \n" + + "PROPERTIES (\n" + + " 'replication_num' = '1', \n" + + " 'session.enable_materialized_view_rewrite' = 'false' \n" + + ")\n" + + " as\n" + + " select t1a, id_date, sum(t1c) as sum1 from table_with_partition group by t1a, id_date", + () -> { + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_mv2")); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - // If the base table has been refreshed and test_mv1 has been refreshed, test_mv2 refresh can use - // test_mv1. - { - ExecPlan execPlan = getMVRefreshExecPlan(taskRun); - Assert.assertNotNull("exec plan should not null", execPlan); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertNotContains(plan, "test_mv1"); - } - }); + // If the base table has been refreshed and test_mv1 has been refreshed, test_mv2 refresh can use + // test_mv1. + { + ExecPlan execPlan = getMVRefreshExecPlan(taskRun); + Assert.assertNotNull("exec plan should not null", execPlan); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertNotContains(plan, "test_mv1"); + } + }); starRocksAssert.dropMaterializedView("test_mv1"); } @Test public void testMVRefreshWithQueryRewrite3() throws Exception { createAndRefreshMv("create materialized view `test_mv1` \n" + - " partition by id_date\n" + - " distributed by hash(`t1a`)\n" + - " refresh deferred manual \n" + - " as\n" + - " select t1a, id_date, t1b, sum(t1c) as sum1 from table_with_partition group by t1a, id_date, t1b"); + " partition by id_date\n" + + " distributed by hash(`t1a`)\n" + + " refresh deferred manual \n" + + " as\n" + + " select t1a, id_date, t1b, sum(t1c) as sum1 from table_with_partition group by t1a, id_date, t1b"); Config.enable_mv_refresh_query_rewrite = false; starRocksAssert.withMaterializedView("create materialized view `test_mv2` \n" + - "partition by id_date\n" + - "distributed by hash(`t1a`)\n" + - "refresh deferred manual \n" + - "PROPERTIES (\n" + - " 'replication_num' = '1', \n" + - " 'session.enable_materialized_view_rewrite' = 'true', \n" + - " 'session.enable_materialized_view_for_insert' = 'true' \n" + - ")\n" + - " as\n" + - " select t1a, id_date, sum(t1c) as sum1 from table_with_partition group by t1a, id_date", - () -> { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - MaterializedView materializedView = ((MaterializedView) testDb.getTable("test_mv2")); - Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); - TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); + "partition by id_date\n" + + "distributed by hash(`t1a`)\n" + + "refresh deferred manual \n" + + "PROPERTIES (\n" + + " 'replication_num' = '1', \n" + + " 'session.enable_materialized_view_rewrite' = 'true', \n" + + " 'session.enable_materialized_view_for_insert' = 'true' \n" + + ")\n" + + " as\n" + + " select t1a, id_date, sum(t1c) as sum1 from table_with_partition group by t1a, id_date", + () -> { + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + MaterializedView materializedView = + ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), "test_mv2")); + Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName()); + TaskRun taskRun = TaskRunBuilder.newBuilder(task).build(); - // If the base table has been refreshed and test_mv1 has been refreshed, test_mv2 refresh can use - // test_mv1. - { - ExecPlan execPlan = getMVRefreshExecPlan(taskRun); - Assert.assertNotNull("exec plan should not null", execPlan); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertContains(plan, "test_mv1"); - } + // If the base table has been refreshed and test_mv1 has been refreshed, test_mv2 refresh can use + // test_mv1. + { + ExecPlan execPlan = getMVRefreshExecPlan(taskRun); + Assert.assertNotNull("exec plan should not null", execPlan); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertContains(plan, "test_mv1"); + } - // If the base table has changed and test_mv1 has not been refreshed, test_mv2 refresh should - // not use test_mv1. - { - executeInsertSql(connectContext, "insert into table_with_partition partition(p1991)" + - " values(\"varchar12\", '1991-03-01', 2, 1, 1)"); - ExecPlan execPlan = getMVRefreshExecPlan(taskRun); - Assert.assertNotNull("exec plan should not null", execPlan); - String plan = execPlan.getExplainString(TExplainLevel.NORMAL); - PlanTestBase.assertNotContains(plan, "test_mv1"); - } - }); + // If the base table has changed and test_mv1 has not been refreshed, test_mv2 refresh should + // not use test_mv1. + { + executeInsertSql(connectContext, "insert into table_with_partition partition(p1991)" + + " values(\"varchar12\", '1991-03-01', 2, 1, 1)"); + ExecPlan execPlan = getMVRefreshExecPlan(taskRun); + Assert.assertNotNull("exec plan should not null", execPlan); + String plan = execPlan.getExplainString(TExplainLevel.NORMAL); + PlanTestBase.assertNotContains(plan, "test_mv1"); + } + }); Config.enable_mv_refresh_query_rewrite = true; starRocksAssert.dropMaterializedView("test_mv1"); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTestBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTestBase.java index 0af6cf706d4c0..c4b69f01e20f6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTestBase.java @@ -123,8 +123,8 @@ public String getFragmentPlan(String sql, TExplainLevel level, String traceModul } public static Table getTable(String dbName, String mvName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(mvName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName); Assert.assertNotNull(table); return table; } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java index b14ff57e92683..4dc41e3fed07d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtilsTest.java @@ -26,6 +26,7 @@ import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; @@ -100,14 +101,14 @@ public void testGetAllPredicate() { BinaryPredicateOperator binaryPredicate = new BinaryPredicateOperator( BinaryType.EQ, columnRef1, columnRef2); - Database db = starRocksAssert.getCtx().getGlobalStateMgr().getDb("test"); - Table table1 = db.getTable("t0"); + Database db = starRocksAssert.getCtx().getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table1 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); LogicalScanOperator scanOperator1 = new LogicalOlapScanOperator(table1); BinaryPredicateOperator binaryPredicate2 = new BinaryPredicateOperator( BinaryType.GE, columnRef1, ConstantOperator.createInt(1)); scanOperator1.setPredicate(binaryPredicate2); OptExpression scanExpr = OptExpression.create(scanOperator1); - Table table2 = db.getTable("t1"); + Table table2 = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t1"); LogicalScanOperator scanOperator2 = new LogicalOlapScanOperator(table2); BinaryPredicateOperator binaryPredicate3 = new BinaryPredicateOperator( BinaryType.GE, columnRef2, ConstantOperator.createInt(1)); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/CachedStatisticStorageTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/CachedStatisticStorageTest.java index 7ae8e209f4d31..080bdc5d7e39b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/CachedStatisticStorageTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/CachedStatisticStorageTest.java @@ -127,8 +127,8 @@ public static void beforeClass() throws Exception { @Test public void testGetColumnStatistic(@Mocked CachedStatisticStorage cachedStatisticStorage) { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t0"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); new Expectations() { { @@ -160,8 +160,8 @@ public void testGetColumnStatistic(@Mocked CachedStatisticStorage cachedStatisti @Test public void testGetColumnStatistics(@Mocked CachedStatisticStorage cachedStatisticStorage) { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t0"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); ColumnStatistic columnStatistic1 = ColumnStatistic.builder().setDistinctValuesCount(888).build(); ColumnStatistic columnStatistic2 = ColumnStatistic.builder().setDistinctValuesCount(999).build(); @@ -367,8 +367,8 @@ public void testExpireConnectorHistogramStatistics() { @Test public void testLoadCacheLoadEmpty(@Mocked CachedStatisticStorage cachedStatisticStorage) { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - Table table = db.getTable("t0"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); new Expectations() { { @@ -388,8 +388,8 @@ public void testLoadCacheLoadEmpty(@Mocked CachedStatisticStorage cachedStatisti @Test public void testConvert2ColumnStatistics() { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t0"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); ColumnBasicStatsCacheLoader cachedStatisticStorage = Deencapsulation.newInstance(ColumnBasicStatsCacheLoader.class); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java index 657857d79b9ae..7aa6087e3bfb2 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java @@ -99,54 +99,54 @@ public static void beforeClass() throws Exception { @Before public void before() throws Exception { starRocksAssert.withTable("CREATE TABLE `test_all_type` (\n" + - " `t1a` varchar(20) NULL COMMENT \"\",\n" + - " `t1b` smallint(6) NULL COMMENT \"\",\n" + - " `t1c` int(11) NULL COMMENT \"\",\n" + - " `t1d` bigint(20) NULL COMMENT \"\",\n" + - " `t1e` float NULL COMMENT \"\",\n" + - " `t1f` double NULL COMMENT \"\",\n" + - " `t1g` bigint(20) NULL COMMENT \"\",\n" + - " `id_datetime` datetime NULL COMMENT \"\",\n" + - " `id_date` date NULL COMMENT \"\", \n" + - " `id_decimal` decimal(10,2) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`t1a`)\n" + - "PARTITION BY RANGE (id_date)\n" + - "(\n" + - "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + - "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"), \n" + - "PARTITION p3 VALUES LESS THAN (\"2014-12-01\") \n" + - ")\n" + - "DISTRIBUTED BY HASH(`t1a`) BUCKETS 3\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\"\n" + - ");"); + " `t1a` varchar(20) NULL COMMENT \"\",\n" + + " `t1b` smallint(6) NULL COMMENT \"\",\n" + + " `t1c` int(11) NULL COMMENT \"\",\n" + + " `t1d` bigint(20) NULL COMMENT \"\",\n" + + " `t1e` float NULL COMMENT \"\",\n" + + " `t1f` double NULL COMMENT \"\",\n" + + " `t1g` bigint(20) NULL COMMENT \"\",\n" + + " `id_datetime` datetime NULL COMMENT \"\",\n" + + " `id_date` date NULL COMMENT \"\", \n" + + " `id_decimal` decimal(10,2) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`t1a`)\n" + + "PARTITION BY RANGE (id_date)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"), \n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\") \n" + + ")\n" + + "DISTRIBUTED BY HASH(`t1a`) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\"\n" + + ");"); starRocksAssert.withTable("CREATE TABLE `test_all_type_day_partition` (\n" + - " `t1a` varchar(20) NULL COMMENT \"\",\n" + - " `t1b` smallint(6) NULL COMMENT \"\",\n" + - " `t1c` int(11) NULL COMMENT \"\",\n" + - " `t1d` bigint(20) NULL COMMENT \"\",\n" + - " `t1e` float NULL COMMENT \"\",\n" + - " `t1f` double NULL COMMENT \"\",\n" + - " `t1g` bigint(20) NULL COMMENT \"\",\n" + - " `id_datetime` datetime NULL COMMENT \"\",\n" + - " `id_date` date NULL COMMENT \"\", \n" + - " `id_decimal` decimal(10,2) NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`t1a`)\n" + - "PARTITION BY RANGE (id_date)\n" + - "(\n" + - "partition p1 values [('2020-04-23'), ('2020-04-24')),\n" + - "partition p2 values [('2020-04-24'), ('2020-04-25')),\n" + - "partition p3 values [('2020-04-25'), ('2020-04-26')) \n" + - ")\n" + - "DISTRIBUTED BY HASH(`t1a`) BUCKETS 3\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\"\n" + - ");"); + " `t1a` varchar(20) NULL COMMENT \"\",\n" + + " `t1b` smallint(6) NULL COMMENT \"\",\n" + + " `t1c` int(11) NULL COMMENT \"\",\n" + + " `t1d` bigint(20) NULL COMMENT \"\",\n" + + " `t1e` float NULL COMMENT \"\",\n" + + " `t1f` double NULL COMMENT \"\",\n" + + " `t1g` bigint(20) NULL COMMENT \"\",\n" + + " `id_datetime` datetime NULL COMMENT \"\",\n" + + " `id_date` date NULL COMMENT \"\", \n" + + " `id_decimal` decimal(10,2) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`t1a`)\n" + + "PARTITION BY RANGE (id_date)\n" + + "(\n" + + "partition p1 values [('2020-04-23'), ('2020-04-24')),\n" + + "partition p2 values [('2020-04-24'), ('2020-04-25')),\n" + + "partition p3 values [('2020-04-25'), ('2020-04-26')) \n" + + ")\n" + + "DISTRIBUTED BY HASH(`t1a`) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\"\n" + + ");"); } @After @@ -176,7 +176,7 @@ public void testLogicalAggregationRowCount() throws Exception { groupExpression.setGroup(new Group(1)); ExpressionContext expressionContext = new ExpressionContext(groupExpression); StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); Assert.assertEquals(50, expressionContext.getStatistics().getOutputRowCount(), 0.001); @@ -186,11 +186,11 @@ public void testLogicalAggregationRowCount() throws Exception { groupExpression.setGroup(new Group(1)); expressionContext = new ExpressionContext(groupExpression); statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); Assert.assertEquals( - 50 * 50 * Math.pow(StatisticsEstimateCoefficient.UNKNOWN_GROUP_BY_CORRELATION_COEFFICIENT, 2), - expressionContext.getStatistics().getOutputRowCount(), 0.001); + 50 * 50 * Math.pow(StatisticsEstimateCoefficient.UNKNOWN_GROUP_BY_CORRELATION_COEFFICIENT, 2), + expressionContext.getStatistics().getOutputRowCount(), 0.001); } @Test @@ -220,13 +220,13 @@ public void testLogicalUnion() throws Exception { childGroup2.setStatistics(childBuilder2.build()); // construct group expression LogicalUnionOperator unionOperator = new LogicalUnionOperator(Lists.newArrayList(v5, v6), - Lists.newArrayList(Lists.newArrayList(v1, v2), Lists.newArrayList(v3, v4)), true); + Lists.newArrayList(Lists.newArrayList(v1, v2), Lists.newArrayList(v3, v4)), true); GroupExpression groupExpression = - new GroupExpression(unionOperator, Lists.newArrayList(childGroup1, childGroup2)); + new GroupExpression(unionOperator, Lists.newArrayList(childGroup1, childGroup2)); groupExpression.setGroup(new Group(2)); ExpressionContext expressionContext = new ExpressionContext(groupExpression); StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); ColumnStatistic columnStatisticV5 = expressionContext.getStatistics().getColumnStatistic(v5); @@ -239,10 +239,10 @@ public void testLogicalUnion() throws Exception { @Test public void testLogicalOlapTableScan() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table = (OlapTable) globalStateMgr.getDb("statistics_test").getTable("test_all_type"); + OlapTable table = (OlapTable) globalStateMgr.getLocalMetastore().getDb("statistics_test").getTable("test_all_type"); Collection partitions = table.getPartitions(); List partitionIds = - partitions.stream().mapToLong(partition -> partition.getId()).boxed().collect(Collectors.toList()); + partitions.stream().mapToLong(partition -> partition.getId()).boxed().collect(Collectors.toList()); for (Partition partition : partitions) { partition.getBaseIndex().setRowCount(1000); } @@ -257,32 +257,32 @@ public void testLogicalOlapTableScan() throws Exception { columnToRef.put(column, ref); LogicalOlapScanOperator olapScanOperator = new LogicalOlapScanOperator(table, - refToColumn, columnToRef, - null, -1, null, - ((OlapTable) table).getBaseIndexId(), - partitionIds, - null, - false, - Lists.newArrayList(), - Lists.newArrayList(), - Lists.newArrayList(), - false); + refToColumn, columnToRef, + null, -1, null, + ((OlapTable) table).getBaseIndexId(), + partitionIds, + null, + false, + Lists.newArrayList(), + Lists.newArrayList(), + Lists.newArrayList(), + false); GroupExpression groupExpression = new GroupExpression(olapScanOperator, Lists.newArrayList()); groupExpression.setGroup(new Group(0)); ExpressionContext expressionContext = new ExpressionContext(groupExpression); Statistics.Builder builder = Statistics.builder(); olapScanOperator.getOutputColumns().forEach(col -> - builder.addColumnStatistic(col, - new ColumnStatistic(-100, 100, 0.0, 5.0, 10)) + builder.addColumnStatistic(col, + new ColumnStatistic(-100, 100, 0.0, 5.0, 10)) ); expressionContext.setStatistics(builder.build()); StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); Assert.assertEquals(1000 * partitions.size(), expressionContext.getStatistics().getOutputRowCount(), 0.001); Assert.assertEquals(ref.getType().getTypeSize() * 1000 * partitions.size(), - expressionContext.getStatistics().getComputeSize(), 0.001); + expressionContext.getStatistics().getComputeSize(), 0.001); } } @@ -306,9 +306,9 @@ public void testLogicalIcebergTableScan() { } BinaryPredicateOperator predicateOperator = new BinaryPredicateOperator(BinaryType.LT, - partitionColumn, ConstantOperator.createInt(50)); + partitionColumn, ConstantOperator.createInt(50)); LogicalIcebergScanOperator icebergScanOperator = new LogicalIcebergScanOperator(icebergTable, refToColumn, - columnToRef, -1, predicateOperator, TableVersionRange.empty()); + columnToRef, -1, predicateOperator, TableVersionRange.empty()); GroupExpression groupExpression = new GroupExpression(icebergScanOperator, Lists.newArrayList()); groupExpression.setGroup(new Group(0)); @@ -317,22 +317,22 @@ public void testLogicalIcebergTableScan() { new MockUp() { @Mock public Statistics getTableStatisticsFromInternalStatistics(Table table, Map columns) { + Column> columns) { Statistics.Builder builder = Statistics.builder(); icebergScanOperator.getOutputColumns().forEach(col -> - builder.addColumnStatistic(col, - new ColumnStatistic(0, 100, 0.0, 5.0, 100)) + builder.addColumnStatistic(col, + new ColumnStatistic(0, 100, 0.0, 5.0, 100)) ); builder.setOutputRowCount(100); return builder.build(); } }; StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); Assert.assertEquals(50, expressionContext.getStatistics().getOutputRowCount(), 0.001); Assert.assertEquals(50, expressionContext.getStatistics(). - getColumnStatistic(partitionColumn).getMaxValue(), 0.001); + getColumnStatistic(partitionColumn).getMaxValue(), 0.001); Assert.assertTrue(optimizerContext.isObtainedFromInternalStatistics()); optimizerContext.setObtainedFromInternalStatistics(false); } @@ -341,7 +341,7 @@ public Statistics getTableStatisticsFromInternalStatistics(Table table, Map partitions = new ArrayList<>(((OlapTable) table).getPartitions()); @@ -353,41 +353,41 @@ public void testLogicalOlapTableEmptyPartition(@Mocked CachedStatisticStorage ca partition2.setVisibleVersion(2, System.currentTimeMillis()); partition3.setVisibleVersion(2, System.currentTimeMillis()); List partitionIds = partitions.stream().filter(partition -> !(partition.getName().equalsIgnoreCase("p1"))). - mapToLong(Partition::getId).boxed().collect(Collectors.toList()); + mapToLong(Partition::getId).boxed().collect(Collectors.toList()); new Expectations() { { cachedStatisticStorage.getColumnStatistics(table, Lists.newArrayList("id_date")); result = new ColumnStatistic(0, Utils.getLongFromDateTime(LocalDateTime.of(2014, 12, 01, 0, 0, 0)), - 0, 0, 30); + 0, 0, 30); minTimes = 0; cachedStatisticStorage.getColumnStatistic(table, "id_date"); result = new ColumnStatistic(0, Utils.getLongFromDateTime(LocalDateTime.of(2014, 12, 01, 0, 0, 0)), - 0, 0, 30); + 0, 0, 30); minTimes = 0; } }; LogicalOlapScanOperator olapScanOperator = - new LogicalOlapScanOperator(table, - ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), - ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), - null, -1, null, - ((OlapTable) table).getBaseIndexId(), - partitionIds, - null, - false, - Lists.newArrayList(), - Lists.newArrayList(), - Lists.newArrayList(), - false); + new LogicalOlapScanOperator(table, + ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), + ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), + null, -1, null, + ((OlapTable) table).getBaseIndexId(), + partitionIds, + null, + false, + Lists.newArrayList(), + Lists.newArrayList(), + Lists.newArrayList(), + false); GroupExpression groupExpression = new GroupExpression(olapScanOperator, Lists.newArrayList()); groupExpression.setGroup(new Group(0)); ExpressionContext expressionContext = new ExpressionContext(groupExpression); StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); ColumnStatistic columnStatistic = expressionContext.getStatistics().getColumnStatistic(idDate); Assert.assertEquals(30, columnStatistic.getDistinctValuesCount(), 0.001); @@ -396,23 +396,23 @@ public void testLogicalOlapTableEmptyPartition(@Mocked CachedStatisticStorage ca @Test public void testLogicalOlapTableScanPartitionPrune1(@Mocked CachedStatisticStorage cachedStatisticStorage) - throws Exception { + throws Exception { FeConstants.runningUnitTest = true; ColumnRefOperator idDate = columnRefFactory.create("id_date", Type.DATE, true); GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - Table table = globalStateMgr.getDb("statistics_test").getTable("test_all_type"); + Table table = globalStateMgr.getLocalMetastore().getDb("statistics_test").getTable("test_all_type"); new Expectations() { { cachedStatisticStorage.getColumnStatistics(table, Lists.newArrayList("id_date")); result = new ColumnStatistic(0, Utils.getLongFromDateTime(LocalDateTime.of(2014, 12, 01, 0, 0, 0)), - 0, 0, 30); + 0, 0, 30); minTimes = 0; cachedStatisticStorage.getColumnStatistic(table, "id_date"); result = new ColumnStatistic(0, Utils.getLongFromDateTime(LocalDateTime.of(2014, 12, 01, 0, 0, 0)), - 0, 0, 30); + 0, 0, 30); minTimes = 0; } }; @@ -420,95 +420,96 @@ public void testLogicalOlapTableScanPartitionPrune1(@Mocked CachedStatisticStora Collection partitions = ((OlapTable) table).getPartitions(); // select partition p1 List partitionIds = partitions.stream().filter(partition -> partition.getName().equalsIgnoreCase("p1")). - mapToLong(Partition::getId).boxed().collect(Collectors.toList()); + mapToLong(Partition::getId).boxed().collect(Collectors.toList()); for (Partition partition : partitions) { partition.getBaseIndex().setRowCount(1000); } LogicalOlapScanOperator olapScanOperator = - new LogicalOlapScanOperator(table, - ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), - ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), - null, -1, - new BinaryPredicateOperator(BinaryType.EQ, - idDate, ConstantOperator.createDate(LocalDateTime.of(2013, 12, 30, 0, 0, 0))), - ((OlapTable) table).getBaseIndexId(), - partitionIds, - null, - false, - Lists.newArrayList(), - Lists.newArrayList(), - Lists.newArrayList(), - false); + new LogicalOlapScanOperator(table, + ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), + ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), + null, -1, + new BinaryPredicateOperator(BinaryType.EQ, + idDate, ConstantOperator.createDate(LocalDateTime.of(2013, 12, 30, 0, 0, 0))), + ((OlapTable) table).getBaseIndexId(), + partitionIds, + null, + false, + Lists.newArrayList(), + Lists.newArrayList(), + Lists.newArrayList(), + false); GroupExpression groupExpression = new GroupExpression(olapScanOperator, Lists.newArrayList()); groupExpression.setGroup(new Group(0)); ExpressionContext expressionContext = new ExpressionContext(groupExpression); StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); // partition column count distinct values is 30 in table level, after partition prune, // the column statistic distinct values is 10, so the estimate row count is 1000 * (1/10) Assert.assertEquals(100, expressionContext.getStatistics().getOutputRowCount(), 0.001); ColumnStatistic columnStatistic = expressionContext.getStatistics().getColumnStatistic(idDate); Assert.assertEquals(Utils.getLongFromDateTime(LocalDateTime.of(2013, 12, 30, 0, 0, 0)), - columnStatistic.getMaxValue(), 0.001); + columnStatistic.getMaxValue(), 0.001); // select partition p2, p3 partitionIds.clear(); partitionIds = partitions.stream().filter(partition -> !(partition.getName().equalsIgnoreCase("p1"))). - mapToLong(Partition::getId).boxed().collect(Collectors.toList()); + mapToLong(Partition::getId).boxed().collect(Collectors.toList()); olapScanOperator = - new LogicalOlapScanOperator(table, - ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), - ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), - null, -1, null, ((OlapTable) table).getBaseIndexId(), - partitionIds, - null, - false, - Lists.newArrayList(), - Lists.newArrayList(), - Lists.newArrayList(), - false); + new LogicalOlapScanOperator(table, + ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), + ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), + null, -1, null, ((OlapTable) table).getBaseIndexId(), + partitionIds, + null, + false, + Lists.newArrayList(), + Lists.newArrayList(), + Lists.newArrayList(), + false); olapScanOperator.setPredicate(new BinaryPredicateOperator(BinaryType.GE, - idDate, ConstantOperator.createDate(LocalDateTime.of(2014, 5, 1, 0, 0, 0)))); + idDate, ConstantOperator.createDate(LocalDateTime.of(2014, 5, 1, 0, 0, 0)))); groupExpression = new GroupExpression(olapScanOperator, Lists.newArrayList()); groupExpression.setGroup(new Group(0)); expressionContext = new ExpressionContext(groupExpression); statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); columnStatistic = expressionContext.getStatistics().getColumnStatistic(idDate); Assert.assertEquals(1281.4371, expressionContext.getStatistics().getOutputRowCount(), 0.001); Assert.assertEquals(Utils.getLongFromDateTime(LocalDateTime.of(2014, 5, 1, 0, 0, 0)), - columnStatistic.getMinValue(), 0.001); + columnStatistic.getMinValue(), 0.001); Assert.assertEquals(Utils.getLongFromDateTime(LocalDateTime.of(2014, 12, 1, 0, 0, 0)), - columnStatistic.getMaxValue(), 0.001); + columnStatistic.getMaxValue(), 0.001); Assert.assertEquals(20, columnStatistic.getDistinctValuesCount(), 0.001); FeConstants.runningUnitTest = false; } @Test public void testLogicalOlapTableScanPartitionPrune2(@Mocked CachedStatisticStorage cachedStatisticStorage) - throws Exception { + throws Exception { FeConstants.runningUnitTest = true; ColumnRefOperator idDate = columnRefFactory.create("id_date", Type.DATE, true); GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table = (OlapTable) globalStateMgr.getDb("statistics_test").getTable("test_all_type_day_partition"); + OlapTable table = (OlapTable) globalStateMgr.getLocalMetastore().getDb("statistics_test") + .getTable("test_all_type_day_partition"); new Expectations() { { cachedStatisticStorage.getColumnStatistics(table, Lists.newArrayList("id_date")); result = new ColumnStatistic(Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 23, 0, 0, 0)), - Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 25, 0, 0, 0)), 0, 0, 3); + Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 25, 0, 0, 0)), 0, 0, 3); minTimes = 0; cachedStatisticStorage.getColumnStatistic(table, "id_date"); result = new ColumnStatistic(Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 23, 0, 0, 0)), - Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 25, 0, 0, 0)), 0, 0, 3); + Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 25, 0, 0, 0)), 0, 0, 3); minTimes = 0; } }; @@ -516,57 +517,57 @@ public void testLogicalOlapTableScanPartitionPrune2(@Mocked CachedStatisticStora Collection partitions = table.getPartitions(); // select partition p2 List partitionIds = partitions.stream().filter(partition -> partition.getName().equalsIgnoreCase("p2")). - mapToLong(partition -> partition.getId()).boxed().collect(Collectors.toList()); + mapToLong(partition -> partition.getId()).boxed().collect(Collectors.toList()); for (Partition partition : partitions) { partition.getBaseIndex().setRowCount(1000); } LogicalOlapScanOperator olapScanOperator = - new LogicalOlapScanOperator(table, - ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), - ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), null, -1, null, - ((OlapTable) table).getBaseIndexId(), - partitionIds, - null, - false, - Lists.newArrayList(), - Lists.newArrayList(), - Lists.newArrayList(), - false); + new LogicalOlapScanOperator(table, + ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), + ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), null, -1, null, + ((OlapTable) table).getBaseIndexId(), + partitionIds, + null, + false, + Lists.newArrayList(), + Lists.newArrayList(), + Lists.newArrayList(), + false); GroupExpression groupExpression = new GroupExpression(olapScanOperator, Lists.newArrayList()); groupExpression.setGroup(new Group(0)); ExpressionContext expressionContext = new ExpressionContext(groupExpression); StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); statisticsCalculator.estimatorStats(); Assert.assertEquals(1000, expressionContext.getStatistics().getOutputRowCount(), 0.001); ColumnStatistic columnStatistic = expressionContext.getStatistics().getColumnStatistic(idDate); Assert.assertEquals(Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 24, 0, 0, 0)), - columnStatistic.getMinValue(), 0.001); + columnStatistic.getMinValue(), 0.001); Assert.assertEquals(Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 25, 0, 0, 0)), - columnStatistic.getMaxValue(), 0.001); + columnStatistic.getMaxValue(), 0.001); Assert.assertEquals(1, columnStatistic.getDistinctValuesCount(), 0.001); // select partition p2, p3 partitionIds.clear(); partitionIds = partitions.stream().filter(partition -> !(partition.getName().equalsIgnoreCase("p1"))). - mapToLong(Partition::getId).boxed().collect(Collectors.toList()); + mapToLong(Partition::getId).boxed().collect(Collectors.toList()); olapScanOperator = - new LogicalOlapScanOperator(table, - ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), - ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), null, -1, null, - ((OlapTable) table).getBaseIndexId(), - partitionIds, - null, - false, - Lists.newArrayList(), - Lists.newArrayList(), - Lists.newArrayList(), - false); + new LogicalOlapScanOperator(table, + ImmutableMap.of(idDate, new Column("id_date", Type.DATE, true)), + ImmutableMap.of(new Column("id_date", Type.DATE, true), idDate), null, -1, null, + ((OlapTable) table).getBaseIndexId(), + partitionIds, + null, + false, + Lists.newArrayList(), + Lists.newArrayList(), + Lists.newArrayList(), + false); olapScanOperator.setPredicate(new BinaryPredicateOperator(BinaryType.GE, - idDate, ConstantOperator.createDate(LocalDateTime.of(2020, 04, 24, 0, 0, 0)))); + idDate, ConstantOperator.createDate(LocalDateTime.of(2020, 04, 24, 0, 0, 0)))); groupExpression = new GroupExpression(olapScanOperator, Lists.newArrayList()); groupExpression.setGroup(new Group(0)); @@ -577,9 +578,9 @@ public void testLogicalOlapTableScanPartitionPrune2(@Mocked CachedStatisticStora // has two partitions Assert.assertEquals(2000, expressionContext.getStatistics().getOutputRowCount(), 0.001); Assert.assertEquals(Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 24, 0, 0, 0)), - columnStatistic.getMinValue(), 0.001); + columnStatistic.getMinValue(), 0.001); Assert.assertEquals(Utils.getLongFromDateTime(LocalDateTime.of(2020, 4, 26, 0, 0, 0)), - columnStatistic.getMaxValue(), 0.001); + columnStatistic.getMaxValue(), 0.001); Assert.assertEquals(2, columnStatistic.getDistinctValuesCount(), 0.001); FeConstants.runningUnitTest = false; } @@ -622,21 +623,21 @@ public void testJoinEstimateWithMultiColumns() { columnRefFactory.updateColumnToRelationIds(v6.getId(), 4); // on predicate : t0.v1 = t1.v3 and t0.v2 = t1.v4 BinaryPredicateOperator eqOnPredicate1 = - new BinaryPredicateOperator(BinaryType.EQ, v1, v3); + new BinaryPredicateOperator(BinaryType.EQ, v1, v3); BinaryPredicateOperator eqOnPredicate2 = - new BinaryPredicateOperator(BinaryType.EQ, v2, v4); + new BinaryPredicateOperator(BinaryType.EQ, v2, v4); BinaryPredicateOperator eqOnPredicate3 = - new BinaryPredicateOperator(BinaryType.EQ, v5, v6); + new BinaryPredicateOperator(BinaryType.EQ, v5, v6); // construct group expression LogicalJoinOperator joinOperator = - new LogicalJoinOperator(JoinOperator.INNER_JOIN, new CompoundPredicateOperator( - CompoundPredicateOperator.CompoundType.AND, eqOnPredicate1, eqOnPredicate2)); + new LogicalJoinOperator(JoinOperator.INNER_JOIN, new CompoundPredicateOperator( + CompoundPredicateOperator.CompoundType.AND, eqOnPredicate1, eqOnPredicate2)); GroupExpression groupExpression = - new GroupExpression(joinOperator, Lists.newArrayList(childGroup1, childGroup2)); + new GroupExpression(joinOperator, Lists.newArrayList(childGroup1, childGroup2)); groupExpression.setGroup(new Group(2)); ExpressionContext expressionContext = new ExpressionContext(groupExpression); StatisticsCalculator statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); // use middle ground method to estimate ConnectContext.get().getSessionVariable().setUseCorrelatedJoinEstimate(false); statisticsCalculator.estimatorStats(); @@ -657,13 +658,13 @@ public void testJoinEstimateWithMultiColumns() { // on predicate : t0.v1 = t1.v3 and t0.v2 = t2.v4 and t3.v5 = t4.v6 // construct group expression joinOperator = new LogicalJoinOperator(JoinOperator.INNER_JOIN, new CompoundPredicateOperator( - CompoundPredicateOperator.CompoundType.AND, eqOnPredicate1, new CompoundPredicateOperator( - CompoundPredicateOperator.CompoundType.AND, eqOnPredicate2, eqOnPredicate3))); + CompoundPredicateOperator.CompoundType.AND, eqOnPredicate1, new CompoundPredicateOperator( + CompoundPredicateOperator.CompoundType.AND, eqOnPredicate2, eqOnPredicate3))); groupExpression = new GroupExpression(joinOperator, Lists.newArrayList(childGroup1, childGroup2)); groupExpression.setGroup(new Group(2)); expressionContext = new ExpressionContext(groupExpression); statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); // use middle ground method to estimate ConnectContext.get().getSessionVariable().setUseCorrelatedJoinEstimate(false); statisticsCalculator.estimatorStats(); @@ -671,18 +672,18 @@ CompoundPredicateOperator.CompoundType.AND, eqOnPredicate1, new CompoundPredicat // on predicate : t0.v1 = t1.v3 + t1.v4 and t0.v2 = t1.v3 + t1.v4 BinaryPredicateOperator eqOnPredicateWithAdd1 = - new BinaryPredicateOperator(BinaryType.EQ, v1, - new CallOperator("add", Type.BIGINT, Lists.newArrayList(v3, v4))); + new BinaryPredicateOperator(BinaryType.EQ, v1, + new CallOperator("add", Type.BIGINT, Lists.newArrayList(v3, v4))); BinaryPredicateOperator eqOnPredicateWithAdd2 = - new BinaryPredicateOperator(BinaryType.EQ, v2, - new CallOperator("add", Type.BIGINT, Lists.newArrayList(v3, v4))); + new BinaryPredicateOperator(BinaryType.EQ, v2, + new CallOperator("add", Type.BIGINT, Lists.newArrayList(v3, v4))); joinOperator = new LogicalJoinOperator(JoinOperator.INNER_JOIN, new CompoundPredicateOperator( - CompoundPredicateOperator.CompoundType.AND, eqOnPredicateWithAdd1, eqOnPredicateWithAdd2)); + CompoundPredicateOperator.CompoundType.AND, eqOnPredicateWithAdd1, eqOnPredicateWithAdd2)); groupExpression = new GroupExpression(joinOperator, Lists.newArrayList(childGroup1, childGroup2)); groupExpression.setGroup(new Group(2)); expressionContext = new ExpressionContext(groupExpression); statisticsCalculator = new StatisticsCalculator(expressionContext, - columnRefFactory, optimizerContext); + columnRefFactory, optimizerContext); // use middle ground method to estimate ConnectContext.get().getSessionVariable().setUseCorrelatedJoinEstimate(false); statisticsCalculator.estimatorStats(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/CTEPlanTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/CTEPlanTest.java index 93f70e381b152..37cbabe328118 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/CTEPlanTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/CTEPlanTest.java @@ -42,10 +42,10 @@ public static void beforeClass() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); globalStateMgr.setStatisticStorage(new TestStorage()); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 20000000); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 2000000); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanTestBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanTestBase.java index 44cfc1dcf4023..119364ffa5bd8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanTestBase.java @@ -111,45 +111,45 @@ public static void beforeClass() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); int scale = 100; connectContext.getGlobalStateMgr().setStatisticStorage(new MockTpchStatisticStorage(connectContext, scale)); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("region"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("region"); setTableStatistics(t0, 5); - OlapTable t5 = (OlapTable) globalStateMgr.getDb("test").getTable("nation"); + OlapTable t5 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("nation"); setTableStatistics(t5, 25); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("supplier"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("supplier"); setTableStatistics(t1, 10000 * scale); - OlapTable t4 = (OlapTable) globalStateMgr.getDb("test").getTable("customer"); + OlapTable t4 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("customer"); setTableStatistics(t4, 150000 * scale); - OlapTable t6 = (OlapTable) globalStateMgr.getDb("test").getTable("part"); + OlapTable t6 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("part"); setTableStatistics(t6, 200000 * scale); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("partsupp"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("partsupp"); setTableStatistics(t2, 800000 * scale); - OlapTable t3 = (OlapTable) globalStateMgr.getDb("test").getTable("orders"); + OlapTable t3 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("orders"); setTableStatistics(t3, 1500000 * scale); - OlapTable t7 = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem"); setTableStatistics(t7, 6000000 * scale); - OlapTable t8 = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem_partition"); + OlapTable t8 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem_partition"); setTableStatistics(t8, 6000000 * scale); - OlapTable testAllType = (OlapTable) globalStateMgr.getDb("test").getTable("test_all_type"); + OlapTable testAllType = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("test_all_type"); setTableStatistics(testAllType, 6000000); OlapTable lineorderNewL = - (OlapTable) globalStateMgr.getDb("test").getTable("lineorder_new_l"); + (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineorder_new_l"); setTableStatistics(lineorderNewL, 1200018434); - OlapTable datesN = (OlapTable) globalStateMgr.getDb("test").getTable("dates_n"); + OlapTable datesN = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("dates_n"); setTableStatistics(datesN, 2556); OlapTable skewTable = - (OlapTable) globalStateMgr.getDb("test").getTable("skew_table"); + (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("skew_table"); setTableStatistics(skewTable, 10000000); UtFrameUtils.addMockBackend(10002); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanWithCostTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanWithCostTest.java index fda82fcb680c6..29a6dfcf1c75f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanWithCostTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvPlanWithCostTest.java @@ -949,7 +949,7 @@ public void testPartitionColumnColumnStatistics() throws Exception { @Test public void testCastDatePredicate() throws Exception { OlapTable lineitem = - (OlapTable) connectContext.getGlobalStateMgr().getDb("test").getTable("lineitem"); + (OlapTable) connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test").getTable("lineitem"); MockTpchStatisticStorage mock = new MockTpchStatisticStorage(connectContext, 100); connectContext.getGlobalStateMgr().setStatisticStorage(mock); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvTPCDSPlanTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvTPCDSPlanTest.java index 4ddd581dd276f..6a235b35fc095 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvTPCDSPlanTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/DistributedEnvTPCDSPlanTest.java @@ -30,19 +30,19 @@ public static void beforeClass() throws Exception { TPCDSPlanTest.beforeClass(); GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); OlapTable customerAddress = - (OlapTable) globalStateMgr.getDb("test").getTable("customer_address"); + (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("customer_address"); setTableStatistics(customerAddress, 1000000); - OlapTable customer = (OlapTable) globalStateMgr.getDb("test").getTable("customer"); + OlapTable customer = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("customer"); setTableStatistics(customer, 2000000); - OlapTable storeSales = (OlapTable) globalStateMgr.getDb("test").getTable("store_sales"); + OlapTable storeSales = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("store_sales"); setTableStatistics(storeSales, 287997024); - OlapTable dateDim = (OlapTable) globalStateMgr.getDb("test").getTable("date_dim"); + OlapTable dateDim = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("date_dim"); setTableStatistics(dateDim, 73048); - OlapTable item = (OlapTable) globalStateMgr.getDb("test").getTable("item"); + OlapTable item = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("item"); setTableStatistics(item, 203999); UtFrameUtils.addMockBackend(10002); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java index 65fd76353b75d..4060dcb388193 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java @@ -236,8 +236,8 @@ public void testMysqlJoinSelf() throws Exception { @Test public void testJoinWithMysqlTable() throws Exception { // set data size and row count for the olap table - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = (OlapTable) db.getTable("jointest"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "jointest"); for (Partition partition : tbl.getPartitions()) { partition.updateVisibleVersion(2); for (MaterializedIndex mIndex : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/GroupingSetsTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/GroupingSetsTest.java index b386116980265..b31ef47682602 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/GroupingSetsTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/GroupingSetsTest.java @@ -30,7 +30,7 @@ public static void beforeClass() throws Exception { PlanTestBase.beforeClass(); Config.alter_scheduler_interval_millisecond = 1; GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, NUM_TABLE0_ROWS); FeConstants.runningUnitTest = true; } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/JoinTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/JoinTest.java index 2f34fb36536e0..e277d97fe9501 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/JoinTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/JoinTest.java @@ -1125,8 +1125,8 @@ public void testShuffleHashBucket2() throws Exception { @Test public void testJoinReorderTakeEffect() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - Database db = globalStateMgr.getDb("test"); - Table table = db.getTable("join2"); + Database db = globalStateMgr.getLocalMetastore().getDb("test"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "join2"); OlapTable olapTable1 = (OlapTable) table; try { setTableStatistics(olapTable1, 2); @@ -1149,7 +1149,7 @@ public void testJoinReorderTakeEffect() throws Exception { public void testJoinReorderWithWithClause() throws Exception { connectContext.setDatabase("test"); GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - Table table = globalStateMgr.getDb("test").getTable("join2"); + Table table = globalStateMgr.getLocalMetastore().getDb("test").getTable("join2"); OlapTable olapTable1 = (OlapTable) table; try { setTableStatistics(olapTable1, 2); @@ -1652,7 +1652,7 @@ public void testOuterJoinEliminate() throws Exception { @Test public void testJoinReorderWithReanalyze() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - Table table = globalStateMgr.getDb("test").getTable("join2"); + Table table = globalStateMgr.getLocalMetastore().getDb("test").getTable("join2"); OlapTable olapTable1 = (OlapTable) table; try { setTableStatistics(olapTable1, 2); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java index 9a40c8050172c..b39d74674c052 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java @@ -787,7 +787,7 @@ public void testLimitPrune() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); // We need to let some tablets have data, some tablets don't data - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); MaterializedIndex index = t0.getPartitions().stream().findFirst().get().getBaseIndex(); LocalTablet tablets = (LocalTablet) index.getTablets().get(0); Replica replica = tablets.getSingleReplica(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/MockTpchStatisticStorage.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/MockTpchStatisticStorage.java index c77762a2682c1..453631b3e0eb1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/MockTpchStatisticStorage.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/MockTpchStatisticStorage.java @@ -58,16 +58,16 @@ public MockTpchStatisticStorage(ConnectContext connectContext, int scale) { private void mockTpchTableStats(ConnectContext connectContext, int scale) { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - Database database = globalStateMgr.getDb("test"); + Database database = globalStateMgr.getLocalMetastore().getDb("test"); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("region"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("region"); rowCountStats.put(t0.getId(), Optional.of(5L)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t0.getId(), null, - StatsConstants.AnalyzeType.FULL, - LocalDateTime.of(2020, 1, 1, 1, 1, 1), - Maps.newHashMap())); + StatsConstants.AnalyzeType.FULL, + LocalDateTime.of(2020, 1, 1, 1, 1, 1), + Maps.newHashMap())); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("nation"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("nation"); rowCountStats.put(t1.getId(), Optional.of( 25L)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t1.getId(), null, @@ -75,7 +75,7 @@ private void mockTpchTableStats(ConnectContext connectContext, int scale) { LocalDateTime.of(2020, 1, 1, 1, 1, 1), Maps.newHashMap())); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("supplier"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("supplier"); rowCountStats.put(t2.getId(), Optional.of( 10000L * scale)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t2.getId(), null, @@ -83,7 +83,7 @@ private void mockTpchTableStats(ConnectContext connectContext, int scale) { LocalDateTime.of(2020, 1, 1, 1, 1, 1), Maps.newHashMap())); - OlapTable t3 = (OlapTable) globalStateMgr.getDb("test").getTable("customer"); + OlapTable t3 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("customer"); rowCountStats.put(t3.getId(), Optional.of( 150000L * scale)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t3.getId(), null, @@ -91,7 +91,7 @@ private void mockTpchTableStats(ConnectContext connectContext, int scale) { LocalDateTime.of(2020, 1, 1, 1, 1, 1), Maps.newHashMap())); - OlapTable t4 = (OlapTable) globalStateMgr.getDb("test").getTable("part"); + OlapTable t4 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("part"); rowCountStats.put(t4.getId(), Optional.of( 200000L * scale)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t4.getId(), null, @@ -99,7 +99,7 @@ private void mockTpchTableStats(ConnectContext connectContext, int scale) { LocalDateTime.of(2020, 1, 1, 1, 1, 1), Maps.newHashMap())); - OlapTable t5 = (OlapTable) globalStateMgr.getDb("test").getTable("partsupp"); + OlapTable t5 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("partsupp"); rowCountStats.put(t5.getId(), Optional.of( 800000L * scale)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t5.getId(), null, @@ -107,7 +107,7 @@ private void mockTpchTableStats(ConnectContext connectContext, int scale) { LocalDateTime.of(2020, 1, 1, 1, 1, 1), Maps.newHashMap())); - OlapTable t6 = (OlapTable) globalStateMgr.getDb("test").getTable("orders"); + OlapTable t6 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("orders"); rowCountStats.put(t6.getId(), Optional.of( 1500000L * scale)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t6.getId(), null, @@ -115,7 +115,7 @@ private void mockTpchTableStats(ConnectContext connectContext, int scale) { LocalDateTime.of(2020, 1, 1, 1, 1, 1), Maps.newHashMap())); - OlapTable t7 = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem"); rowCountStats.put(t7.getId(), Optional.of( 6000000L * scale)); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(database.getId(), t7.getId(), null, diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/MultiJoinReorderTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/MultiJoinReorderTest.java index 2c3854d9c8db8..8bd673941d50f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/MultiJoinReorderTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/MultiJoinReorderTest.java @@ -34,15 +34,15 @@ public static void beforeClass() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 1); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 10); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("t2"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t2"); setTableStatistics(t2, 100000); - OlapTable t3 = (OlapTable) globalStateMgr.getDb("test").getTable("t3"); + OlapTable t3 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t3"); setTableStatistics(t3, 1000000000); connectContext.getSessionVariable().setMaxTransformReorderJoins(2); FeConstants.runningUnitTest = true; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java index 0b64ee554beb2..ef79f77413be3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanFragmentWithCostTest.java @@ -64,16 +64,16 @@ public static void beforeClass() throws Exception { Config.alter_scheduler_interval_millisecond = 1; GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table2 = (OlapTable) globalStateMgr.getDb("test").getTable("test_all_type"); + OlapTable table2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("test_all_type"); setTableStatistics(table2, NUM_TABLE2_ROWS); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, NUM_TABLE0_ROWS); - OlapTable colocateT0 = (OlapTable) globalStateMgr.getDb("test").getTable("colocate_t0"); + OlapTable colocateT0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("colocate_t0"); setTableStatistics(colocateT0, NUM_TABLE0_ROWS); - OlapTable lineitem = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem"); + OlapTable lineitem = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem"); setTableStatistics(lineitem, NUM_TABLE0_ROWS * NUM_TABLE0_ROWS); StarRocksAssert starRocksAssert = new StarRocksAssert(connectContext); @@ -181,9 +181,9 @@ public void before() { public void testCrossJoinBroadCast() throws Exception { // check cross join generate plan without exception GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table2 = (OlapTable) globalStateMgr.getDb("test").getTable("test_all_type"); + OlapTable table2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("test_all_type"); setTableStatistics(table2, 20000000); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 20000000); String sql = "select t1a,v1 from test_all_type, t0"; @@ -424,9 +424,9 @@ public void testShuffleInnerJoin() throws Exception { UtFrameUtils.addMockBackend(10002); UtFrameUtils.addMockBackend(10003); GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table1 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable table1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(table1, 10000); - OlapTable table2 = (OlapTable) globalStateMgr.getDb("test").getTable("test_all_type"); + OlapTable table2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("test_all_type"); setTableStatistics(table2, 5000); String sql = "SELECT v2,t1d from t0 join test_all_type on t0.v2 = test_all_type.t1d ;"; String plan = getFragmentPlan(sql); @@ -469,7 +469,7 @@ public void testBroadcastInnerJoin() throws Exception { @Test public void testBroadcastInnerJoinWithCommutativity() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable table = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(table, 1000); String sql = "SELECT * from t0 join test_all_type on t0.v1 = test_all_type.t1d ;"; String plan = getFragmentPlan(sql); @@ -731,9 +731,9 @@ public void testOuterJoinPushDownPredicate() throws Exception { @Test public void testThriftWaitingNodeIds() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 10000000); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 10000); String sql = "select * from t0 inner join t1 on t0.v1 = t1.v4 and t0.v2 = t1.v5 and t0.v1 = 1 and t1.v5 = 2"; @@ -746,11 +746,11 @@ public void testThriftWaitingNodeIds() throws Exception { public void testIntersectReorder() throws Exception { // check cross join generate plan without exception GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 1000); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 100); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("t2"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t2"); setTableStatistics(t2, 1); String sql = "select v1 from t0 intersect select v7 from t2 intersect select v4 from t1"; @@ -774,10 +774,10 @@ public void testIntersectReorder() throws Exception { @Test public void testNotPushDownRuntimeFilterToSortNode() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 10); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 1000000000L); String sql = "select t0.v1 from (select v4 from t1 order by v4 limit 1000000000) as t1x " + @@ -809,10 +809,10 @@ public void testPushMultiColumnRuntimeFiltersCrossDataExchange() throws Exceptio connectContext.getSessionVariable().setGlobalRuntimeFilterBuildMaxSize(0); connectContext.getSessionVariable().setGlobalRuntimeFilterProbeMinSize(0); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 10000000L); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 1000000000L); String sql = "select * from t0 join[shuffle] t1 on t0.v2 = t1.v5 and t0.v3 = t1.v6"; @@ -835,13 +835,13 @@ public void testPushMultiColumnRuntimeFiltersCrossDataExchangeOnMultiJoins() thr connectContext.getSessionVariable().setGlobalRuntimeFilterBuildMaxSize(0); connectContext.getSessionVariable().setGlobalRuntimeFilterProbeMinSize(0); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 10000000L); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 10000000L); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("t2"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t2"); setTableStatistics(t2, 10000000L); String sql = "select * from (select t1.v5 as v5, t0.v3 as v3 from t0 join[shuffle] t1 on t0.v2 = " + @@ -878,11 +878,11 @@ public void testMergeTwoAggArgTypes() throws Exception { public void testPushDownRuntimeFilterAcrossSetOperationNode() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 1000); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 100); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("t2"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t2"); List tables = new ArrayList<>(Arrays.asList(t0, t1, t2)); List tabletIdsStrList = new ArrayList<>(); tables.forEach(olapTable -> tabletIdsStrList.add(Joiner.on(",") @@ -1051,8 +1051,8 @@ public void testNotPushDownRuntimeFilterAcrossCTE() throws Exception { connectContext.getSessionVariable().setEnablePipelineEngine(true); connectContext.getSessionVariable().setCboCTERuseRatio(0); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("t2"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t2"); List tables = new ArrayList<>(Arrays.asList(t1, t2)); List tabletIdsStrList = new ArrayList<>(); tables.forEach(olapTable -> tabletIdsStrList.add(Joiner.on(",") @@ -1119,8 +1119,8 @@ public void testLocalGroupingSetAgg() throws Exception { @Test public void testOverflowFilterOnColocateJoin() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("colocate1"); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("colocate2"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("colocate1"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("colocate2"); StatisticStorage ss = globalStateMgr.getCurrentState().getStatisticStorage(); new Expectations(ss) { @@ -1150,8 +1150,8 @@ public void testOverflowFilterOnColocateJoin() throws Exception { @Test public void testPruneShuffleColumns() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); StatisticStorage ss = globalStateMgr.getCurrentState().getStatisticStorage(); new Expectations(ss) { @@ -1227,7 +1227,7 @@ public void testDateDiffWithStringConstant() throws Exception { @Test public void testToDateToDays() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("test_all_type"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("test_all_type"); StatisticStorage ss = GlobalStateMgr.getCurrentState().getStatisticStorage(); new Expectations(ss) { { @@ -1249,8 +1249,8 @@ public void testToDateToDays() throws Exception { @Test public void testMultiJoinColumnPruneShuffleColumnsAndGRF() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); StatisticStorage ss = GlobalStateMgr.getCurrentState().getStatisticStorage(); new Expectations(ss) { @@ -1973,7 +1973,7 @@ public void testTimeOutOnlyOr20() throws Exception { @Ignore public void testDeepTreePredicate() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table2 = (OlapTable) globalStateMgr.getDb("test").getTable("test_dict"); + OlapTable table2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("test_dict"); setTableStatistics(table2, 20000000); String sql = getSQLFile("optimized-plan/large_predicate"); @@ -1987,12 +1987,12 @@ public void testDeepTreePredicate() throws Exception { @Test public void testJoinUnreorder() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(t0, 10000000); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1"); setTableStatistics(t1, 10000); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("t2"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t2"); setTableStatistics(t1, 10); String sql = "Select * " + @@ -2034,7 +2034,7 @@ public void testJoinUnreorder() throws Exception { @Test public void testPruneLimit() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table2 = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem_partition"); + OlapTable table2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem_partition"); setTableStatistics(table2, 10); new MockUp() { @@ -2054,7 +2054,7 @@ public long getRowCount(long version) { @Test public void testPruneInvalidPredicate() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table2 = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem_partition"); + OlapTable table2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem_partition"); setTableStatistics(table2, 10); new MockUp() { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestBase.java index 46248f4552ef9..1a0d7aef4fd60 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestBase.java @@ -967,7 +967,7 @@ public static void afterClass() { public static void cleanupEphemeralMVs(StarRocksAssert starRocksAssert, long startTime) throws Exception { String currentDb = starRocksAssert.getCtx().getDatabase(); if (StringUtils.isNotEmpty(currentDb)) { - Database testDb = GlobalStateMgr.getCurrentState().getDb(currentDb); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(currentDb); for (MaterializedView mv : ListUtils.emptyIfNull(testDb.getMaterializedViews())) { if (startTime > 0 && mv.getCreateTime() > startTime) { starRocksAssert.dropMaterializedView(mv.getName()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java index 683b70734e583..3d63995d4bef8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java @@ -806,7 +806,7 @@ protected void assertExceptionMsgContains(String sql, String message) { public Table getTable(String t) { GlobalStateMgr globalStateMgr = starRocksAssert.getCtx().getGlobalStateMgr(); - return globalStateMgr.getDb("test").getTable(t); + return globalStateMgr.getLocalMetastore().getDb("test").getTable(t); } public OlapTable getOlapTable(String t) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java index 4437a32bd3fc7..e89844cc4b506 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java @@ -42,28 +42,28 @@ public static void beforeClass() throws Exception { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); connectContext.getSessionVariable().setEnableStatsToOptimizeSkewJoin(true); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("region"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("region"); setTableStatistics(t0, 5); - OlapTable t5 = (OlapTable) globalStateMgr.getDb("test").getTable("nation"); + OlapTable t5 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("nation"); setTableStatistics(t5, 25); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("supplier"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("supplier"); setTableStatistics(t1, 10000 * scale); - OlapTable t4 = (OlapTable) globalStateMgr.getDb("test").getTable("customer"); + OlapTable t4 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("customer"); setTableStatistics(t4, 150000 * scale); - OlapTable t6 = (OlapTable) globalStateMgr.getDb("test").getTable("part"); + OlapTable t6 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("part"); setTableStatistics(t6, 200000 * scale); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("partsupp"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("partsupp"); setTableStatistics(t2, 800000 * scale); - OlapTable t3 = (OlapTable) globalStateMgr.getDb("test").getTable("orders"); + OlapTable t3 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("orders"); setTableStatistics(t3, 1500000 * scale); - OlapTable t7 = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem"); setTableStatistics(t7, 6000000 * scale); starRocksAssert.withTable("create table struct_tbl(c0 INT, " + diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanTest.java index c1df154abdc37..a7c297a1c3904 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanTest.java @@ -43,7 +43,7 @@ public static void beforeClass() throws Exception { @Test public void testJoin() { GlobalStateMgr globalStateMgr = connectContext.getGlobalStateMgr(); - OlapTable table1 = (OlapTable) globalStateMgr.getDb("test").getTable("t0"); + OlapTable table1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0"); setTableStatistics(table1, 10000); runFileUnitTest("optimized-plan/join"); setTableStatistics(table1, 0); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanWithHistogramCostTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanWithHistogramCostTest.java index 93da26f10d635..b3b3bbe1f8b4a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanWithHistogramCostTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/TPCHPlanWithHistogramCostTest.java @@ -45,28 +45,28 @@ public static void beforeClass() throws Exception { connectContext.getGlobalStateMgr().setStatisticStorage(new MockHistogramStatisticStorage(scale)); connectContext.getSessionVariable().setEnableStatsToOptimizeSkewJoin(true); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("region"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("region"); setTableStatistics(t0, 5); - OlapTable t5 = (OlapTable) globalStateMgr.getDb("test").getTable("nation"); + OlapTable t5 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("nation"); setTableStatistics(t5, 25); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("supplier"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("supplier"); setTableStatistics(t1, 10000 * scale); - OlapTable t4 = (OlapTable) globalStateMgr.getDb("test").getTable("customer"); + OlapTable t4 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("customer"); setTableStatistics(t4, 150000 * scale); - OlapTable t6 = (OlapTable) globalStateMgr.getDb("test").getTable("part"); + OlapTable t6 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("part"); setTableStatistics(t6, 200000 * scale); - OlapTable t2 = (OlapTable) globalStateMgr.getDb("test").getTable("partsupp"); + OlapTable t2 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("partsupp"); setTableStatistics(t2, 800000 * scale); - OlapTable t3 = (OlapTable) globalStateMgr.getDb("test").getTable("orders"); + OlapTable t3 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("orders"); setTableStatistics(t3, 1500000 * scale); - OlapTable t7 = (OlapTable) globalStateMgr.getDb("test").getTable("lineitem"); + OlapTable t7 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("lineitem"); setTableStatistics(t7, 6000000 * scale); } diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/CacheLoaderTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/CacheLoaderTest.java index cc4b56f1e40ab..4f46b47f73c15 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/CacheLoaderTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/CacheLoaderTest.java @@ -18,6 +18,7 @@ import com.starrocks.catalog.OlapTable; import com.starrocks.common.jmockit.Deencapsulation; import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.optimizer.statistics.Bucket; import com.starrocks.sql.optimizer.statistics.ColumnBasicStatsCacheLoader; import com.starrocks.sql.optimizer.statistics.ColumnHistogramStatsCacheLoader; @@ -62,7 +63,7 @@ public static void beforeClass() throws Exception { @Test public void testCovertBasicStatistics() { - Database db = connectContext.getGlobalStateMgr().getDb("test"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); OlapTable table = (OlapTable) db.getTable("t0"); ColumnBasicStatsCacheLoader basicStatsCacheLoader = Deencapsulation.newInstance(ColumnBasicStatsCacheLoader.class); @@ -103,8 +104,8 @@ public void testCovertBasicStatistics() { @Test public void testCovertHistogramStatistics() { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t0"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); ColumnHistogramStatsCacheLoader columnHistogramStatsCacheLoader = Deencapsulation.newInstance(ColumnHistogramStatsCacheLoader.class); @@ -146,8 +147,8 @@ public void testCovertHistogramStatistics() { @Test public void testCovertHistogramStatisticsDate() { - Database db = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) db.getTable("t0"); + Database db = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); ColumnHistogramStatsCacheLoader columnHistogramStatsCacheLoader = Deencapsulation.newInstance(ColumnHistogramStatsCacheLoader.class); diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticExecutorTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticExecutorTest.java index ce2c643dc359c..f4a0054cee95b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticExecutorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticExecutorTest.java @@ -32,8 +32,8 @@ public class StatisticExecutorTest extends PlanTestBase { @Test public void testEmpty() throws Exception { StatisticExecutor statisticExecutor = new StatisticExecutor(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable olapTable = (OlapTable) db.getTable("t0"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0"); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(db.getId(), olapTable.getId(), null, StatsConstants.AnalyzeType.FULL, @@ -48,7 +48,7 @@ public void testEmpty() throws Exception { @Test public void testDroppedDB() throws Exception { StatisticExecutor statisticExecutor = new StatisticExecutor(); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(new BasicStatsMeta(db.getId(), 1000, null, StatsConstants.AnalyzeType.FULL, diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java index f9843e934250c..ed74fc726c8d4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java @@ -95,7 +95,7 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0_stats"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0_stats"); t0StatsTableId = t0.getId(); Partition partition = new ArrayList<>(t0.getPartitions()).get(0); partition.updateVisibleVersion(2, t0UpdateTime @@ -114,7 +114,7 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); - OlapTable t1 = (OlapTable) globalStateMgr.getDb("test").getTable("t1_stats"); + OlapTable t1 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t1_stats"); new ArrayList<>(t1.getPartitions()).get(0).updateVisibleVersion(2); setTableStatistics(t1, 20000000); @@ -142,7 +142,7 @@ public static void beforeClass() throws Exception { "\"replication_num\" = \"1\",\n" + "\"in_memory\" = \"false\"\n" + ");"); - OlapTable t0p = (OlapTable) globalStateMgr.getDb("test").getTable("t0_stats_partition"); + OlapTable t0p = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0_stats_partition"); new ArrayList<>(t0p.getPartitions()).get(0).updateVisibleVersion(2); setTableStatistics(t0p, 20000000); @@ -160,7 +160,7 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); - OlapTable tps = (OlapTable) globalStateMgr.getDb("stats").getTable("tprimary_stats"); + OlapTable tps = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("tprimary_stats"); new ArrayList<>(tps.getPartitions()).get(0).updateVisibleVersion(2); setTableStatistics(tps, 20000000); @@ -176,7 +176,7 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); - OlapTable tus = (OlapTable) globalStateMgr.getDb("stats").getTable("tunique_stats"); + OlapTable tus = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("tunique_stats"); new ArrayList<>(tus.getPartitions()).get(0).updateVisibleVersion(2); setTableStatistics(tps, 20000000); @@ -191,7 +191,7 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); - OlapTable tcount = (OlapTable) globalStateMgr.getDb("stats").getTable("tcount"); + OlapTable tcount = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("tcount"); new ArrayList<>(tcount.getPartitions()).get(0).updateVisibleVersion(2); setTableStatistics(tcount, 20000000); @@ -206,7 +206,7 @@ public static void beforeClass() throws Exception { " \"replication_num\" = \"1\"\n" + ");"; starRocksAssert.withTable(createStructTableSql); - OlapTable structTable = (OlapTable) globalStateMgr.getDb("stats").getTable("struct_a"); + OlapTable structTable = (OlapTable) globalStateMgr.getLocalMetastore().getDb("stats").getTable("struct_a"); new ArrayList<>(structTable.getPartitions()).get(0).updateVisibleVersion(2); setTableStatistics(structTable, 20000000); } @@ -239,7 +239,7 @@ public void testAnalyzeALLDB() { @Test public void testAnalyzeDB() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List jobs = StatisticsCollectJobFactory.buildStatisticsCollectJob( new NativeAnalyzeJob(db.getId(), StatsConstants.DEFAULT_ALL_ID, null, null, StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.SCHEDULE, @@ -258,7 +258,7 @@ public void testAnalyzeDB() { @Test public void testAnalyzeTable() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List jobs = StatisticsCollectJobFactory.buildStatisticsCollectJob( new NativeAnalyzeJob(testDb.getId(), t0StatsTableId, null, null, StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.SCHEDULE, @@ -271,8 +271,8 @@ public void testAnalyzeTable() { Assert.assertEquals("t0_stats", fullStatisticsCollectJob.getTable().getName()); Assert.assertEquals("[v1, v2, v3, v4, v5]", fullStatisticsCollectJob.getColumnNames().toString()); - Database db = GlobalStateMgr.getCurrentState().getDb("stats"); - Table table = db.getTable("tprimary_stats"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("stats"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tprimary_stats"); jobs = StatisticsCollectJobFactory.buildStatisticsCollectJob( new NativeAnalyzeJob(db.getId(), table.getId(), null, null, StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.SCHEDULE, @@ -285,7 +285,7 @@ public void testAnalyzeTable() { Assert.assertEquals("tprimary_stats", fullStatisticsCollectJob.getTable().getName()); Assert.assertEquals("[pk, v1, v2]", fullStatisticsCollectJob.getColumnNames().toString()); - table = db.getTable("tunique_stats"); + table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tunique_stats"); jobs = StatisticsCollectJobFactory.buildStatisticsCollectJob( new NativeAnalyzeJob(db.getId(), table.getId(), null, null, StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.SCHEDULE, @@ -301,8 +301,8 @@ public void testAnalyzeTable() { @Test public void testAnalyzeStructSubFiled() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("stats"); - Table table = db.getTable("struct_a"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("stats"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "struct_a"); List jobs = StatisticsCollectJobFactory.buildStatisticsCollectJob( new NativeAnalyzeJob(db.getId(), table.getId(), ImmutableList.of("b.a", "b.c", "d.c.a"), ImmutableList.of(Type.INT, Type.INT, Type.INT), @@ -319,7 +319,7 @@ public void testAnalyzeStructSubFiled() throws Exception { @Test public void testAnalyzeColumn() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List jobs = StatisticsCollectJobFactory.buildStatisticsCollectJob( new NativeAnalyzeJob(db.getId(), t0StatsTableId, Lists.newArrayList("v2"), Lists.newArrayList(Type.BIGINT), @@ -335,7 +335,7 @@ public void testAnalyzeColumn() { @Test public void testAnalyzeColumnSample() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List jobs = StatisticsCollectJobFactory.buildStatisticsCollectJob( new NativeAnalyzeJob(db.getId(), t0StatsTableId, Lists.newArrayList("v2"), Lists.newArrayList(Type.BIGINT), @@ -351,8 +351,9 @@ public void testAnalyzeColumnSample() { @Test public void testAnalyzeColumnSample2() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable olapTable = (OlapTable) db.getTable("t0_stats"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0_stats"); BasicStatsMeta basicStatsMeta = new BasicStatsMeta(db.getId(), olapTable.getId(), null, StatsConstants.AnalyzeType.SAMPLE, @@ -396,8 +397,9 @@ public void testAnalyzeColumnSample2() { @Test public void testAnalyzeHistogram() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable olapTable = (OlapTable) db.getTable("t0_stats"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "t0_stats"); long dbid = db.getId(); Map properties = new HashMap<>(); @@ -461,7 +463,7 @@ public void testAnalyzeHistogram() { @Test public void testNativeAnalyzeJob() { - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); NativeAnalyzeJob nativeAnalyzeJob = new NativeAnalyzeJob(testDb.getId(), t0StatsTableId, null, null, StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.SCHEDULE, @@ -486,7 +488,8 @@ public void testNativeAnalyzeJob() { new MockUp() { @Mock - public void collect(ConnectContext context, AnalyzeStatus analyzeStatus) throws Exception {} + public void collect(ConnectContext context, AnalyzeStatus analyzeStatus) throws Exception { + } }; nativeAnalyzeJob.run(statsConnectCtx, statisticExecutor); @@ -499,8 +502,8 @@ public void collect(ConnectContext context, AnalyzeStatus analyzeStatus) throws } }; - Database db = GlobalStateMgr.getCurrentState().getDb("stats"); - Table table = db.getTable("tprimary_stats"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("stats"); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tprimary_stats"); nativeAnalyzeJob = new NativeAnalyzeJob(db.getId(), table.getId(), null, null, StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.SCHEDULE, Maps.newHashMap(), @@ -557,7 +560,8 @@ public void testExternalAnalyzeJobCollect() { new MockUp() { @Mock - public void collect(ConnectContext context, AnalyzeStatus analyzeStatus) throws Exception {} + public void collect(ConnectContext context, AnalyzeStatus analyzeStatus) throws Exception { + } }; externalAnalyzeJob.run(statsConnectCtx, statisticExecutor); @@ -869,7 +873,7 @@ public List getConnectorTableStatisticsSync(Table tab @Test public void testFullStatisticsBuildCollectSQLList() { OlapTable t0p = (OlapTable) connectContext.getGlobalStateMgr() - .getDb("test").getTable("t0_stats_partition"); + .getLocalMetastore().getDb("test").getTable("t0_stats_partition"); int i = 1; for (Partition p : t0p.getAllPartitions()) { p.updateVisibleVersion(2); @@ -877,8 +881,9 @@ public void testFullStatisticsBuildCollectSQLList() { i++; } - Database database = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) database.getTable("t0_stats_partition"); + Database database = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(database.getFullName(), "t0_stats_partition"); List partitionIdList = table.getAllPartitions().stream().map(Partition::getId).collect(Collectors.toList()); @@ -919,12 +924,12 @@ public void testExternalFullStatisticsBuildCollectSQLList() { ExternalFullStatisticsCollectJob collectJob = (ExternalFullStatisticsCollectJob) StatisticsCollectJobFactory.buildExternalStatisticsCollectJob("hive0", - database, - table, null, - Lists.newArrayList("c1", "c2", "c3", "par_col"), - StatsConstants.AnalyzeType.FULL, - StatsConstants.ScheduleType.ONCE, - Maps.newHashMap()); + database, + table, null, + Lists.newArrayList("c1", "c2", "c3", "par_col"), + StatsConstants.AnalyzeType.FULL, + StatsConstants.ScheduleType.ONCE, + Maps.newHashMap()); List> collectSqlList = collectJob.buildCollectSQLList(1); Assert.assertEquals(12, collectSqlList.size()); @@ -1191,9 +1196,9 @@ public void testExternalFullStatisticsBuildCollectSQLWithException3() { @Test public void testExcludeStatistics() { OlapTable table = (OlapTable) connectContext.getGlobalStateMgr() - .getDb("test").getTable("t0_stats_partition"); + .getLocalMetastore().getDb("test").getTable("t0_stats_partition"); - Database database = connectContext.getGlobalStateMgr().getDb("test"); + Database database = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); NativeAnalyzeJob job = new NativeAnalyzeJob(StatsConstants.DEFAULT_ALL_ID, StatsConstants.DEFAULT_ALL_ID, null, null, @@ -1244,8 +1249,9 @@ public void testExcludeStatistics() { @Test public void testCount() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb("stats"); - OlapTable olapTable = (OlapTable) db.getTable("tcount"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("stats"); + OlapTable olapTable = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "tcount"); long dbid = db.getId(); SampleStatisticsCollectJob sampleStatisticsCollectJob = new SampleStatisticsCollectJob( @@ -1268,7 +1274,7 @@ public void testCount() throws Exception { @Test public void testAnalyzeBeforeUpdate() { - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); NativeAnalyzeJob job = new NativeAnalyzeJob(db.getId(), t0StatsTableId, null, null, StatsConstants.AnalyzeType.FULL, StatsConstants.ScheduleType.SCHEDULE, Maps.newHashMap(), @@ -1319,7 +1325,7 @@ public long getDataSize() { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); BasicStatsMeta execMeta1 = new BasicStatsMeta(db.getId(), t0StatsTableId, null, StatsConstants.AnalyzeType.FULL, now.minusSeconds(Config.statistic_auto_collect_large_table_interval).minusHours(1), @@ -1383,7 +1389,7 @@ public long getDataSize() { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); BasicStatsMeta execMeta1 = new BasicStatsMeta(db.getId(), t0StatsTableId, null, StatsConstants.AnalyzeType.FULL, now.minusSeconds(Config.statistic_auto_collect_large_table_interval).minusHours(1), @@ -1443,7 +1449,7 @@ public long getDataSize() { } }; - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); BasicStatsMeta execMeta = new BasicStatsMeta(db.getId(), t0StatsTableId, null, StatsConstants.AnalyzeType.FULL, now.minusSeconds(Config.statistic_auto_collect_large_table_interval).minusHours(1), diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java index 68577aed3345f..ee9d6fecae224 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsExecutorTest.java @@ -71,7 +71,7 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("t0_stats"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("t0_stats"); Partition partition = new ArrayList<>(t0.getPartitions()).get(0); partition.updateVisibleVersion(2, LocalDateTime.of(2022, 1, 1, 1, 1, 1) .atZone(Clock.systemDefaultZone().getZone()).toEpochSecond() * 1000); @@ -87,8 +87,9 @@ public void execute() { } }; - Database database = connectContext.getGlobalStateMgr().getDb("test"); - OlapTable table = (OlapTable) database.getTable("t0_stats"); + Database database = connectContext.getGlobalStateMgr().getLocalMetastore().getDb("test"); + OlapTable table = + (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getFullName(), "t0_stats"); List partitionIdList = table.getAllPartitions().stream().map(Partition::getId).collect(Collectors.toList()); diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsSQLTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsSQLTest.java index 04a296eeece4f..915f2b591e8af 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsSQLTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsSQLTest.java @@ -121,14 +121,14 @@ public static void beforeClass() throws Exception { "\"in_memory\" = \"false\"\n" + ");"); - OlapTable t0 = (OlapTable) globalStateMgr.getDb("test").getTable("stat0"); + OlapTable t0 = (OlapTable) globalStateMgr.getLocalMetastore().getDb("test").getTable("stat0"); t0StatsTableId = t0.getId(); } @Test public void testSampleStatisticsSQL() throws Exception { - Table t0 = GlobalStateMgr.getCurrentState().getDb("test").getTable("stat0"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Table t0 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("stat0"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List columnNames = Lists.newArrayList("v3", "j1", "s1"); List columnTypes = Lists.newArrayList(Type.BIGINT, Type.JSON, Type.STRING); @@ -159,8 +159,8 @@ public void testSampleStatisticsSQL() throws Exception { @Test public void testFullStatisticsSQL() throws Exception { - Table t0 = GlobalStateMgr.getCurrentState().getDb("test").getTable("stat0"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Table t0 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("stat0"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List pids = t0.getPartitions().stream().map(Partition::getId).collect(Collectors.toList()); List columnNames = Lists.newArrayList("j1", "s1"); @@ -183,8 +183,8 @@ public void testFullStatisticsSQL() throws Exception { @Test public void testFullStatisticsSQLWithStruct() throws Exception { - Table t0 = GlobalStateMgr.getCurrentState().getDb("test").getTable("struct_a"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Table t0 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("struct_a"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List pids = t0.getPartitions().stream().map(Partition::getId).collect(Collectors.toList()); List columnNames = Lists.newArrayList("b.a", "b.c", "d.c.a"); @@ -207,8 +207,8 @@ public void testFullStatisticsSQLWithStruct() throws Exception { @Test public void testHistogramStatisticsSQLWithStruct() throws Exception { - Table t0 = GlobalStateMgr.getCurrentState().getDb("test").getTable("struct_a"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Table t0 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("struct_a"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List columnNames = Lists.newArrayList("b.a", "b.c", "d.c.a"); HistogramStatisticsCollectJob histogramStatisticsCollectJob = new HistogramStatisticsCollectJob( @@ -269,8 +269,8 @@ public void testHiveHistogramStatisticsSQLWithStruct() throws Exception { @Test public void testEscapeFullSQL() throws Exception { - Table t0 = GlobalStateMgr.getCurrentState().getDb("test").getTable("escape0['abc']"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Table t0 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("escape0['abc']"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); List pids = t0.getPartitions().stream().map(Partition::getId).collect(Collectors.toList()); List columnNames = t0.getColumns().stream().map(Column::getName).collect(Collectors.toList()); @@ -293,8 +293,8 @@ public void testEscapeFullSQL() throws Exception { @Test public void testEscapeSampleSQL() throws Exception { - Table t0 = GlobalStateMgr.getCurrentState().getDb("test").getTable("escape0['abc']"); - Database db = GlobalStateMgr.getCurrentState().getDb("test"); + Table t0 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("escape0['abc']"); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); for (Column column : t0.getColumns()) { if (!column.getType().canStatistic()) { @@ -414,7 +414,7 @@ public void testCacheExternalQueryColumnStatics() { @Test public void testQuota() { - Table t0 = GlobalStateMgr.getCurrentState().getDb("test").getTable("complex_table"); + Table t0 = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test").getTable("complex_table"); assertContains(StatisticUtils.quoting(t0, "v2.a2.b2['+']"), "`v2.a2.b2['+']`"); assertContains(StatisticUtils.quoting(t0, "struct_a.c3.d3"), "`struct_a.c3.d3`"); assertContains(StatisticUtils.quoting(t0, "struct_a.c3.d3.struct_b"), "`struct_a.c3.d3`.`struct_b`"); diff --git a/fe/fe-core/src/test/java/com/starrocks/system/SystemInfoServiceTest.java b/fe/fe-core/src/test/java/com/starrocks/system/SystemInfoServiceTest.java index e1031901324b7..e66624c2e58ca 100644 --- a/fe/fe-core/src/test/java/com/starrocks/system/SystemInfoServiceTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/system/SystemInfoServiceTest.java @@ -315,7 +315,7 @@ public void testUpdateReportVersionIncreasing() throws Exception { Assert.assertEquals(10L, version.get()); } - + @Test public void testGetHostAndPort() { String ipv4 = "192.168.1.2:9050"; diff --git a/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java index 30d1f5be268c5..9bc3051327df9 100644 --- a/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/transaction/GlobalTransactionMgrTest.java @@ -116,14 +116,14 @@ public class GlobalTransactionMgrTest { private static GlobalStateMgr slaveGlobalStateMgr; private TransactionState.TxnCoordinator transactionSource = - new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, "localfe"); + new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, "localfe"); private TransactionState.TxnCoordinator transactionSourceBe = - new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.BE, "localbe"); + new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.BE, "localbe"); @Before public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { + InvocationTargetException, NoSuchMethodException, SecurityException { fakeEditLog = new FakeEditLog(); fakeGlobalStateMgr = new FakeGlobalStateMgr(); fakeTransactionIDGenerator = new FakeTransactionIDGenerator(); @@ -143,15 +143,15 @@ public void tearDown() { @Test public void testBeginTransaction() throws LabelAlreadyUsedException, AnalysisException, - RunningTxnExceedException, DuplicatedRequestException { + RunningTxnExceedException, DuplicatedRequestException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); TransactionState transactionState = - masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); + masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); assertNotNull(transactionState); assertEquals(transactionId, transactionState.getTransactionId()); assertEquals(TransactionStatus.PREPARE, transactionState.getTransactionStatus()); @@ -161,20 +161,21 @@ public void testBeginTransaction() throws LabelAlreadyUsedException, AnalysisExc @Test public void testBeginTransactionWithSameLabel() throws LabelAlreadyUsedException, AnalysisException, - RunningTxnExceedException, DuplicatedRequestException { + RunningTxnExceedException, DuplicatedRequestException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = 0; try { transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, + Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); } catch (AnalysisException | LabelAlreadyUsedException e) { e.printStackTrace(); } TransactionState transactionState = - masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); + masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId); assertNotNull(transactionState); assertEquals(transactionId, transactionState.getTransactionId()); assertEquals(TransactionStatus.PREPARE, transactionState.getTransactionStatus()); @@ -183,10 +184,11 @@ public void testBeginTransactionWithSameLabel() throws LabelAlreadyUsedException try { transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, + Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); } catch (Exception e) { // TODO: handle exception } @@ -197,36 +199,36 @@ public void testBeginTransactionWithSameLabel() throws LabelAlreadyUsedException public void testCommitTransaction1() throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); TransactionState transactionState = fakeEditLog.getTransaction(transactionId); // check status is committed assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); // check replica version - Partition testPartition = - masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + Partition testPartition = masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getVisibleVersion()); assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getNextVersion()); // check partition next version LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); } @@ -242,20 +244,20 @@ public void testCommitTransactionWithOneFailed() throws UserException { TransactionState transactionState = null; FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction with 1,2 success TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); // follower globalStateMgr replay the transaction transactionState = fakeEditLog.getTransaction(transactionId); @@ -266,19 +268,19 @@ public void testCommitTransactionWithOneFailed() throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); // commit another transaction with 1,3 success long transactionId2 = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable2, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable2, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo3); try { masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); Assert.fail(); } catch (TabletQuorumFailedException e) { transactionState = masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId2); @@ -287,14 +289,15 @@ public void testCommitTransactionWithOneFailed() throws UserException { } // check replica version Partition testPartition = - masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getVisibleVersion()); assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getNextVersion()); // check partition next version LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); } @@ -310,19 +313,20 @@ public void testCommitTransactionWithOneFailed() throws UserException { transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); transactionState = fakeEditLog.getTransaction(transactionId2); // check status is commit assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); // check replica version - testPartition = masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + testPartition = masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version assertEquals(GlobalStateMgrTestUtil.testStartVersion, testPartition.getVisibleVersion()); assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getNextVersion()); // check partition next version tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); } @@ -353,35 +357,35 @@ public void testCommitTransactionWithOneFailed() throws UserException { @Test public void testCommitRoutineLoadTransaction(@Injectable TabletCommitInfo tabletCommitInfo, @Mocked EditLog editLog) - throws UserException { + throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); TabletCommitInfo tabletCommitInfo1 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "test", 1L, 1L, "host:port", - "topic"); + "topic"); List routineLoadTaskInfoList = - Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); + Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); Map partitionIdToOffset = Maps.newHashMap(); partitionIdToOffset.put(1, 0L); KafkaTaskInfo routineLoadTaskInfo = - new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), - partitionIdToOffset, Config.routine_load_task_timeout_second); + new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), + partitionIdToOffset, Config.routine_load_task_timeout_second); Deencapsulation.setField(routineLoadTaskInfo, "txnId", 1L); routineLoadTaskInfoList.add(routineLoadTaskInfo); TransactionState transactionState = new TransactionState(1L, Lists.newArrayList(1L), 1L, "label", null, - LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), - routineLoadJob.getId(), - Config.stream_load_default_timeout_second); + LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), + routineLoadJob.getId(), + Config.stream_load_default_timeout_second); transactionState.setTransactionStatus(TransactionStatus.PREPARE); masterTransMgr.getCallbackFactory().addCallback(routineLoadJob); // Deencapsulation.setField(transactionState, "txnStateChangeListener", routineLoadJob); @@ -429,9 +433,9 @@ LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1") routineLoadManager.addRoutineLoadJob(routineLoadJob, "db"); Deencapsulation.setField(masterTransMgr.getDatabaseTransactionMgr(GlobalStateMgrTestUtil.testDbId1), - "idToRunningTransactionState", idToTransactionState); + "idToRunningTransactionState", idToTransactionState); masterTransMgr.commitTransaction(1L, 1L, transTablets, Lists.newArrayList(), - txnCommitAttachment); + txnCommitAttachment); Assert.assertEquals(Long.valueOf(101), Deencapsulation.getField(routineLoadJob, "currentTotalRows")); Assert.assertEquals(Long.valueOf(1), Deencapsulation.getField(routineLoadJob, "currentErrorRows")); @@ -448,31 +452,31 @@ public void testCommitRoutineLoadTransactionWithErrorMax(@Injectable TabletCommi FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); TabletCommitInfo tabletCommitInfo1 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = - new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); + new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); KafkaRoutineLoadJob routineLoadJob = - new KafkaRoutineLoadJob(1L, "test", 1L, 1L, "host:port", "topic"); + new KafkaRoutineLoadJob(1L, "test", 1L, 1L, "host:port", "topic"); List routineLoadTaskInfoList = - Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); + Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); Map partitionIdToOffset = Maps.newHashMap(); partitionIdToOffset.put(1, 0L); KafkaTaskInfo routineLoadTaskInfo = - new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), - partitionIdToOffset, Config.routine_load_task_timeout_second); + new KafkaTaskInfo(UUID.randomUUID(), routineLoadJob, 20000, System.currentTimeMillis(), + partitionIdToOffset, Config.routine_load_task_timeout_second); Deencapsulation.setField(routineLoadTaskInfo, "txnId", 1L); routineLoadTaskInfoList.add(routineLoadTaskInfo); TransactionState transactionState = new TransactionState(1L, Lists.newArrayList(1L), 1L, "label", null, - LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), - routineLoadJob.getId(), - Config.stream_load_default_timeout_second); + LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1"), + routineLoadJob.getId(), + Config.stream_load_default_timeout_second); transactionState.setTransactionStatus(TransactionStatus.PREPARE); masterTransMgr.getCallbackFactory().addCallback(routineLoadJob); Map idToTransactionState = Maps.newHashMap(); @@ -519,14 +523,14 @@ LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1") routineLoadManager.addRoutineLoadJob(routineLoadJob, "db"); Deencapsulation.setField(masterTransMgr.getDatabaseTransactionMgr(GlobalStateMgrTestUtil.testDbId1), - "idToRunningTransactionState", idToTransactionState); + "idToRunningTransactionState", idToTransactionState); masterTransMgr.commitTransaction(1L, 1L, transTablets, Lists.newArrayList(), - txnCommitAttachment); + txnCommitAttachment); Assert.assertEquals(Long.valueOf(0), Deencapsulation.getField(routineLoadJob, "currentTotalRows")); Assert.assertEquals(Long.valueOf(0), Deencapsulation.getField(routineLoadJob, "currentErrorRows")); Assert.assertEquals(Long.valueOf(111L), - ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); + ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); // todo(ml): change to assert queue // Assert.assertEquals(0, routineLoadManager.getNeedScheduleTasksQueue().size()); Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, routineLoadJob.getState()); @@ -534,6 +538,8 @@ LoadJobSourceType.ROUTINE_LOAD_TASK, new TxnCoordinator(TxnSourceType.BE, "be1") @Test public void testFinishTransaction() throws UserException { + FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); + long transactionId = masterTransMgr .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), GlobalStateMgrTestUtil.testTxnLable1, @@ -561,8 +567,8 @@ public void testFinishTransaction() throws UserException { transactionState = fakeEditLog.getTransaction(transactionId); assertEquals(TransactionStatus.VISIBLE, transactionState.getTransactionStatus()); // check replica version - Partition testPartition = - masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) + Partition testPartition = masterGlobalStateMgr.getLocalMetastore() + .getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, testPartition.getVisibleVersion()); @@ -587,26 +593,27 @@ public void testFinishTransaction() throws UserException { public void testFinishTransactionWithOneFailed() throws UserException { TransactionState transactionState = null; Partition testPartition = - masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + .getTablet(GlobalStateMgrTestUtil.testTabletId1); FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction with 1,2 success TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); // follower globalStateMgr replay the transaction transactionState = fakeEditLog.getTransaction(transactionId); @@ -654,19 +661,19 @@ public void testFinishTransactionWithOneFailed() throws UserException { FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); // commit another transaction with 1,3 success long transactionId2 = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable2, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable2, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo3); try { masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); Assert.fail(); } catch (TabletQuorumFailedException e) { transactionState = masterTransMgr.getTransactionState(GlobalStateMgrTestUtil.testDbId1, transactionId2); @@ -683,13 +690,14 @@ public void testFinishTransactionWithOneFailed() throws UserException { transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); transactionState = fakeEditLog.getTransaction(transactionId2); // check status is commit assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); // check replica version - testPartition = masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + testPartition = masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, testPartition.getVisibleVersion()); assertEquals(GlobalStateMgrTestUtil.testStartVersion + 3, testPartition.getNextVersion()); @@ -704,6 +712,7 @@ public void testFinishTransactionWithOneFailed() throws UserException { errorReplicaIds = Sets.newHashSet(); assertEquals(masterTransMgr.canTxnFinished(transactionState, errorReplicaIds, unfinishedBackends), false); errorReplicaIds = Sets.newHashSet(); + FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); masterTransMgr.finishTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId2, errorReplicaIds); assertEquals(TransactionStatus.VISIBLE, transactionState.getTransactionStatus()); assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, replcia1.getVersion()); @@ -734,8 +743,8 @@ public void replayWithExpiredJob() throws Exception { // 1. replay a normal finished transaction TransactionState state = - new TransactionState(dbId, new ArrayList<>(), 1, "label_a", null, LoadJobSourceType.BACKEND_STREAMING, - transactionSource, -1, -1); + new TransactionState(dbId, new ArrayList<>(), 1, "label_a", null, LoadJobSourceType.BACKEND_STREAMING, + transactionSource, -1, -1); state.setTransactionStatus(TransactionStatus.ABORTED); state.setReason("fake reason"); state.setFinishTime(System.currentTimeMillis() - 2000); @@ -744,8 +753,8 @@ public void replayWithExpiredJob() throws Exception { // 2. replay a expired transaction TransactionState state2 = - new TransactionState(dbId, new ArrayList<>(), 2, "label_b", null, LoadJobSourceType.BACKEND_STREAMING, - transactionSource, -1, -1); + new TransactionState(dbId, new ArrayList<>(), 2, "label_b", null, LoadJobSourceType.BACKEND_STREAMING, + transactionSource, -1, -1); state2.setTransactionStatus(TransactionStatus.ABORTED); state2.setReason("fake reason"); state2.setFinishTime(System.currentTimeMillis()); @@ -755,8 +764,8 @@ public void replayWithExpiredJob() throws Exception { Thread.sleep(2000); // 3. replay a valid transaction, let state expire TransactionState state3 = - new TransactionState(dbId, new ArrayList<>(), 3, "label_c", null, LoadJobSourceType.BACKEND_STREAMING, - transactionSource, -1, -1); + new TransactionState(dbId, new ArrayList<>(), 3, "label_c", null, LoadJobSourceType.BACKEND_STREAMING, + transactionSource, -1, -1); state3.setTransactionStatus(TransactionStatus.ABORTED); state3.setReason("fake reason"); state3.setFinishTime(System.currentTimeMillis()); @@ -787,30 +796,33 @@ public void replayWithExpiredJob() throws Exception { @Test public void testPrepareTransaction() throws UserException { + FakeGlobalStateMgr.setGlobalStateMgr(masterGlobalStateMgr); + long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); // commit a transaction TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId1); + GlobalStateMgrTestUtil.testBackendId1); TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId2); + GlobalStateMgrTestUtil.testBackendId2); TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1, - GlobalStateMgrTestUtil.testBackendId3); + GlobalStateMgrTestUtil.testBackendId3); List transTablets = Lists.newArrayList(); transTablets.add(tabletCommitInfo1); transTablets.add(tabletCommitInfo2); transTablets.add(tabletCommitInfo3); masterTransMgr.prepareTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId, transTablets, - Lists.newArrayList(), null); + Lists.newArrayList(), null); TransactionState transactionState = fakeEditLog.getTransaction(transactionId); assertEquals(TransactionStatus.PREPARED, transactionState.getTransactionStatus()); try { - masterTransMgr.commitPreparedTransaction(masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1), transactionId, - (long) 1000); + masterTransMgr.commitPreparedTransaction( + masterGlobalStateMgr.getLocalMetastore().getDb(GlobalStateMgrTestUtil.testDbId1), transactionId, + (long) 1000); Assert.fail("should throw publish timeout exception"); } catch (UserException e) { } @@ -825,14 +837,15 @@ public void testPrepareTransaction() throws UserException { assertEquals(TransactionStatus.VISIBLE, transactionState.getTransactionStatus()); // check replica version Partition testPartition = - masterGlobalStateMgr.getDb(GlobalStateMgrTestUtil.testDbId1).getTable(GlobalStateMgrTestUtil.testTableId1) - .getPartition(GlobalStateMgrTestUtil.testPartition1); + masterGlobalStateMgr.getLocalMetastore() + .getTable(GlobalStateMgrTestUtil.testDbId1, GlobalStateMgrTestUtil.testTableId1) + .getPartition(GlobalStateMgrTestUtil.testPartition1); // check partition version assertEquals(GlobalStateMgrTestUtil.testStartVersion + 1, testPartition.getVisibleVersion()); assertEquals(GlobalStateMgrTestUtil.testStartVersion + 2, testPartition.getNextVersion()); // check partition next version LocalTablet tablet = (LocalTablet) testPartition.getIndex(GlobalStateMgrTestUtil.testIndexId1) - .getTablet(GlobalStateMgrTestUtil.testTabletId1); + .getTablet(GlobalStateMgrTestUtil.testTabletId1); for (Replica replica : tablet.getImmutableReplicas()) { if (replica.getId() == GlobalStateMgrTestUtil.testReplicaId1) { assertEquals(GlobalStateMgrTestUtil.testStartVersion, replica.getVersion()); @@ -849,10 +862,10 @@ public void testPrepareTransaction() throws UserException { @Test public void testSaveLoadJsonFormatImage() throws Exception { long transactionId = masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSource, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); UtFrameUtils.PseudoImage pseudoImage = new UtFrameUtils.PseudoImage(); masterTransMgr.saveTransactionStateV2(pseudoImage.getImageWriter()); @@ -868,7 +881,7 @@ public void testSaveLoadJsonFormatImage() throws Exception { @Test public void testRetryCommitOnRateLimitExceededTimeout() - throws UserException { + throws UserException { Database db = new Database(10, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); DatabaseTransactionMgr dbTransactionMgr = spy(new DatabaseTransactionMgr(10L, GlobalStateMgr.getCurrentState())); @@ -878,15 +891,15 @@ public void testRetryCommitOnRateLimitExceededTimeout() doReturn(transactionState).when(globalTransactionMgr).getTransactionState(db.getId(), 1001); doReturn(dbTransactionMgr).when(globalTransactionMgr).getDatabaseTransactionMgr(db.getId()); doThrow(new CommitRateExceededException(1001, now + 60 * 1000L)) - .when(dbTransactionMgr) - .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); + .when(dbTransactionMgr) + .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); Assert.assertThrows(CommitRateExceededException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001, - Collections.emptyList(), Collections.emptyList(), 10, null)); + Collections.emptyList(), Collections.emptyList(), 10, null)); } @Test public void testPublishVersionTimeout() - throws UserException, LockTimeoutException { + throws UserException, LockTimeoutException { Database db = new Database(10, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); DatabaseTransactionMgr dbTransactionMgr = spy(new DatabaseTransactionMgr(10L, GlobalStateMgr.getCurrentState())); @@ -896,52 +909,52 @@ public void testPublishVersionTimeout() doReturn(dbTransactionMgr).when(globalTransactionMgr).getDatabaseTransactionMgr(db.getId()); doReturn(transactionState).when(globalTransactionMgr).getTransactionState(db.getId(), 1001); doReturn(new VisibleStateWaiter(new TransactionState())) - .when(dbTransactionMgr) - .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); + .when(dbTransactionMgr) + .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); Assert.assertFalse(globalTransactionMgr.commitAndPublishTransaction(db, 1001, - Collections.emptyList(), Collections.emptyList(), 2, null)); + Collections.emptyList(), Collections.emptyList(), 2, null)); } @Test public void testRetryCommitOnRateLimitExceededThrowUnexpectedException() - throws UserException { + throws UserException { Database db = new Database(10, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); DatabaseTransactionMgr dbTransactionMgr = spy(new DatabaseTransactionMgr(10L, GlobalStateMgr.getCurrentState())); doReturn(dbTransactionMgr).when(globalTransactionMgr).getDatabaseTransactionMgr(db.getId()); doThrow(NullPointerException.class) - .when(dbTransactionMgr) - .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); + .when(dbTransactionMgr) + .commitTransaction(1001L, Collections.emptyList(), Collections.emptyList(), null); Assert.assertThrows(UserException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001, - Collections.emptyList(), Collections.emptyList(), 10, null)); + Collections.emptyList(), Collections.emptyList(), 10, null)); } @Test public void testRetryCommitOnRateLimitExceededThrowLockTimeoutException() - throws UserException, LockTimeoutException { + throws UserException, LockTimeoutException { Database db = new Database(10L, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); TransactionState transactionState = new TransactionState(); doReturn(transactionState) - .when(globalTransactionMgr) - .getTransactionState(10L, 1001L); + .when(globalTransactionMgr) + .getTransactionState(10L, 1001L); doThrow(LockTimeoutException.class) - .when(globalTransactionMgr) - .retryCommitOnRateLimitExceeded(db, 1001L, Collections.emptyList(), Collections.emptyList(), null, 10L); + .when(globalTransactionMgr) + .retryCommitOnRateLimitExceeded(db, 1001L, Collections.emptyList(), Collections.emptyList(), null, 10L); Assert.assertThrows(LockTimeoutException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001L, - Collections.emptyList(), Collections.emptyList(), 10L, null)); + Collections.emptyList(), Collections.emptyList(), 10L, null)); } @Test public void testGetTransactionNumByCoordinateBe() throws LabelAlreadyUsedException, AnalysisException, - RunningTxnExceedException, DuplicatedRequestException { + RunningTxnExceedException, DuplicatedRequestException { masterTransMgr - .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), - GlobalStateMgrTestUtil.testTxnLable1, - transactionSourceBe, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); + .beginTransaction(GlobalStateMgrTestUtil.testDbId1, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1), + GlobalStateMgrTestUtil.testTxnLable1, + transactionSourceBe, + LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); long res = masterTransMgr.getTransactionNumByCoordinateBe("localbe"); assertEquals(1, res); } @@ -993,16 +1006,16 @@ public void testCommitLockTimeout() throws UserException, LockTimeoutException { Database db = new Database(10L, "db0"); GlobalTransactionMgr globalTransactionMgr = spy(new GlobalTransactionMgr(GlobalStateMgr.getCurrentState())); doThrow(LockTimeoutException.class) - .when(globalTransactionMgr) - .commitAndPublishTransaction(db, 1001L, Collections.emptyList(), Collections.emptyList(), 10L, null); + .when(globalTransactionMgr) + .commitAndPublishTransaction(db, 1001L, Collections.emptyList(), Collections.emptyList(), 10L, null); Assert.assertThrows(ErrorReportException.class, () -> globalTransactionMgr.commitAndPublishTransaction(db, 1001L, - Collections.emptyList(), Collections.emptyList(), 10L)); + Collections.emptyList(), Collections.emptyList(), 10L)); } @Test public void testCheckValidTimeoutSecond() { ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, - "Invalid timeout: '1'. Expected values should be between 2 and 3 seconds", - () -> GlobalTransactionMgr.checkValidTimeoutSecond(1, 3, 2)); + "Invalid timeout: '1'. Expected values should be between 2 and 3 seconds", + () -> GlobalTransactionMgr.checkValidTimeoutSecond(1, 3, 2)); } } diff --git a/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java b/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java index 6a2ba20a979ac..de9f838446e43 100644 --- a/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/transaction/LakePublishBatchTest.java @@ -24,6 +24,7 @@ import com.starrocks.common.Config; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; import com.starrocks.server.RunMode; import com.starrocks.utframe.StarRocksAssert; import com.starrocks.utframe.UtFrameUtils; @@ -90,8 +91,8 @@ public static void afterClass() { @Test public void testNormal() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - Table table = db.getTable(TABLE); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), TABLE); List transTablets1 = Lists.newArrayList(); List transTablets2 = Lists.newArrayList(); @@ -159,8 +160,8 @@ public void testNormal() throws Exception { @Test public void testPublishTransactionState() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - Table table = db.getTable(TABLE); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), TABLE); List transTablets = Lists.newArrayList(); for (Partition partition : table.getPartitions()) { @@ -192,8 +193,8 @@ public void testPublishTransactionState() throws Exception { @Test public void testPublishDbDroped() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - Table table = db.getTable(TABLE); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), TABLE); List transTablets = Lists.newArrayList(); for (Partition partition : table.getPartitions()) { MaterializedIndex baseIndex = partition.getBaseIndex(); @@ -225,7 +226,7 @@ public void testPublishDbDroped() throws Exception { globalTransactionMgr.commitTransaction(db.getId(), transactionId6, transTablets, Lists.newArrayList(), null); - new MockUp() { + new MockUp() { @Mock public Database getDb(long dbId) { return null; @@ -248,8 +249,8 @@ public Database getDb(long dbId) { @Test public void testPublishTableDropped() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - Table table = db.getTable(TABLE); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), TABLE); List transTablets = Lists.newArrayList(); for (Partition partition : table.getPartitions()) { MaterializedIndex baseIndex = partition.getBaseIndex(); @@ -304,8 +305,8 @@ public Table getTable(String tableName) { @Test public void testTransformBatchToSingle() throws Exception { - Database db = GlobalStateMgr.getCurrentState().getDb(DB); - Table table = db.getTable(TABLE); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(DB); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), TABLE); List transTablets1 = Lists.newArrayList(); List transTablets2 = Lists.newArrayList(); diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java b/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java index 284dc0a057a9a..561b0ca9c6a60 100644 --- a/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java @@ -172,7 +172,7 @@ public StarRocksAssert withEnableMV() { public StarRocksAssert withDatabase(String dbName) throws Exception { DropDbStmt dropDbStmt = - (DropDbStmt) UtFrameUtils.parseStmtWithNewParser("drop database if exists `" + dbName + "`;", ctx); + (DropDbStmt) UtFrameUtils.parseStmtWithNewParser("drop database if exists `" + dbName + "`;", ctx); try { GlobalStateMgr.getCurrentState().getMetadata().dropDb(dropDbStmt.getDbName(), dropDbStmt.isForceDrop()); } catch (MetaNotFoundException e) { @@ -182,7 +182,7 @@ public StarRocksAssert withDatabase(String dbName) throws Exception { } CreateDbStmt createDbStmt = - (CreateDbStmt) UtFrameUtils.parseStmtWithNewParser("create database `" + dbName + "`;", ctx); + (CreateDbStmt) UtFrameUtils.parseStmtWithNewParser("create database `" + dbName + "`;", ctx); GlobalStateMgr.getCurrentState().getMetadata().createDb(createDbStmt.getFullDbName()); return this; } @@ -190,8 +190,8 @@ public StarRocksAssert withDatabase(String dbName) throws Exception { public StarRocksAssert createDatabaseIfNotExists(String dbName) throws Exception { try { CreateDbStmt createDbStmt = - (CreateDbStmt) UtFrameUtils.parseStmtWithNewParser("create database if not exists `" - + dbName + "`;", ctx); + (CreateDbStmt) UtFrameUtils.parseStmtWithNewParser("create database if not exists `" + + dbName + "`;", ctx); GlobalStateMgr.getCurrentState().getMetadata().createDb(createDbStmt.getFullDbName()); } catch (AlreadyExistsException e) { // ignore @@ -201,15 +201,15 @@ public StarRocksAssert createDatabaseIfNotExists(String dbName) throws Exception public StarRocksAssert withRole(String roleName) throws Exception { CreateRoleStmt createRoleStmt = - (CreateRoleStmt) UtFrameUtils.parseStmtWithNewParser("create role " + roleName + ";", ctx); + (CreateRoleStmt) UtFrameUtils.parseStmtWithNewParser("create role " + roleName + ";", ctx); GlobalStateMgr.getCurrentState().getAuthorizationMgr().createRole(createRoleStmt); return this; } public StarRocksAssert withUser(String user) throws Exception { CreateUserStmt createUserStmt = - (CreateUserStmt) UtFrameUtils.parseStmtWithNewParser( - "create user " + user + " identified by '';", ctx); + (CreateUserStmt) UtFrameUtils.parseStmtWithNewParser( + "create user " + user + " identified by '';", ctx); GlobalStateMgr.getCurrentState().getAuthenticationMgr().createUser(createUserStmt); return this; } @@ -221,7 +221,7 @@ public StarRocksAssert withDatabaseWithoutAnalyze(String dbName) throws Exceptio } public boolean databaseExist(String dbName) { - Database db = GlobalStateMgr.getCurrentState().getDb("" + dbName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("" + dbName); return db != null; } @@ -326,7 +326,7 @@ public static void executeWithRetry(RetryableOperation operation, String operati } retryTime++; System.out.println(operationMsg + " failed with " + retryTime + " time retry, msg: " + - e.getMessage() + ", " + Arrays.toString(e.getStackTrace())); + e.getMessage() + ", " + Arrays.toString(e.getStackTrace())); Thread.sleep(500); } } @@ -337,7 +337,7 @@ public static void utCreateTableWithRetry(CreateTableStmt createTableStmt, Conne CreateTableStmt createTableStmtCopied = createTableStmt; if (retryTime > 0) { createTableStmtCopied = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser( - createTableStmt.getOrigStmt().originStmt, ctx); + createTableStmt.getOrigStmt().originStmt, ctx); } GlobalStateMgr.getCurrentState().getLocalMetastore().createTable(createTableStmtCopied); }, "Create Table", 3); @@ -348,7 +348,7 @@ public static void utDropTableWithRetry(DropTableStmt dropTableStmt, ConnectCont DropTableStmt dropTableStmtCopied = dropTableStmt; if (retryTime > 0) { dropTableStmtCopied = (DropTableStmt) UtFrameUtils.parseStmtWithNewParser( - dropTableStmt.getOrigStmt().originStmt, ctx); + dropTableStmt.getOrigStmt().originStmt, ctx); } GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmtCopied); }, "Drop Table", 3); @@ -369,7 +369,7 @@ public StarRocksAssert withTable(MTable mTable) throws Exception { public StarRocksAssert withTemporaryTable(String sql) throws Exception { CreateTemporaryTableStmt createTableStmt = - (CreateTemporaryTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (CreateTemporaryTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); createTableStmt.setSessionId(ctx.getSessionId()); utCreateTableWithRetry(createTableStmt, ctx); return this; @@ -378,7 +378,7 @@ public StarRocksAssert withTemporaryTable(String sql) throws Exception { public StarRocksAssert withTemporaryTableLike(String dstTable, String srcTable) throws Exception { String sql = "create temporary table " + dstTable + " like " + srcTable; CreateTemporaryTableLikeStmt createTemporaryTableLikeStmt = - (CreateTemporaryTableLikeStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (CreateTemporaryTableLikeStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); createTemporaryTableLikeStmt.setSessionId(ctx.getSessionId()); utCreateTableWithRetry(createTemporaryTableLikeStmt.getCreateTableStmt(), ctx); return this; @@ -390,7 +390,7 @@ public StarRocksAssert withTable(MTable mTable, } public StarRocksAssert withMTable(String mTable, - ExceptionRunnable action) { + ExceptionRunnable action) { return withMTableNames(null, ImmutableList.of(mTable), action); } @@ -401,9 +401,10 @@ public StarRocksAssert withMTables(List mTables, /** * Create tables by using MTables and do insert datas and later actions. + * * @param cluster pseudo cluster * @param mTables mtables to be used for creating tables - * @param action action after creating base tables + * @param action action after creating base tables * @return return this */ public StarRocksAssert withMTables(PseudoCluster cluster, @@ -420,7 +421,7 @@ public StarRocksAssert withMTables(PseudoCluster cluster, String sql = mTable.getCreateTableSql(); System.out.println(sql); CreateTableStmt createTableStmt = - (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); utCreateTableWithRetry(createTableStmt, ctx); names.add(Pair.create(dbName, mTable.getTableName())); @@ -455,17 +456,18 @@ public StarRocksAssert withMTables(PseudoCluster cluster, /** * With input base table's create table sqls and actions. - * @param sqls the base table's create table sqls + * + * @param sqls the base table's create table sqls * @param action action after creating base tables * @return return this */ public StarRocksAssert withTables(List sqls, - ExceptionRunnable action) { + ExceptionRunnable action) { Set tables = Sets.newHashSet(); try { for (String sql : sqls) { CreateTableStmt createTableStmt = - (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); utCreateTableWithRetry(createTableStmt, ctx); tables.add(createTableStmt.getTableName()); } @@ -474,7 +476,7 @@ public StarRocksAssert withTables(List sqls, } } catch (Exception e) { e.printStackTrace(); - Assert.fail("Do action failed:" + e.getMessage()); + Assert.fail("Do action failed:" + e.getMessage()); } finally { if (action != null) { for (String table : tables) { @@ -505,7 +507,7 @@ public StarRocksAssert withMTableNames(PseudoCluster cluster, MTable mTable = MSchema.getTable(table); String sql = mTable.getCreateTableSql(); CreateTableStmt createTableStmt = - (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); + (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, ctx); utCreateTableWithRetry(createTableStmt, ctx); if (cluster != null) { @@ -569,12 +571,16 @@ public StarRocksAssert useTable(String table) { } public Table getTable(String dbName, String tableName) { - return ctx.getGlobalStateMgr().mayGetDb(dbName).map(db -> db.getTable(tableName)).orElse(null); + return ctx.getGlobalStateMgr().getLocalMetastore().mayGetDb(dbName) + .map(db -> GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName)) + .orElse(null); } public MaterializedView getMv(String dbName, String tableName) { - return (MaterializedView) ctx.getGlobalStateMgr().mayGetDb(dbName).map(db -> db.getTable(tableName)) - .orElse(null); + return (MaterializedView) ctx.getGlobalStateMgr().getLocalMetastore() + .mayGetDb(dbName) + .map(db -> GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName)) + .orElse(null); } public StarRocksAssert withSingleReplicaTable(String sql) throws Exception { @@ -609,9 +615,9 @@ public StarRocksAssert withAsyncMvAndRefresh(String sql) throws Exception { } public void withAsyncMv( - CreateMaterializedViewStatement createMaterializedViewStatement, - boolean isOnlySingleReplica, - boolean isRefresh) throws Exception { + CreateMaterializedViewStatement createMaterializedViewStatement, + boolean isOnlySingleReplica, + boolean isRefresh) throws Exception { if (isOnlySingleReplica) { createMaterializedViewStatement.getProperties().put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "1"); } @@ -623,8 +629,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { if (stmt instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) stmt; TableName tableName = insertStmt.getTableName(); - Database testDb = GlobalStateMgr.getCurrentState().getDb(stmt.getTableName().getDb()); - OlapTable tbl = ((OlapTable) testDb.getTable(tableName.getTbl())); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(stmt.getTableName().getDb()); + OlapTable tbl = ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), tableName.getTbl())); for (Partition partition : tbl.getPartitions()) { if (insertStmt.getTargetPartitionIds().contains(partition.getId())) { long version = partition.getVisibleVersion() + 1; @@ -643,8 +650,8 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { } }; String refreshSql = String.format("refresh materialized view `%s`.`%s` with sync mode;", - createMaterializedViewStatement.getTableName().getDb(), - createMaterializedViewStatement.getTableName().getTbl()); + createMaterializedViewStatement.getTableName().getDb(), + createMaterializedViewStatement.getTableName().getTbl()); ctx.executeSql(refreshSql); } } @@ -671,21 +678,21 @@ public StarRocksAssert withView(String sql, ExceptionRunnable action) throws Exc public StarRocksAssert dropView(String viewName) throws Exception { DropTableStmt dropViewStmt = - (DropTableStmt) UtFrameUtils.parseStmtWithNewParser("drop view " + viewName + ";", ctx); + (DropTableStmt) UtFrameUtils.parseStmtWithNewParser("drop view " + viewName + ";", ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropViewStmt); return this; } public StarRocksAssert dropCatalog(String catalogName) throws Exception { DropCatalogStmt dropCatalogStmt = - (DropCatalogStmt) UtFrameUtils.parseStmtWithNewParser("drop catalog " + catalogName + ";", ctx); + (DropCatalogStmt) UtFrameUtils.parseStmtWithNewParser("drop catalog " + catalogName + ";", ctx); GlobalStateMgr.getCurrentState().getCatalogMgr().dropCatalog(dropCatalogStmt); return this; } public StarRocksAssert dropDatabase(String dbName) throws Exception { DropDbStmt dropDbStmt = - (DropDbStmt) UtFrameUtils.parseStmtWithNewParser("drop database " + dbName + ";", ctx); + (DropDbStmt) UtFrameUtils.parseStmtWithNewParser("drop database " + dbName + ";", ctx); GlobalStateMgr.getCurrentState().getMetadata().dropDb(dropDbStmt.getDbName(), dropDbStmt.isForceDrop()); return this; } @@ -721,22 +728,22 @@ public StarRocksAssert dropTables(List tableNames) throws Exception { public StarRocksAssert dropTable(String tableName) throws Exception { DropTableStmt dropTableStmt = - (DropTableStmt) UtFrameUtils.parseStmtWithNewParser("drop table " + tableName + ";", ctx); + (DropTableStmt) UtFrameUtils.parseStmtWithNewParser("drop table " + tableName + ";", ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropTable(dropTableStmt); return this; } public StarRocksAssert dropTemporaryTable(String tableName, boolean ifExists) throws Exception { DropTemporaryTableStmt dropTemporaryTableStmt = (DropTemporaryTableStmt) - UtFrameUtils.parseStmtWithNewParser("drop temporary table " + (ifExists ? "if exists " : "") - + tableName + ";", ctx); + UtFrameUtils.parseStmtWithNewParser("drop temporary table " + (ifExists ? "if exists " : "") + + tableName + ";", ctx); GlobalStateMgr.getCurrentState().getMetadataMgr().dropTemporaryTable(dropTemporaryTableStmt); return this; } public StarRocksAssert dropMaterializedView(String materializedViewName) throws Exception { DropMaterializedViewStmt dropMaterializedViewStmt = (DropMaterializedViewStmt) UtFrameUtils. - parseStmtWithNewParser("drop materialized view if exists " + materializedViewName + ";", ctx); + parseStmtWithNewParser("drop materialized view if exists " + materializedViewName + ";", ctx); GlobalStateMgr.getCurrentState().getLocalMetastore().dropMaterializedView(dropMaterializedViewStmt); return this; } @@ -840,8 +847,8 @@ public StarRocksAssert withMaterializedView(String sql, ExceptionRunnable action } public void assertMVWithoutComplexExpression(String dbName, String tableName) { - Database db = GlobalStateMgr.getCurrentState().getDb(dbName); - Table table = db.getTable(tableName); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName); if (!(table instanceof OlapTable)) { return; } @@ -899,8 +906,8 @@ public StarRocksAssert refreshMvPartition(String sql) throws Exception { RefreshMaterializedViewStatement refreshMaterializedViewStatement = (RefreshMaterializedViewStatement) stmt; TableName mvName = refreshMaterializedViewStatement.getMvName(); - Database db = GlobalStateMgr.getCurrentState().getDb(mvName.getDb()); - Table table = db.getTable(mvName.getTbl()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvName.getDb()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName.getTbl()); Assert.assertNotNull(table); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; @@ -922,6 +929,7 @@ public StarRocksAssert refreshMvPartition(String sql) throws Exception { /** * Wait the input mv refresh task finished. + * * @param mvId: mv id * @return true if the mv refresh task finished, otherwise false. */ @@ -944,7 +952,8 @@ public boolean waitRefreshFinished(long mvId) { /** * Refresh materialized view asynchronously. - * @param ctx connnect context + * + * @param ctx connnect context * @param mvName mv's name */ public StarRocksAssert refreshMV(ConnectContext ctx, String mvName) throws Exception { @@ -952,8 +961,8 @@ public StarRocksAssert refreshMV(ConnectContext ctx, String mvName) throws Excep StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(sql, ctx); RefreshMaterializedViewStatement refreshMaterializedViewStatement = (RefreshMaterializedViewStatement) stmt; TableName tableName = refreshMaterializedViewStatement.getMvName(); - Database db = GlobalStateMgr.getCurrentState().getDb(tableName.getDb()); - Table table = db.getTable(tableName.getTbl()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(tableName.getDb()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName.getTbl()); Assert.assertNotNull(table); Assert.assertTrue(table instanceof MaterializedView); ctx.executeSql(sql); @@ -964,8 +973,8 @@ public StarRocksAssert refreshMV(String sql) throws Exception { StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(sql, ctx); RefreshMaterializedViewStatement refreshMaterializedViewStatement = (RefreshMaterializedViewStatement) stmt; TableName mvName = refreshMaterializedViewStatement.getMvName(); - Database db = GlobalStateMgr.getCurrentState().getDb(mvName.getDb()); - Table table = db.getTable(mvName.getTbl()); + Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(mvName.getDb()); + Table table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), mvName.getTbl()); Assert.assertNotNull(table); Assert.assertTrue(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; @@ -988,7 +997,7 @@ private void waitingTaskFinish(TaskRun taskRun) { } public void updateTablePartitionVersion(String dbName, String tableName, long version) { - OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getDb(dbName).getTable(tableName); + OlapTable table = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName).getTable(tableName); for (PhysicalPartition partition : table.getPhysicalPartitions()) { partition.setVisibleVersion(version, System.currentTimeMillis()); MaterializedIndex baseIndex = partition.getBaseIndex(); @@ -1049,8 +1058,8 @@ public String executeShowResourceUsageSql(String sql) throws DdlException, Analy ShowResultSet res = ShowExecutor.execute((ShowStmt) stmt, ctx); String header = res.getMetaData().getColumns().stream().map(Column::getName).collect(Collectors.joining("|")); String body = res.getResultRows().stream() - .map(row -> String.join("|", row)) - .collect(Collectors.joining("\n")); + .map(row -> String.join("|", row)) + .collect(Collectors.joining("\n")); return header + "\n" + body; } @@ -1076,8 +1085,9 @@ private void checkAlterJob() throws InterruptedException { if (alterJobV2.getJobState().isFinalState()) { continue; } - Database database = GlobalStateMgr.getCurrentState().getDb(alterJobV2.getDbId()); - Table table = database.getTable(alterJobV2.getTableId()); + Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(alterJobV2.getDbId()); + Table table = + GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(database.getId(), alterJobV2.getTableId()); Preconditions.checkState(table instanceof OlapTable); OlapTable olapTable = (OlapTable) table; int retry = 0; @@ -1168,7 +1178,7 @@ public void analysisError(String... keywords) { explainQuery(); } catch (AnalysisException | StarRocksPlannerException analysisException) { Assert.assertTrue(analysisException.getMessage(), - Stream.of(keywords).allMatch(analysisException.getMessage()::contains)); + Stream.of(keywords).allMatch(analysisException.getMessage()::contains)); return; } catch (Exception ex) { Assert.fail(); diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java index c407f4d7d04f1..315b3ad1e0a36 100644 --- a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java @@ -176,26 +176,26 @@ public class UtFrameUtils { private static final AtomicBoolean CREATED_MIN_CLUSTER = new AtomicBoolean(false); public static final String CREATE_STATISTICS_TABLE_STMT = "CREATE TABLE `table_statistic_v1` (\n" + - " `table_id` bigint(20) NOT NULL COMMENT \"\",\n" + - " `column_name` varchar(65530) NOT NULL COMMENT \"\",\n" + - " `db_id` bigint(20) NOT NULL COMMENT \"\",\n" + - " `table_name` varchar(65530) NOT NULL COMMENT \"\",\n" + - " `db_name` varchar(65530) NOT NULL COMMENT \"\",\n" + - " `row_count` bigint(20) NOT NULL COMMENT \"\",\n" + - " `data_size` bigint(20) NOT NULL COMMENT \"\",\n" + - " `distinct_count` bigint(20) NOT NULL COMMENT \"\",\n" + - " `null_count` bigint(20) NOT NULL COMMENT \"\",\n" + - " `max` varchar(65530) NOT NULL COMMENT \"\",\n" + - " `min` varchar(65530) NOT NULL COMMENT \"\",\n" + - " `update_time` datetime NOT NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "UNIQUE KEY(`table_id`, `column_name`, `db_id`)\n" + - "COMMENT \"OLAP\"\n" + - "DISTRIBUTED BY HASH(`table_id`, `column_name`, `db_id`) BUCKETS 10\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\"\n" + - ");"; + " `table_id` bigint(20) NOT NULL COMMENT \"\",\n" + + " `column_name` varchar(65530) NOT NULL COMMENT \"\",\n" + + " `db_id` bigint(20) NOT NULL COMMENT \"\",\n" + + " `table_name` varchar(65530) NOT NULL COMMENT \"\",\n" + + " `db_name` varchar(65530) NOT NULL COMMENT \"\",\n" + + " `row_count` bigint(20) NOT NULL COMMENT \"\",\n" + + " `data_size` bigint(20) NOT NULL COMMENT \"\",\n" + + " `distinct_count` bigint(20) NOT NULL COMMENT \"\",\n" + + " `null_count` bigint(20) NOT NULL COMMENT \"\",\n" + + " `max` varchar(65530) NOT NULL COMMENT \"\",\n" + + " `min` varchar(65530) NOT NULL COMMENT \"\",\n" + + " `update_time` datetime NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY(`table_id`, `column_name`, `db_id`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`table_id`, `column_name`, `db_id`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\"\n" + + ");"; // Help to create a mocked ConnectContext. public static ConnectContext createDefaultCtx() { @@ -212,11 +212,11 @@ public static ConnectContext createDefaultCtx() { // Parse an origin stmt . Return a StatementBase instance. public static StatementBase parseStmtWithNewParser(String originStmt, ConnectContext ctx) - throws Exception { + throws Exception { StatementBase statementBase; try { statementBase = - com.starrocks.sql.parser.SqlParser.parse(originStmt, ctx.getSessionVariable().getSqlMode()).get(0); + com.starrocks.sql.parser.SqlParser.parse(originStmt, ctx.getSessionVariable().getSqlMode()).get(0); com.starrocks.sql.analyzer.Analyzer.analyze(statementBase, ctx); } catch (ParsingException | SemanticException e) { System.err.println("parse failed: " + e.getMessage()); @@ -231,11 +231,11 @@ public static StatementBase parseStmtWithNewParser(String originStmt, ConnectCon } public static StatementBase parseStmtWithNewParserNotIncludeAnalyzer(String originStmt, ConnectContext ctx) - throws Exception { + throws Exception { StatementBase statementBase; try { statementBase = - com.starrocks.sql.parser.SqlParser.parse(originStmt, ctx.getSessionVariable().getSqlMode()).get(0); + com.starrocks.sql.parser.SqlParser.parse(originStmt, ctx.getSessionVariable().getSqlMode()).get(0); } catch (ParsingException e) { if (e.getMessage() == null) { throw e; @@ -302,7 +302,7 @@ public static synchronized void createMinStarRocksCluster(boolean startBDB, RunM // sleep to wait first heartbeat int retry = 0; while (GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackend(10001).getBePort() == -1 && - retry++ < 600) { + retry++ < 600) { Thread.sleep(100); } CREATED_MIN_CLUSTER.set(true); @@ -362,7 +362,7 @@ private static Backend addMockBackend(MockedBackend backend, int backendId) { int starletPort = backend.getStarletPort(); String workerAddress = backend.getHost() + ":" + starletPort; GlobalStateMgr.getCurrentState().getStarOSAgent().addWorker(be.getId(), workerAddress, - StarOSAgent.DEFAULT_WORKER_GROUP_ID); + StarOSAgent.DEFAULT_WORKER_GROUP_ID); } return be; } @@ -380,7 +380,7 @@ private static ComputeNode addMockComputeNode(MockedBackend backend, int backend int starletPort = backend.getStarletPort(); String workerAddress = backend.getHost() + ":" + starletPort; GlobalStateMgr.getCurrentState().getStarOSAgent().addWorker(cn.getId(), workerAddress, - StarOSAgent.DEFAULT_WORKER_GROUP_ID); + StarOSAgent.DEFAULT_WORKER_GROUP_ID); } return cn; } @@ -452,7 +452,7 @@ public static void validatePlanConnectedness(ExecPlan plan) { } Assert.assertEquals("Some fragments do not belong to the fragment tree", - plan.getFragments().size(), visitedFragments.size()); + plan.getFragments().size(), visitedFragments.size()); } /* @@ -460,11 +460,11 @@ public static void validatePlanConnectedness(ExecPlan plan) { */ public static Pair planMVMaintenance(ConnectContext connectContext, String sql) - throws DdlException, CloneNotSupportedException { + throws DdlException, CloneNotSupportedException { connectContext.setDumpInfo(new QueryDumpInfo(connectContext)); List statements = - com.starrocks.sql.parser.SqlParser.parse(sql, connectContext.getSessionVariable().getSqlMode()); + com.starrocks.sql.parser.SqlParser.parse(sql, connectContext.getSessionVariable().getSqlMode()); connectContext.getDumpInfo().setOriginStmt(sql); SessionVariable oldSessionVariable = connectContext.getSessionVariable(); StatementBase statementBase = statements.get(0); @@ -532,7 +532,7 @@ private static R buildPlan(ConnectContext connectContext, String originStmt, public static String getFragmentPlan(ConnectContext connectContext, String sql, String traceModule) { try { Pair> result = - UtFrameUtils.getFragmentPlanWithTrace(connectContext, sql, traceModule); + UtFrameUtils.getFragmentPlanWithTrace(connectContext, sql, traceModule); Pair execPlanWithQuery = result.second; String traceLog = execPlanWithQuery.second; if (!Strings.isNullOrEmpty(traceLog)) { @@ -546,7 +546,7 @@ public static String getFragmentPlan(ConnectContext connectContext, String sql, } public static Pair> getFragmentPlanWithTrace( - ConnectContext connectContext, String sql, String module) throws Exception { + ConnectContext connectContext, String sql, String module) throws Exception { if (Strings.isNullOrEmpty(module)) { Pair planPair = UtFrameUtils.getPlanAndFragment(connectContext, sql); Pair planAndTrace = Pair.create(planPair.second, ""); @@ -573,34 +573,34 @@ public static Pair> getFragmentPlanWithTrace( } public static Pair getPlanAndFragment(ConnectContext connectContext, String originStmt) - throws Exception { + throws Exception { return buildPlan(connectContext, originStmt, - (context, statementBase, execPlan) -> new Pair<>(printPhysicalPlan(execPlan.getPhysicalPlan()), - execPlan)); + (context, statementBase, execPlan) -> new Pair<>(printPhysicalPlan(execPlan.getPhysicalPlan()), + execPlan)); } public static DefaultCoordinator startScheduling(ConnectContext connectContext, String originStmt) throws Exception { return buildPlan(connectContext, originStmt, - (context, statementBase, execPlan) -> { - DefaultCoordinator scheduler = createScheduler(context, statementBase, execPlan); + (context, statementBase, execPlan) -> { + DefaultCoordinator scheduler = createScheduler(context, statementBase, execPlan); - scheduler.startScheduling(); + scheduler.startScheduling(); - return scheduler; - }); + return scheduler; + }); } public static Pair getPlanAndStartScheduling(ConnectContext connectContext, String originStmt) - throws Exception { + throws Exception { return buildPlan(connectContext, originStmt, - (context, statementBase, execPlan) -> { - DefaultCoordinator scheduler = createScheduler(context, statementBase, execPlan); + (context, statementBase, execPlan) -> { + DefaultCoordinator scheduler = createScheduler(context, statementBase, execPlan); - scheduler.startSchedulingWithoutDeploy(); - String plan = scheduler.getSchedulerExplain(); + scheduler.startSchedulingWithoutDeploy(); + String plan = scheduler.getSchedulerExplain(); - return Pair.create(plan, scheduler); - }); + return Pair.create(plan, scheduler); + }); } public static DefaultCoordinator getScheduler(ConnectContext connectContext, String originStmt) throws Exception { @@ -613,14 +613,14 @@ private static DefaultCoordinator createScheduler(ConnectContext context, Statem if (statementBase instanceof DmlStmt) { if (statementBase instanceof InsertStmt) { scheduler = new DefaultCoordinator.Factory().createInsertScheduler(context, - execPlan.getFragments(), execPlan.getScanNodes(), - execPlan.getDescTbl().toThrift()); + execPlan.getFragments(), execPlan.getScanNodes(), + execPlan.getDescTbl().toThrift()); } else { throw new RuntimeException("can only handle insert DML"); } } else { scheduler = new DefaultCoordinator.Factory().createQueryScheduler(context, - execPlan.getFragments(), execPlan.getScanNodes(), execPlan.getDescTbl().toThrift()); + execPlan.getFragments(), execPlan.getScanNodes(), execPlan.getDescTbl().toThrift()); } return scheduler; @@ -635,12 +635,12 @@ public static boolean isPrintPlanTableNames() { } private static void testView(ConnectContext connectContext, String originStmt, StatementBase statementBase) - throws Exception { + throws Exception { if (!FeConstants.unitTestView) { return; } if (statementBase instanceof QueryStatement && !connectContext.getDatabase().isEmpty() && - !statementBase.isExplain()) { + !statementBase.isExplain()) { String viewName = "view" + INDEX.getAndIncrement(); String createView = "create view " + viewName + " as " + originStmt; CreateViewStmt createTableStmt; @@ -648,12 +648,12 @@ private static void testView(ConnectContext connectContext, String originStmt, S createTableStmt = (CreateViewStmt) UtFrameUtils.parseStmtWithNewParser(createView, connectContext); try { StatementBase viewStatement = - SqlParser.parse(createTableStmt.getInlineViewDef(), - connectContext.getSessionVariable().getSqlMode()).get(0); + SqlParser.parse(createTableStmt.getInlineViewDef(), + connectContext.getSessionVariable().getSqlMode()).get(0); com.starrocks.sql.analyzer.Analyzer.analyze(viewStatement, connectContext); } catch (Exception e) { System.out.println("invalid view def: " + createTableStmt.getInlineViewDef() - + "\nError msg:" + e.getMessage()); + + "\nError msg:" + e.getMessage()); throw e; } } catch (SemanticException | AnalysisException e) { @@ -675,8 +675,8 @@ public static String explainLogicalPlan(ConnectContext ctx, String sql) throws E public static String getStmtDigest(ConnectContext connectContext, String originStmt) throws Exception { StatementBase statementBase = - com.starrocks.sql.parser.SqlParser.parse(originStmt, connectContext.getSessionVariable()) - .get(0); + com.starrocks.sql.parser.SqlParser.parse(originStmt, connectContext.getSessionVariable()) + .get(0); Preconditions.checkState(statementBase instanceof QueryStatement); return ConnectProcessor.computeStatementDigest(statementBase); } @@ -686,7 +686,7 @@ private static String initMockEnv(ConnectContext connectContext, QueryDumpInfo r StarRocksAssert starRocksAssert = new StarRocksAssert(connectContext); if (!starRocksAssert.databaseExist("_statistics_")) { starRocksAssert.withDatabaseWithoutAnalyze(StatsConstants.STATISTICS_DB_NAME) - .useDatabase(StatsConstants.STATISTICS_DB_NAME); + .useDatabase(StatsConstants.STATISTICS_DB_NAME); starRocksAssert.withTable(CREATE_STATISTICS_TABLE_STMT); } // prepare dump mock environment @@ -702,11 +702,11 @@ private static String initMockEnv(ConnectContext connectContext, QueryDumpInfo r // mock replay external table info if (!replayDumpInfo.getHmsTableMap().isEmpty()) { ReplayMetadataMgr replayMetadataMgr = new ReplayMetadataMgr( - GlobalStateMgr.getCurrentState().getLocalMetastore(), - GlobalStateMgr.getCurrentState().getConnectorMgr(), - GlobalStateMgr.getCurrentState().getResourceMgr(), - replayDumpInfo.getHmsTableMap(), - replayDumpInfo.getTableStatisticsMap()); + GlobalStateMgr.getCurrentState().getLocalMetastore(), + GlobalStateMgr.getCurrentState().getConnectorMgr(), + GlobalStateMgr.getCurrentState().getResourceMgr(), + replayDumpInfo.getHmsTableMap(), + replayDumpInfo.getTableStatisticsMap()); GlobalStateMgr.getCurrentState().setMetadataMgr(replayMetadataMgr); } @@ -727,7 +727,7 @@ private static String initMockEnv(ConnectContext connectContext, QueryDumpInfo r } Set dbSet = replayDumpInfo.getCreateTableStmtMap().keySet().stream().map(key -> key.split("\\.")[0]) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); dbSet.forEach(db -> { if (starRocksAssert.databaseExist(db)) { try { @@ -765,7 +765,7 @@ private static String initMockEnv(ConnectContext connectContext, QueryDumpInfo r String dropView = String.format("drop view if exists `%s`.`%s`;", dbName, viewName); connectContext.executeSql(dropView); String createView = String.format("create view `%s`.`%s` as %s", - dbName, viewName, replayDumpInfo.getCreateViewStmtMap().get(normalizedViewName)); + dbName, viewName, replayDumpInfo.getCreateViewStmtMap().get(normalizedViewName)); starRocksAssert.withView(createView); } // create materialized views @@ -794,8 +794,8 @@ private static String initMockEnv(ConnectContext connectContext, QueryDumpInfo r // mock table row count for (Map.Entry> entry : replayDumpInfo.getPartitionRowCountMap().entrySet()) { String dbName = entry.getKey().split("\\.")[0]; - OlapTable replayTable = (OlapTable) GlobalStateMgr.getCurrentState().getDb("" + dbName) - .getTable(entry.getKey().split("\\.")[1]); + OlapTable replayTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("" + dbName) + .getTable(entry.getKey().split("\\.")[1]); for (Map.Entry partitionEntry : entry.getValue().entrySet()) { setPartitionStatistics(replayTable, partitionEntry.getKey(), partitionEntry.getValue()); @@ -803,14 +803,14 @@ private static String initMockEnv(ConnectContext connectContext, QueryDumpInfo r } // mock table column statistics for (Map.Entry> entry : replayDumpInfo.getTableStatisticsMap() - .entrySet()) { + .entrySet()) { String dbName = entry.getKey().split("\\.")[0]; - Table replayTable = GlobalStateMgr.getCurrentState().getDb("" + dbName) - .getTable(entry.getKey().split("\\.")[1]); + Table replayTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("" + dbName) + .getTable(entry.getKey().split("\\.")[1]); for (Map.Entry columnStatisticEntry : entry.getValue().entrySet()) { GlobalStateMgr.getCurrentState().getStatisticStorage() - .addColumnStatistic(replayTable, columnStatisticEntry.getKey(), - columnStatisticEntry.getValue()); + .addColumnStatistic(replayTable, columnStatisticEntry.getKey(), + columnStatisticEntry.getValue()); } } return replaySql; @@ -834,7 +834,7 @@ public static LogicalPlan getQueryLogicalPlan(ConnectContext connectContext, LogicalPlan logicalPlan; try (Timer t = Tracers.watchScope("Transformer")) { logicalPlan = new RelationTransformer(columnRefFactory, connectContext) - .transform((statement).getQueryRelation()); + .transform((statement).getQueryRelation()); } return logicalPlan; @@ -853,11 +853,11 @@ public static OptExpression getQueryOptExpression(ConnectContext connectContext, optimizer = new Optimizer(); } optimizedPlan = optimizer.optimize( - connectContext, - logicalPlan.getRoot(), - new PhysicalPropertySet(), - new ColumnRefSet(logicalPlan.getOutputColumn()), - columnRefFactory); + connectContext, + logicalPlan.getRoot(), + new PhysicalPropertySet(), + new ColumnRefSet(logicalPlan.getOutputColumn()), + columnRefFactory); } return optimizedPlan; } @@ -869,7 +869,7 @@ public static OptExpression getQueryOptExpression(ConnectContext connectContext, } private static Pair getQueryExecPlan(QueryStatement statement, ConnectContext connectContext) - throws Exception { + throws Exception { SessionVariable oldSessionVariable = connectContext.getSessionVariable(); try { if (statement.isExistQueryScopeHint()) { @@ -883,9 +883,9 @@ private static Pair getQueryExecPlan(QueryStatement statement, ExecPlan execPlan; try (Timer t = Tracers.watchScope("Builder")) { execPlan = PlanFragmentBuilder - .createPhysicalPlan(optimizedPlan, connectContext, - logicalPlan.getOutputColumn(), columnRefFactory, new ArrayList<>(), - TResultSinkType.MYSQL_PROTOCAL, true); + .createPhysicalPlan(optimizedPlan, connectContext, + logicalPlan.getOutputColumn(), columnRefFactory, new ArrayList<>(), + TResultSinkType.MYSQL_PROTOCAL, true); } return new Pair<>(LogicalPlanPrinter.print(optimizedPlan), execPlan); } finally { @@ -932,7 +932,7 @@ public static void tearDownTestDump() { } public static ExecPlan getPlanFragmentFromQueryDump(ConnectContext connectContext, QueryDumpInfo dumpInfo) - throws Exception { + throws Exception { String q = initMockEnv(connectContext, dumpInfo); try { return UtFrameUtils.getPlanAndFragment(connectContext, q).second; @@ -950,7 +950,7 @@ public static Pair getNewPlanAndFragmentFromDump(ConnectContex StatementBase statementBase; try (Timer st = Tracers.watchScope("Parse")) { statementBase = com.starrocks.sql.parser.SqlParser.parse(replaySql, - connectContext.getSessionVariable()).get(0); + connectContext.getSessionVariable()).get(0); if (statementBase instanceof QueryStatement) { replaceTableCatalogName(statementBase); } @@ -983,7 +983,7 @@ private static void replaceTableCatalogName(StatementBase statementBase) { if (tableRelation.getName().getCatalog() != null) { String catalogName = tableRelation.getName().getCatalog(); tableRelation.getName().setCatalog( - CatalogMgr.ResourceMappingCatalog.getResourceMappingCatalogName(catalogName, "hive")); + CatalogMgr.ResourceMappingCatalog.getResourceMappingCatalogName(catalogName, "hive")); } } } @@ -1088,13 +1088,13 @@ public DataInputStream getDataInputStream() throws IOException { public SRMetaBlockReader getMetaBlockReader() throws IOException { JsonReader jsonReader = new JsonReader(new InputStreamReader(new ByteArrayInputStream(buffer.getData(), - 0, buffer.getLength()))); + 0, buffer.getLength()))); return new SRMetaBlockReaderV2(jsonReader); } public JsonReader getJsonReader() throws IOException { return new JsonReader(new InputStreamReader(new ByteArrayInputStream(buffer.getData(), - 0, buffer.getLength()))); + 0, buffer.getLength()))); } } @@ -1104,10 +1104,10 @@ public JsonReader getJsonReader() throws IOException { public static class PseudoJournalReplayer { // master journal queue private static BlockingQueue masterJournalQueue = - new ArrayBlockingQueue<>(Config.metadata_journal_queue_size); + new ArrayBlockingQueue<>(Config.metadata_journal_queue_size); // follower journal queue private static BlockingQueue followerJournalQueue = - new ArrayBlockingQueue<>(Config.metadata_journal_queue_size); + new ArrayBlockingQueue<>(Config.metadata_journal_queue_size); // constantly move master journal to follower and mark succeed private static Thread fakeJournalWriter = null; @@ -1151,7 +1151,7 @@ public static synchronized Writable replayNextJournal(short expectCode) throws E } DataOutputBuffer buffer = followerJournalQueue.take().getBuffer(); DataInputStream dis = - new DataInputStream(new ByteArrayInputStream(buffer.getData(), 0, buffer.getLength())); + new DataInputStream(new ByteArrayInputStream(buffer.getData(), 0, buffer.getLength())); try { JournalEntity je = new JournalEntity(); je.readFields(dis); @@ -1172,7 +1172,7 @@ public static synchronized void replayJournalToEnd() throws InterruptedException DataOutputBuffer buffer = followerJournalQueue.take().getBuffer(); JournalEntity je = new JournalEntity(); try (DataInputStream dis = new DataInputStream( - new ByteArrayInputStream(buffer.getData(), 0, buffer.getLength()))) { + new ByteArrayInputStream(buffer.getData(), 0, buffer.getLength()))) { je.readFields(dis); GlobalStateMgr.getCurrentState().getEditLog().loadJournal(GlobalStateMgr.getCurrentState(), je); // System.out.println("replayed journal type: " + je.getOpCode()); @@ -1213,11 +1213,11 @@ public static ConnectContext initCtxForNewPrivilege(UserIdentity userIdentity) t public static boolean matchPlanWithoutId(String expect, String actual) { String trimedExpect = expect.replaceAll("\\d+:\\s*", "") - .replaceAll("\\[\\d+,", "[") - .replaceAll("", ""); + .replaceAll("\\[\\d+,", "[") + .replaceAll("", ""); String trimedActual = actual.replaceAll("\\d+:\\s*", "") - .replaceAll("\\[\\d+,", "[") - .replaceAll("", ""); + .replaceAll("\\[\\d+,", "[") + .replaceAll("", ""); boolean ret = trimedActual.contains(trimedExpect); if (!ret) { System.out.println("trimedExpect:"); @@ -1338,8 +1338,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { if (stmt instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) stmt; TableName tableName = insertStmt.getTableName(); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = ((OlapTable) testDb.getTable(tableName.getTbl())); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), tableName.getTbl())); if (tbl != null) { for (Long partitionId : insertStmt.getTargetPartitionIds()) { Partition partition = tbl.getPartition(partitionId); @@ -1349,8 +1350,9 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception { } else if (stmt instanceof DeleteStmt) { DeleteStmt delete = (DeleteStmt) stmt; TableName tableName = delete.getTableName(); - Database testDb = GlobalStateMgr.getCurrentState().getDb("test"); - OlapTable tbl = ((OlapTable) testDb.getTable(tableName.getTbl())); + Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test"); + OlapTable tbl = ((OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore() + .getTable(testDb.getFullName(), tableName.getTbl())); tbl.setHasDelete(); } } @@ -1367,7 +1369,7 @@ private static void processQueryScopeHint(StatementBase parsedStmt, ConnectConte } for (Map.Entry entry : hint.getValue().entrySet()) { VariableMgr.setSystemVariable(clonedSessionVariable, - new SystemVariable(entry.getKey(), new StringLiteral(entry.getValue())), true); + new SystemVariable(entry.getKey(), new StringLiteral(entry.getValue())), true); } } else if (hint instanceof UserVariableHint) { UserVariableHint userVariableHint = (UserVariableHint) hint;