From d2e24449e52101031adb42897de52eb536c64d77 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Mon, 23 Sep 2024 16:52:07 +0800 Subject: [PATCH] 3 --- be/src/cloud/cloud_meta_mgr.cpp | 10 ++++++++++ be/src/common/config.cpp | 3 +++ be/src/common/config.h | 3 +++ be/src/olap/tablet.cpp | 14 ++++++++++++++ cloud/src/common/config.h | 3 +++ .../pipeline/cloud_p0/conf/be_custom.conf | 1 + .../pipeline/cloud_p1/conf/be_custom.conf | 1 + regression-test/pipeline/p0/conf/be.conf | 1 + regression-test/pipeline/p1/conf/be.conf | 1 + 9 files changed, 37 insertions(+) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 36217ddb61de7d5..9f96e2f1be02749 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -748,6 +748,16 @@ Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, Status ret_st; TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::commit_rowset", ret_st); } + if (config::enable_table_size_correctness_check) { + if (rs_meta.data_disk_size() + rs_meta.index_disk_size() != rs_meta.total_disk_size()) { + LOG(FATAL) << "[Cloud table size check failed]:" + << " tablet id: " << rs_meta.tablet_id() + << ", rowset id:" << rs_meta.rowset_id() + << ", rowset data disk size:" << rs_meta.data_disk_size() + << ", rowset index disk size:" << rs_meta.index_disk_size() + << ", rowset total disk size:" << rs_meta.total_disk_size() << "."; + } + } CreateRowsetRequest req; CreateRowsetResponse resp; req.set_cloud_unique_id(config::cloud_unique_id); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 60766d8c7557eba..4e549e180609c84 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1340,6 +1340,9 @@ DEFINE_mBool(enable_pipeline_task_leakage_detect, "false"); DEFINE_Int32(query_cache_size, "512"); +// Enable validation to check the correctness of table size. +DEFINE_Bool(enable_table_size_correctness_check, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 3e52bd6cc8db472..ac9690439476b07 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1428,6 +1428,9 @@ DECLARE_mBool(enable_pipeline_task_leakage_detect); // MB DECLARE_Int32(query_cache_size); +// Enable validation to check the correctness of table size. +DECLARE_Bool(enable_table_size_correctness_check); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 938716dfedbc139..2b31e4905628709 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -330,6 +330,20 @@ Status Tablet::init() { // should save tablet meta to remote meta store // if it's a primary replica void Tablet::save_meta() { + if (config::enable_table_size_correctness_check) { + const std::vector& all_rs_metas = _tablet_meta->all_rs_metas(); + for (const auto& rs_meta : all_rs_metas) { + if (rs_meta->data_disk_size() + rs_meta->index_disk_size() != + rs_meta->total_disk_size()) { + LOG(FATAL) << "[Local table size check failed]:" + << " tablet id: " << get_tablet_info().tablet_id + << ", rowset id:" << rs_meta->rowset_id() + << ", rowset data disk size:" << rs_meta->data_disk_size() + << ", rowset index disk size:" << rs_meta->index_disk_size() + << ", rowset total disk size:" << rs_meta->total_disk_size() << "."; + } + } + } auto res = _tablet_meta->save_meta(_data_dir); CHECK_EQ(res, Status::OK()) << "fail to save tablet_meta. res=" << res << ", root=" << _data_dir->path(); diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 2c0eb16268a688a..b93a87b51a692d4 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -218,4 +218,7 @@ CONF_Int32(max_tablet_index_num_per_batch, "1000"); // Max aborted txn num for the same label name CONF_mInt64(max_num_aborted_txn, "100"); + +// Enables validation to check the correctness of table size in cloud mode. +CONF_Bool(enable_cloud_table_size_correctness_check, "false"); } // namespace doris::cloud::config diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf b/regression-test/pipeline/cloud_p0/conf/be_custom.conf index 377a02536c6d292..5fbf89fd75cc77e 100644 --- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf @@ -37,3 +37,4 @@ enable_new_tablet_do_compaction = true arrow_flight_sql_port = 8181 pipeline_task_leakage_detect_period_sec=1 crash_in_memory_tracker_inaccurate = true +enable_table_size_correctness_check=true diff --git a/regression-test/pipeline/cloud_p1/conf/be_custom.conf b/regression-test/pipeline/cloud_p1/conf/be_custom.conf index 0dc78140ed9d44a..aefdda985903f70 100644 --- a/regression-test/pipeline/cloud_p1/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/be_custom.conf @@ -32,3 +32,4 @@ save_load_error_log_to_s3 = true arrow_flight_sql_port = 8181 pipeline_task_leakage_detect_period_sec=1 crash_in_memory_tracker_inaccurate = true +enable_table_size_correctness_check=true diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf index c5c8104ecf1279e..17113b01e1e7e0b 100644 --- a/regression-test/pipeline/p0/conf/be.conf +++ b/regression-test/pipeline/p0/conf/be.conf @@ -71,3 +71,4 @@ be_proc_monitor_interval_ms = 30000 webserver_num_workers = 128 pipeline_task_leakage_detect_period_sec=1 crash_in_memory_tracker_inaccurate = true +enable_table_size_correctness_check=true diff --git a/regression-test/pipeline/p1/conf/be.conf b/regression-test/pipeline/p1/conf/be.conf index 01510e6422b9753..bacf4caae1b0819 100644 --- a/regression-test/pipeline/p1/conf/be.conf +++ b/regression-test/pipeline/p1/conf/be.conf @@ -63,3 +63,4 @@ enable_missing_rows_correctness_check=true enable_jvm_monitor = true pipeline_task_leakage_detect_period_sec=1 crash_in_memory_tracker_inaccurate = true +enable_table_size_correctness_check=true