3

Yukang-Lian · Sep 23, 2024 · d2e2444 · d2e2444
1 parent eb9d091
commit d2e2444
Show file tree

Hide file tree

Showing 9 changed files with 37 additions and 0 deletions.
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
@@ -748,6 +748,16 @@ Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta,
         Status ret_st;
         TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::commit_rowset", ret_st);
     }
+    if (config::enable_table_size_correctness_check) {
+        if (rs_meta.data_disk_size() + rs_meta.index_disk_size() != rs_meta.total_disk_size()) {
+            LOG(FATAL) << "[Cloud table size check failed]:"
+                       << " tablet id: " << rs_meta.tablet_id()
+                       << ", rowset id:" << rs_meta.rowset_id()
+                       << ", rowset data disk size:" << rs_meta.data_disk_size()
+                       << ", rowset index disk size:" << rs_meta.index_disk_size()
+                       << ", rowset total disk size:" << rs_meta.total_disk_size() << ".";
+        }
+    }
     CreateRowsetRequest req;
     CreateRowsetResponse resp;
     req.set_cloud_unique_id(config::cloud_unique_id);

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1340,6 +1340,9 @@ DEFINE_mBool(enable_pipeline_task_leakage_detect, "false");
 
 DEFINE_Int32(query_cache_size, "512");
 
+// Enable validation to check the correctness of table size.
+DEFINE_Bool(enable_table_size_correctness_check, "false");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1428,6 +1428,9 @@ DECLARE_mBool(enable_pipeline_task_leakage_detect);
 // MB
 DECLARE_Int32(query_cache_size);
 
+// Enable validation to check the correctness of table size.
+DECLARE_Bool(enable_table_size_correctness_check);
+
 #ifdef BE_TEST
 // test s3
 DECLARE_String(test_s3_resource);

diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
@@ -330,6 +330,20 @@ Status Tablet::init() {
 // should save tablet meta to remote meta store
 // if it's a primary replica
 void Tablet::save_meta() {
+    if (config::enable_table_size_correctness_check) {
+        const std::vector<RowsetMetaSharedPtr>& all_rs_metas = _tablet_meta->all_rs_metas();
+        for (const auto& rs_meta : all_rs_metas) {
+            if (rs_meta->data_disk_size() + rs_meta->index_disk_size() !=
+                rs_meta->total_disk_size()) {
+                LOG(FATAL) << "[Local table size check failed]:"
+                           << " tablet id: " << get_tablet_info().tablet_id
+                           << ", rowset id:" << rs_meta->rowset_id()
+                           << ", rowset data disk size:" << rs_meta->data_disk_size()
+                           << ", rowset index disk size:" << rs_meta->index_disk_size()
+                           << ", rowset total disk size:" << rs_meta->total_disk_size() << ".";
+            }
+        }
+    }
     auto res = _tablet_meta->save_meta(_data_dir);
     CHECK_EQ(res, Status::OK()) << "fail to save tablet_meta. res=" << res
                                 << ", root=" << _data_dir->path();

diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
@@ -218,4 +218,7 @@ CONF_Int32(max_tablet_index_num_per_batch, "1000");
 
 // Max aborted txn num for the same label name
 CONF_mInt64(max_num_aborted_txn, "100");
+
+// Enables validation to check the correctness of table size in cloud mode.
+CONF_Bool(enable_cloud_table_size_correctness_check, "false");
 } // namespace doris::cloud::config
diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
@@ -37,3 +37,4 @@ enable_new_tablet_do_compaction = true
 arrow_flight_sql_port = 8181
 pipeline_task_leakage_detect_period_sec=1
 crash_in_memory_tracker_inaccurate = true
+enable_table_size_correctness_check=true
diff --git a/regression-test/pipeline/cloud_p1/conf/be_custom.conf b/regression-test/pipeline/cloud_p1/conf/be_custom.conf
@@ -32,3 +32,4 @@ save_load_error_log_to_s3 = true
 arrow_flight_sql_port = 8181
 pipeline_task_leakage_detect_period_sec=1
 crash_in_memory_tracker_inaccurate = true
+enable_table_size_correctness_check=true
diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf
@@ -71,3 +71,4 @@ be_proc_monitor_interval_ms = 30000
 webserver_num_workers = 128
 pipeline_task_leakage_detect_period_sec=1
 crash_in_memory_tracker_inaccurate = true
+enable_table_size_correctness_check=true
diff --git a/regression-test/pipeline/p1/conf/be.conf b/regression-test/pipeline/p1/conf/be.conf
@@ -63,3 +63,4 @@ enable_missing_rows_correctness_check=true
 enable_jvm_monitor = true
 pipeline_task_leakage_detect_period_sec=1
 crash_in_memory_tracker_inaccurate = true
+enable_table_size_correctness_check=true