[cherry-pick](branch-2.1) Pick "[Fix](Rowset Id) Use a randomly gener…

…ated rowset ID to handle memory write failures (#42949)" (#44086)
apache · Dec 10, 2024 · d0737c4 · d0737c4
1 parent e29d125
commit d0737c4
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 1 deletion.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1346,6 +1346,7 @@ DEFINE_mInt32(snappy_compression_block_size, "262144");
 DEFINE_mInt32(lz4_compression_block_size, "262144");
 
 DEFINE_mBool(enable_pipeline_task_leakage_detect, "false");
+DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false");
 
 // clang-format off
 #ifdef BE_TEST

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1417,6 +1417,7 @@ DECLARE_mInt32(snappy_compression_block_size);
 DECLARE_mInt32(lz4_compression_block_size);
 
 DECLARE_mBool(enable_pipeline_task_leakage_detect);
+DECLARE_Bool(force_regenerate_rowsetid_on_start_error);
 
 #ifdef BE_TEST
 // test s3

diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
@@ -34,6 +34,7 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "common/config.h"
 #include "io/io_common.h"
 #include "olap/olap_define.h"
 #include "olap/rowset/rowset_fwd.h"
@@ -396,6 +397,8 @@ using ColumnId = uint32_t;
 using UniqueIdSet = std::set<uint32_t>;
 // Column unique Id -> column id map
 using UniqueIdToColumnIdMap = std::map<ColumnId, ColumnId>;
+struct RowsetId;
+RowsetId next_rowset_id();
 
 // 8 bit rowset id version
 // 56 bit, inc number from 1
@@ -414,7 +417,12 @@ struct RowsetId {
             auto [_, ec] = std::from_chars(rowset_id_str.data(),
                                            rowset_id_str.data() + rowset_id_str.length(), high);
             if (ec != std::errc {}) [[unlikely]] {
-                LOG(FATAL) << "failed to init rowset id: " << rowset_id_str;
+                if (config::force_regenerate_rowsetid_on_start_error) {
+                    LOG(WARNING) << "failed to init rowset id: " << rowset_id_str;
+                    high = next_rowset_id().hi;
+                } else {
+                    LOG(FATAL) << "failed to init rowset id: " << rowset_id_str;
+                }
             }
             init(1, high, 0, 0);
         } else {

diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp
@@ -17,8 +17,17 @@
 
 #include "olap/rowset/unique_rowset_id_generator.h"
 
+#include <atomic>
+
+#include "olap/storage_engine.h"
+#include "runtime/exec_env.h"
+
 namespace doris {
 
+RowsetId next_rowset_id() {
+    return ExecEnv::GetInstance()->get_storage_engine()->next_rowset_id();
+}
+
 UniqueRowsetIdGenerator::UniqueRowsetIdGenerator(const UniqueId& backend_uid)
         : _backend_uid(backend_uid), _inc_id(1) {}