[Fix](Rowset Id) Use a randomly generated rowset ID to handle memory …

…write failures (apache#42949)
Yukang-Lian · Dec 27, 2024 · 2ff8637 · 2ff8637
1 parent fdff4a6
commit 2ff8637
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 2 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1191,6 +1191,7 @@ DEFINE_mInt64(tablet_meta_serialize_size_limit, "1610612736");
 // 1717986918 = 2GB * 0.8
 DEFINE_Validator(tablet_meta_serialize_size_limit,
                  [](const int64_t config) -> bool { return config < 1717986918; });
+DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false");
 
 // clang-format off
 #ifdef BE_TEST

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1239,6 +1239,7 @@ DECLARE_Int32(partition_disk_index_lru_size);
 DECLARE_mBool(ignore_schema_change_check);
 
 DECLARE_mInt64(tablet_meta_serialize_size_limit);
+DECLARE_Bool(force_regenerate_rowsetid_on_start_error);
 
 #ifdef BE_TEST
 // test s3

diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
@@ -20,6 +20,7 @@
 #include <gen_cpp/Types_types.h>
 #include <netinet/in.h>
 
+#include <charconv>
 #include <cstdint>
 #include <functional>
 #include <list>
@@ -32,6 +33,7 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "common/config.h"
 #include "io/io_common.h"
 #include "olap/olap_define.h"
 #include "util/hash_util.hpp"
@@ -381,6 +383,8 @@ using ColumnId = uint32_t;
 using UniqueIdSet = std::set<uint32_t>;
 // Column unique Id -> column id map
 using UniqueIdToColumnIdMap = std::map<ColumnId, ColumnId>;
+struct RowsetId;
+RowsetId next_rowset_id();
 
 // 8 bit rowset id version
 // 56 bit, inc number from 1
@@ -394,8 +398,18 @@ struct RowsetId {
     void init(const std::string& rowset_id_str) {
         // for new rowsetid its a 48 hex string
         // if the len < 48, then it is an old format rowset id
-        if (rowset_id_str.length() < 48) {
-            int64_t high = std::stol(rowset_id_str, nullptr, 10);
+        if (rowset_id_str.length() < 48) [[unlikely]] {
+            int64_t high;
+            auto [_, ec] = std::from_chars(rowset_id_str.data(),
+                                           rowset_id_str.data() + rowset_id_str.length(), high);
+            if (ec != std::errc {}) [[unlikely]] {
+                if (config::force_regenerate_rowsetid_on_start_error) {
+                    LOG(WARNING) << "failed to init rowset id: " << rowset_id_str;
+                    high = MAX_ROWSET_ID - 1;
+                } else {
+                    LOG(FATAL) << "failed to init rowset id: " << rowset_id_str;
+                }
+            }
             init(1, high, 0, 0);
         } else {
             int64_t high = 0;

diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp
@@ -20,6 +20,8 @@
 #include <memory>
 #include <mutex>
 
+#include "olap/storage_engine.h"
+#include "runtime/exec_env.h"
 #include "util/doris_metrics.h"
 #include "util/metrics.h"
 #include "util/spinlock.h"
@@ -29,6 +31,10 @@ namespace doris {
 
 DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(rowset_count_generated_and_in_use, MetricUnit::ROWSETS);
 
+RowsetId next_rowset_id() {
+    return ExecEnv::GetInstance()->storage_engine()->next_rowset_id();
+}
+
 UniqueRowsetIdGenerator::UniqueRowsetIdGenerator(const UniqueId& backend_uid)
         : _backend_uid(backend_uid), _inc_id(0) {
     REGISTER_HOOK_METRIC(rowset_count_generated_and_in_use, [this]() {