diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 486717d9bf33a2..fc86856a644d2a 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1191,6 +1191,7 @@ DEFINE_mInt64(tablet_meta_serialize_size_limit, "1610612736"); // 1717986918 = 2GB * 0.8 DEFINE_Validator(tablet_meta_serialize_size_limit, [](const int64_t config) -> bool { return config < 1717986918; }); +DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false"); // clang-format off #ifdef BE_TEST diff --git a/be/src/common/config.h b/be/src/common/config.h index 51f02afee2f712..5f6e48c772fc44 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1239,6 +1239,7 @@ DECLARE_Int32(partition_disk_index_lru_size); DECLARE_mBool(ignore_schema_change_check); DECLARE_mInt64(tablet_meta_serialize_size_limit); +DECLARE_Bool(force_regenerate_rowsetid_on_start_error); #ifdef BE_TEST // test s3 diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 811e77590f98b8..8a1e12fad24121 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include +#include "common/config.h" #include "io/io_common.h" #include "olap/olap_define.h" #include "util/hash_util.hpp" @@ -381,6 +383,8 @@ using ColumnId = uint32_t; using UniqueIdSet = std::set; // Column unique Id -> column id map using UniqueIdToColumnIdMap = std::map; +struct RowsetId; +RowsetId next_rowset_id(); // 8 bit rowset id version // 56 bit, inc number from 1 @@ -394,8 +398,18 @@ struct RowsetId { void init(const std::string& rowset_id_str) { // for new rowsetid its a 48 hex string // if the len < 48, then it is an old format rowset id - if (rowset_id_str.length() < 48) { - int64_t high = std::stol(rowset_id_str, nullptr, 10); + if (rowset_id_str.length() < 48) [[unlikely]] { + int64_t high; + auto [_, ec] = std::from_chars(rowset_id_str.data(), + rowset_id_str.data() + rowset_id_str.length(), high); + if (ec != std::errc {}) [[unlikely]] { + if (config::force_regenerate_rowsetid_on_start_error) { + LOG(WARNING) << "failed to init rowset id: " << rowset_id_str; + high = MAX_ROWSET_ID - 1; + } else { + LOG(FATAL) << "failed to init rowset id: " << rowset_id_str; + } + } init(1, high, 0, 0); } else { int64_t high = 0; diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp index 0d7d77915e0cb5..44d049b3002655 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.cpp +++ b/be/src/olap/rowset/unique_rowset_id_generator.cpp @@ -20,6 +20,8 @@ #include #include +#include "olap/storage_engine.h" +#include "runtime/exec_env.h" #include "util/doris_metrics.h" #include "util/metrics.h" #include "util/spinlock.h" @@ -29,6 +31,10 @@ namespace doris { DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(rowset_count_generated_and_in_use, MetricUnit::ROWSETS); +RowsetId next_rowset_id() { + return ExecEnv::GetInstance()->storage_engine()->next_rowset_id(); +} + UniqueRowsetIdGenerator::UniqueRowsetIdGenerator(const UniqueId& backend_uid) : _backend_uid(backend_uid), _inc_id(0) { REGISTER_HOOK_METRIC(rowset_count_generated_and_in_use, [this]() {