Merge branch 'StarRocks:main' into fix-cves

be/CMakeLists.txt

@@ -625,6 +625,10 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-switch-default")
     endif ()
 else ()
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0.0")
+        # ignore error from apache-orc
+        set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-error=dangling-reference")
+    endif ()
     set(CXX_GCC_FLAGS "${CXX_GCC_FLAGS} -fcoroutines")
 endif()
 
@@ -773,6 +777,8 @@ endif()
 
 set(WL_START_GROUP "-Wl,--start-group")
 set(WL_END_GROUP "-Wl,--end-group")
+set(WL_LINK_STATIC "-Wl,-Bstatic")
+set(WL_LINK_DYNAMIC "-Wl,-Bdynamic")
 
 # Set starrocks libraries
 set(STARROCKS_LINK_LIBS
@@ -963,7 +969,8 @@ if (NOT ("${MAKE_TEST}" STREQUAL "ON" AND "${BUILD_FOR_SANITIZE}" STREQUAL "ON")
 endif()
 
 set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS}
-    -lresolv -lbfd -liberty -lc -lm -ldl -rdynamic -pthread -Wl,-wrap=__cxa_throw
+    ${WL_LINK_STATIC} -lbfd
+    ${WL_LINK_DYNAMIC} -lresolv -liberty -lc -lm -ldl -rdynamic -pthread -Wl,-wrap=__cxa_throw
 )
 
 # link gcov if WITH_GCOV is on

be/src/column/adaptive_nullable_column.h

@@ -276,9 +276,7 @@ class AdaptiveNullableColumn final : public ColumnFactory<NullableColumn, Adapti
 
     StatusOr<ColumnPtr> upgrade_if_overflow() override {
         materialized_nullable();
-        if (_null_column->capacity_limit_reached()) {
-            return Status::InternalError("Size of NullableColumn exceed the limit");
-        }
+        RETURN_IF_ERROR(_null_column->capacity_limit_reached());
 
         return upgrade_helper_func(&_data_column);
     }
@@ -556,9 +554,9 @@ class AdaptiveNullableColumn final : public ColumnFactory<NullableColumn, Adapti
         return NullableColumn::debug_string();
     }
 
-    bool capacity_limit_reached(std::string* msg = nullptr) const override {
+    Status capacity_limit_reached() const override {
         materialized_nullable();
-        return NullableColumn::capacity_limit_reached(msg);
+        return NullableColumn::capacity_limit_reached();
     }
 
     void check_or_die() const override {

be/src/column/array_column.h

@@ -182,8 +182,9 @@ class ArrayColumn final : public ColumnFactory<Column, ArrayColumn> {
 
     std::string debug_string() const override;
 
-    bool capacity_limit_reached(std::string* msg = nullptr) const override {
-        return _elements->capacity_limit_reached(msg) || _offsets->capacity_limit_reached(msg);
+    Status capacity_limit_reached() const override {
+        RETURN_IF_ERROR(_elements->capacity_limit_reached());
+        return _offsets->capacity_limit_reached();
     }
 
     StatusOr<ColumnPtr> upgrade_if_overflow() override;

be/src/column/binary_column.cpp

@@ -24,6 +24,7 @@
 #include "gutil/bits.h"
 #include "gutil/casts.h"
 #include "gutil/strings/fastmem.h"
+#include "gutil/strings/substitute.h"
 #include "util/hash_util.hpp"
 #include "util/mysql_row_buffer.h"
 #include "util/raw_container.h"
@@ -738,45 +739,37 @@ bool BinaryColumnBase<T>::has_large_column() const {
 }
 
 template <typename T>
-bool BinaryColumnBase<T>::capacity_limit_reached(std::string* msg) const {
+Status BinaryColumnBase<T>::capacity_limit_reached() const {
     static_assert(std::is_same_v<T, uint32_t> || std::is_same_v<T, uint64_t>);
     if constexpr (std::is_same_v<T, uint32_t>) {
         // The size limit of a single element is 2^32 - 1.
         // The size limit of all elements is 2^32 - 1.
         // The number limit of elements is 2^32 - 1.
         if (_bytes.size() >= Column::MAX_CAPACITY_LIMIT) {
-            if (msg != nullptr) {
-                msg->append("Total byte size of binary column exceed the limit: " +
-                            std::to_string(Column::MAX_CAPACITY_LIMIT));
-            }
-            return true;
+            return Status::CapacityLimitExceed(
+                    strings::Substitute("Total byte size of binary column exceed the limit: $0",
+                                        std::to_string(Column::MAX_CAPACITY_LIMIT)));
         } else if (_offsets.size() >= Column::MAX_CAPACITY_LIMIT) {
-            if (msg != nullptr) {
-                msg->append("Total row count of binary column exceed the limit: " +
-                            std::to_string(Column::MAX_CAPACITY_LIMIT));
-            }
-            return true;
+            return Status::CapacityLimitExceed(
+                    strings::Substitute("Total row count of binary column exceed the limit: $0",
+                                        std::to_string(Column::MAX_CAPACITY_LIMIT)));
         } else {
-            return false;
+            return Status::OK();
         }
     } else {
         // The size limit of a single element is 2^32 - 1.
         // The size limit of all elements is 2^64 - 1.
         // The number limit of elements is 2^32 - 1.
         if (_bytes.size() >= Column::MAX_LARGE_CAPACITY_LIMIT) {
-            if (msg != nullptr) {
-                msg->append("Total byte size of large binary column exceed the limit: " +
-                            std::to_string(Column::MAX_LARGE_CAPACITY_LIMIT));
-            }
-            return true;
+            return Status::CapacityLimitExceed(
+                    strings::Substitute("Total byte size of large binary column exceed the limit: $0",
+                                        std::to_string(Column::MAX_LARGE_CAPACITY_LIMIT)));
         } else if (_offsets.size() >= Column::MAX_CAPACITY_LIMIT) {
-            if (msg != nullptr) {
-                msg->append("Total row count of large binary column exceed the limit: " +
-                            std::to_string(Column::MAX_CAPACITY_LIMIT));
-            }
-            return true;
+            return Status::CapacityLimitExceed(
+                    strings::Substitute("Total row count of large binary column exceed the limit: $0",
+                                        std::to_string(Column::MAX_CAPACITY_LIMIT)));
         } else {
-            return false;
+            return Status::OK();
         }
     }
 }

be/src/column/binary_column.h

@@ -339,7 +339,7 @@ class BinaryColumnBase final : public ColumnFactory<Column, BinaryColumnBase<T>>
         return ss.str();
     }
 
-    bool capacity_limit_reached(std::string* msg = nullptr) const override;
+    Status capacity_limit_reached() const override;
 
 private:
     void _build_slices() const;

be/src/column/chunk.h

@@ -277,13 +277,11 @@ class Chunk {
 
     std::string rebuild_csv_row(size_t index, const std::string& delimiter) const;
 
-    bool capacity_limit_reached(std::string* msg = nullptr) const {
+    Status capacity_limit_reached() const {
         for (const auto& column : _columns) {
-            if (column->capacity_limit_reached(msg)) {
-                return true;
-            }
+            RETURN_IF_ERROR(column->capacity_limit_reached());
         }
-        return false;
+        return Status::OK();
     }
 
     query_cache::owner_info& owner_info() { return _owner_info; }

be/src/column/column.h

@@ -394,7 +394,7 @@ class Column {
     // The interface will not free memory!!!
     virtual void reset_column() { _delete_state = DEL_NOT_SATISFIED; }
 
-    virtual bool capacity_limit_reached(std::string* msg = nullptr) const = 0;
+    virtual Status capacity_limit_reached() const = 0;
 
     virtual Status accept(ColumnVisitor* visitor) const = 0;
 

be/src/column/column_builder.h

@@ -57,7 +57,7 @@ class ColumnBuilder {
     ColumnBuilder(DataColumnPtr column, NullColumnPtr null_column, bool has_null)
             : _column(std::move(column)), _null_column(std::move(null_column)), _has_null(has_null) {}
     //do nothing ctor, members are initialized by its offsprings.
-    explicit ColumnBuilder<Type>(void*) {}
+    explicit ColumnBuilder(void*) {}
 
     void append(const DatumType& value) {
         _null_column->append(DATUM_NOT_NULL);

be/src/column/column_helper.h

@@ -525,19 +525,21 @@ struct ChunkSliceTemplate {
 template <LogicalType ltype>
 struct GetContainer {
     using ColumnType = typename RunTimeTypeTraits<ltype>::ColumnType;
-    const auto& get_data(const Column* column) { return ColumnHelper::as_raw_column<ColumnType>(column)->get_data(); }
-    const auto& get_data(const ColumnPtr& column) {
+    static const auto& get_data(const Column* column) {
+        return ColumnHelper::as_raw_column<ColumnType>(column)->get_data();
+    }
+    static const auto& get_data(const ColumnPtr& column) {
         return ColumnHelper::as_raw_column<ColumnType>(column.get())->get_data();
     }
 };
 
 #define GET_CONTAINER(ltype)                                                                  \
     template <>                                                                               \
     struct GetContainer<ltype> {                                                              \
-        const auto& get_data(const Column* column) {                                          \
+        static const auto& get_data(const Column* column) {                                   \
             return ColumnHelper::as_raw_column<BinaryColumn>(column)->get_proxy_data();       \
         }                                                                                     \
-        const auto& get_data(const ColumnPtr& column) {                                       \
+        static const auto& get_data(const ColumnPtr& column) {                                \
             return ColumnHelper::as_raw_column<BinaryColumn>(column.get())->get_proxy_data(); \
         }                                                                                     \
     };

be/src/column/const_column.h

@@ -18,6 +18,7 @@
 #include "column/datum.h"
 #include "column/vectorized_fwd.h"
 #include "common/logging.h"
+#include "gutil/strings/substitute.h"
 
 namespace starrocks {
 
@@ -251,15 +252,13 @@ class ConstColumn final : public ColumnFactory<Column, ConstColumn> {
         return ss.str();
     }
 
-    bool capacity_limit_reached(std::string* msg = nullptr) const override {
-        RETURN_IF_UNLIKELY(_data->capacity_limit_reached(msg), true);
+    Status capacity_limit_reached() const override {
+        RETURN_IF_ERROR(_data->capacity_limit_reached());
         if (_size > Column::MAX_CAPACITY_LIMIT) {
-            if (msg != nullptr) {
-                msg->append("Row count of const column reach limit: " + std::to_string(Column::MAX_CAPACITY_LIMIT));
-            }
-            return true;
+            return Status::CapacityLimitExceed(strings::Substitute("Row count of const column reach limit: $0",
+                                                                   std::to_string(Column::MAX_CAPACITY_LIMIT)));
         }
-        return false;
+        return Status::OK();
     }
 
     void check_or_die() const override;

be/src/column/field.h

@@ -29,22 +29,36 @@
 namespace starrocks {
 
 class Datum;
+class AggStateDesc;
 
 class Field {
 public:
     Field(ColumnId id, std::string_view name, TypeInfoPtr type, starrocks::StorageAggregateType agg,
-          uint8_t short_key_length, bool is_key, bool nullable)
+          AggStateDesc* agg_state_desc, uint8_t short_key_length, bool is_key, bool nullable)
             : _id(id),
               _agg_method(agg),
+              _agg_state_desc(agg_state_desc),
               _name(name),
               _type(std::move(type)),
               _sub_fields(nullptr),
               _short_key_length(short_key_length),
-              _flags(static_cast<uint8_t>((is_key << kIsKeyShift) | (nullable << kNullableShift))) {}
+              _flags(static_cast<uint8_t>((is_key << kIsKeyShift) | (nullable << kNullableShift))) {
+        if (_agg_method == STORAGE_AGGREGATE_AGG_STATE_UNION) {
+            DCHECK(_agg_state_desc != nullptr);
+        }
+    }
+
+    // AggMethod is not STORAGE_AGGREGATE_AGG_STATE_UNION
+    Field(ColumnId id, std::string_view name, TypeInfoPtr type, starrocks::StorageAggregateType agg,
+          uint8_t short_key_length, bool is_key, bool nullable)
+            : Field(id, name, std::move(type), agg, nullptr, short_key_length, is_key, nullable) {
+        DCHECK(_agg_method != STORAGE_AGGREGATE_AGG_STATE_UNION);
+    }
 
     // Non-key field of any type except for ARRAY
     Field(ColumnId id, std::string_view name, LogicalType type, int precision, int scale, bool nullable)
-            : Field(id, name, get_type_info(type, precision, scale), STORAGE_AGGREGATE_NONE, 0, false, nullable) {}
+            : Field(id, name, get_type_info(type, precision, scale), STORAGE_AGGREGATE_NONE, nullptr, 0, false,
+                    nullable) {}
 
     // Non-key field of any type except for DECIMAL32, DECIMAL64, DECIMAL128, and ARRAY
     Field(ColumnId id, std::string_view name, LogicalType type, bool nullable)
@@ -57,7 +71,7 @@ class Field {
 
     // Non-key field of any type
     Field(ColumnId id, std::string_view name, TypeInfoPtr type, bool nullable = true)
-            : Field(id, name, std::move(type), STORAGE_AGGREGATE_NONE, 0, false, nullable) {}
+            : Field(id, name, std::move(type), STORAGE_AGGREGATE_NONE, nullptr, 0, false, nullable) {}
 
     ~Field() { delete _sub_fields; }
 
@@ -66,6 +80,7 @@ class Field {
     Field(const Field& rhs)
             : _id(rhs._id),
               _agg_method(rhs._agg_method),
+              _agg_state_desc(rhs._agg_state_desc),
               _name(rhs._name),
               _type(rhs._type),
               _sub_fields(rhs._sub_fields ? new Buffer<Field>(*rhs._sub_fields) : nullptr),
@@ -76,6 +91,7 @@ class Field {
     Field(Field&& rhs) noexcept
             : _id(rhs._id),
               _agg_method(rhs._agg_method),
+              _agg_state_desc(rhs._agg_state_desc),
               _name(std::move(rhs._name)),
               _type(std::move(rhs._type)),
               _sub_fields(rhs._sub_fields),
@@ -92,6 +108,7 @@ class Field {
             _name = rhs._name;
             _type = rhs._type;
             _agg_method = rhs._agg_method;
+            _agg_state_desc = rhs._agg_state_desc;
             _short_key_length = rhs._short_key_length;
             _flags = rhs._flags;
             _sub_fields = rhs._sub_fields ? new Buffer<Field>(*rhs._sub_fields) : nullptr;
@@ -106,6 +123,7 @@ class Field {
             _name = std::move(rhs._name);
             _type = std::move(rhs._type);
             _agg_method = rhs._agg_method;
+            _agg_state_desc = rhs._agg_state_desc;
             _short_key_length = rhs._short_key_length;
             _flags = rhs._flags;
             _uid = rhs._uid;
@@ -171,6 +189,9 @@ class Field {
     void set_uid(ColumnUID uid) { _uid = uid; }
     const ColumnUID& uid() const { return _uid; }
 
+    void set_agg_state_desc(AggStateDesc* agg_state_desc) { _agg_state_desc = agg_state_desc; }
+    AggStateDesc* get_agg_state_desc() const { return _agg_state_desc; }
+
     static FieldPtr convert_to_dict_field(const Field& field);
 
 private:
@@ -179,6 +200,8 @@ class Field {
 
     ColumnId _id = 0;
     starrocks::StorageAggregateType _agg_method;
+    // agg_state_desc if agg_method is STORAGE_AGGREGATE_AGG_STATE_UNION
+    AggStateDesc* _agg_state_desc;
     CString _name;
     TypeInfoPtr _type = nullptr;
     std::vector<Field>* _sub_fields;
@@ -225,16 +248,17 @@ inline const Field& Field::sub_field(int i) const {
 
 inline FieldPtr Field::with_type(const TypeInfoPtr& type) {
     return std::make_shared<Field>(_id, std::string_view(_name.data(), _name.size()), type, _agg_method,
-                                   _short_key_length, is_key(), is_nullable());
+                                   _agg_state_desc, _short_key_length, is_key(), is_nullable());
 }
 
 inline FieldPtr Field::with_name(std::string_view name) {
-    return std::make_shared<Field>(_id, name, _type, _agg_method, _short_key_length, is_key(), is_nullable());
+    return std::make_shared<Field>(_id, name, _type, _agg_method, _agg_state_desc, _short_key_length, is_key(),
+                                   is_nullable());
 }
 
 inline FieldPtr Field::with_nullable(bool nullable) {
     return std::make_shared<Field>(_id, std::string_view(_name.data(), _name.size()), _type, _agg_method,
-                                   _short_key_length, is_key(), nullable);
+                                   _agg_state_desc, _short_key_length, is_key(), nullable);
 }
 
 inline std::ostream& operator<<(std::ostream& os, const Field& field) {

be/src/column/fixed_length_column_base.cpp

@@ -29,11 +29,8 @@ namespace starrocks {
 
 template <typename T>
 StatusOr<ColumnPtr> FixedLengthColumnBase<T>::upgrade_if_overflow() {
-    if (capacity_limit_reached()) {
-        return Status::InternalError("Size of FixedLengthColumn exceed the limit");
-    } else {
-        return nullptr;
-    }
+    RETURN_IF_ERROR(capacity_limit_reached());
+    return nullptr;
 }
 
 template <typename T>

be/src/column/fixed_length_column_base.h

@@ -21,6 +21,7 @@
 #include "column/datum.h"
 #include "column/vectorized_fwd.h"
 #include "common/statusor.h"
+#include "gutil/strings/substitute.h"
 #include "runtime/decimalv2_value.h"
 #include "types/date_value.hpp"
 #include "types/timestamp_value.h"
@@ -236,15 +237,13 @@ class FixedLengthColumnBase : public ColumnFactory<Column, FixedLengthColumnBase
 
     // The `_data` support one size(> 2^32), but some interface such as update_rows() will use index of uint32_t to
     // access the item, so we should use 2^32 as the limit
-    bool capacity_limit_reached(std::string* msg = nullptr) const override {
+    Status capacity_limit_reached() const override {
         if (_data.size() > Column::MAX_CAPACITY_LIMIT) {
-            if (msg != nullptr) {
-                msg->append("row count of fixed length column exceend the limit: " +
-                            std::to_string(Column::MAX_CAPACITY_LIMIT));
-            }
-            return true;
+            return Status::CapacityLimitExceed(
+                    strings::Substitute("row count of fixed length column exceend the limit: $0",
+                                        std::to_string(Column::MAX_CAPACITY_LIMIT)));
         }
-        return false;
+        return Status::OK();
     }
 
     void check_or_die() const override {}