diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp index 850ac5766fc119..4512cd0dfff538 100644 --- a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp @@ -250,6 +250,9 @@ Status DataTypeDateTimeV2SerDe::write_column_to_orc(const std::string& timezone, Status DataTypeDateTimeV2SerDe::deserialize_column_from_fixed_json( IColumn& column, Slice& slice, int rows, int* num_deserialized, const FormatOptions& options) const { + if (rows < 1) [[unlikely]] { + return Status::OK(); + } Status st = deserialize_one_cell_from_json(column, slice, options); if (!st.ok()) { return st; @@ -262,6 +265,9 @@ Status DataTypeDateTimeV2SerDe::deserialize_column_from_fixed_json( void DataTypeDateTimeV2SerDe::insert_column_last_value_multiple_times(IColumn& column, int times) const { + if (times < 1) [[unlikely]] { + return; + } auto& col = static_cast&>(column); auto sz = col.size(); UInt64 val = col.get_element(sz - 1); diff --git a/be/src/vec/data_types/serde/data_type_datev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datev2_serde.cpp index f2d595b87c452f..95109ee408caee 100644 --- a/be/src/vec/data_types/serde/data_type_datev2_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_datev2_serde.cpp @@ -178,6 +178,9 @@ Status DataTypeDateV2SerDe::write_column_to_orc(const std::string& timezone, con Status DataTypeDateV2SerDe::deserialize_column_from_fixed_json(IColumn& column, Slice& slice, int rows, int* num_deserialized, const FormatOptions& options) const { + if (rows < 1) [[unlikely]] { + return Status::OK(); + } Status st = deserialize_one_cell_from_json(column, slice, options); if (!st.ok()) { return st; @@ -189,6 +192,9 @@ Status DataTypeDateV2SerDe::deserialize_column_from_fixed_json(IColumn& column, void DataTypeDateV2SerDe::insert_column_last_value_multiple_times(IColumn& column, int times) const { + if (times < 1) [[unlikely]] { + return; + } auto& col = static_cast&>(column); auto sz = col.size(); UInt32 val = col.get_element(sz - 1); diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp index e979211d6d720b..acb09ee773ec62 100644 --- a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp @@ -280,6 +280,9 @@ template Status DataTypeDecimalSerDe::deserialize_column_from_fixed_json( IColumn& column, Slice& slice, int rows, int* num_deserialized, const FormatOptions& options) const { + if (rows < 1) [[unlikely]] { + return Status::OK(); + } Status st = deserialize_one_cell_from_json(column, slice, options); if (!st.ok()) { return st; @@ -293,6 +296,9 @@ Status DataTypeDecimalSerDe::deserialize_column_from_fixed_json( template void DataTypeDecimalSerDe::insert_column_last_value_multiple_times(IColumn& column, int times) const { + if (times < 1) [[unlikely]] { + return; + } auto& col = static_cast&>(column); auto sz = col.size(); diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp index 1af85bd040d1e2..3b46e0e784f8f3 100644 --- a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp @@ -131,12 +131,15 @@ Status DataTypeNullableSerDe::deserialize_column_from_hive_text_vector( Status DataTypeNullableSerDe::deserialize_column_from_fixed_json( IColumn& column, Slice& slice, int rows, int* num_deserialized, const FormatOptions& options) const { + if (rows < 1) [[unlikely]] { + return Status::OK(); + } auto& col = static_cast(column); Status st = deserialize_one_cell_from_json(column, slice, options); if (!st.ok()) { return st; } - if (rows - 1 != 0) { + if (rows > 1) { auto& null_map = col.get_null_map_data(); auto& nested_column = col.get_nested_column(); diff --git a/be/src/vec/data_types/serde/data_type_number_serde.cpp b/be/src/vec/data_types/serde/data_type_number_serde.cpp index 299779ea267961..efa41e346bfa6e 100644 --- a/be/src/vec/data_types/serde/data_type_number_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_number_serde.cpp @@ -228,6 +228,9 @@ template Status DataTypeNumberSerDe::deserialize_column_from_fixed_json( IColumn& column, Slice& slice, int rows, int* num_deserialized, const FormatOptions& options) const { + if (rows < 1) [[unlikely]] { + return Status::OK(); + } Status st = deserialize_one_cell_from_json(column, slice, options); if (!st.ok()) { return st; @@ -241,6 +244,9 @@ Status DataTypeNumberSerDe::deserialize_column_from_fixed_json( template void DataTypeNumberSerDe::insert_column_last_value_multiple_times(IColumn& column, int times) const { + if (times < 1) [[unlikely]] { + return; + } auto& col = static_cast&>(column); auto sz = col.size(); T val = col.get_element(sz - 1); diff --git a/be/src/vec/data_types/serde/data_type_serde.h b/be/src/vec/data_types/serde/data_type_serde.h index fd773718000a0e..7dedf30ac32aa6 100644 --- a/be/src/vec/data_types/serde/data_type_serde.h +++ b/be/src/vec/data_types/serde/data_type_serde.h @@ -243,17 +243,26 @@ class DataTypeSerDe { virtual Status deserialize_column_from_fixed_json(IColumn& column, Slice& slice, int rows, int* num_deserialized, const FormatOptions& options) const { + //In this function implementation, we need to consider the case where rows is 0, 1, and other larger integers. + if (rows < 1) [[unlikely]] { + return Status::OK(); + } Status st = deserialize_one_cell_from_json(column, slice, options); if (!st.ok()) { *num_deserialized = 0; return st; } - insert_column_last_value_multiple_times(column, rows - 1); + if (rows > 1) [[likely]] { + insert_column_last_value_multiple_times(column, rows - 1); + } *num_deserialized = rows; return Status::OK(); } // Insert the last value to the end of this column multiple times. virtual void insert_column_last_value_multiple_times(IColumn& column, int times) const { + if (times < 1) [[unlikely]] { + return; + } //If you try to simplify this operation by using `column.insert_many_from(column, column.size() - 1, rows - 1);` // you are likely to get incorrect data results. MutableColumnPtr dum_col = column.clone_empty(); diff --git a/be/src/vec/data_types/serde/data_type_string_serde.h b/be/src/vec/data_types/serde/data_type_string_serde.h index fe09ff615f4742..583772c582530f 100644 --- a/be/src/vec/data_types/serde/data_type_string_serde.h +++ b/be/src/vec/data_types/serde/data_type_string_serde.h @@ -218,6 +218,9 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status deserialize_column_from_fixed_json(IColumn& column, Slice& slice, int rows, int* num_deserialized, const FormatOptions& options) const override { + if (rows < 1) [[unlikely]] { + return Status::OK(); + } Status st = deserialize_one_cell_from_json(column, slice, options); if (!st.ok()) { return st; @@ -229,6 +232,9 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { } void insert_column_last_value_multiple_times(IColumn& column, int times) const override { + if (times < 1) [[unlikely]] { + return; + } auto& col = static_cast(column); auto sz = col.size(); diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index 56e855dc216bcf..6b869e138d47e4 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -654,6 +654,7 @@ Status RowGroupReader::_fill_partition_columns( auto _text_serde = slot_desc->get_data_type_ptr()->get_serde(); Slice slice(value.data(), value.size()); int num_deserialized = 0; + // Be careful when reading empty rows from parquet row groups. if (_text_serde->deserialize_column_from_fixed_json(*col_ptr, slice, rows, &num_deserialized, _text_formatOptions) != Status::OK()) {