Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feature](hive)support hive catalog read json table. #43469

Merged
merged 8 commits into from
Nov 28, 2024
Merged

[feature](hive)support hive catalog read json table. #43469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7d30eab
1
morningman Nov 3, 2024
0cfd0ab
impl be
hubgeter Nov 7, 2024
abddfc8
fix lower and upper
hubgeter Nov 10, 2024
fbd7441
fix comment
hubgeter Nov 11, 2024
612c4cf
fix comment.
hubgeter Nov 13, 2024
3ff999e
fix compile
hubgeter Nov 14, 2024
72b173b
fix {} and select col and col not in json.
hubgeter Nov 17, 2024
5e9e0ad
add LEGACY_HIVE_JSON_SERDE
morningman Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions be/src/vec/data_types/serde/data_type_array_serde.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ class DataTypeArraySerDe : public DataTypeSerDe {
nested_serde->set_return_object_as_string(value);
}

virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {nested_serde}; }

private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,
Expand Down
4 changes: 4 additions & 0 deletions be/src/vec/data_types/serde/data_type_map_serde.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ class DataTypeMapSerDe : public DataTypeSerDe {
value_serde->set_return_object_as_string(value);
}

virtual DataTypeSerDeSPtrs get_nested_serdes() const override {
return {key_serde, value_serde};
}

private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,
Expand Down
2 changes: 2 additions & 0 deletions be/src/vec/data_types/serde/data_type_nullable_serde.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ class DataTypeNullableSerDe : public DataTypeSerDe {
int64_t row_num) const override;
Status read_one_cell_from_json(IColumn& column, const rapidjson::Value& result) const override;

virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {nested_serde}; }

private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,
Expand Down
12 changes: 9 additions & 3 deletions be/src/vec/data_types/serde/data_type_serde.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ class IColumn;
class Arena;
class IDataType;

class DataTypeSerDe;
using DataTypeSerDeSPtr = std::shared_ptr<DataTypeSerDe>;
using DataTypeSerDeSPtrs = std::vector<DataTypeSerDeSPtr>;

// Deserialize means read from different file format or memory format,
// for example read from arrow, read from parquet.
// Serialize means write the column cell or the total column into another
Expand Down Expand Up @@ -337,6 +341,11 @@ class DataTypeSerDe {
Arena& mem_pool, int64_t row_num) const;
virtual Status read_one_cell_from_json(IColumn& column, const rapidjson::Value& result) const;

virtual DataTypeSerDeSPtrs get_nested_serdes() const {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe we can support
get_key_serde() get_value_serde() for map
get_data_serde() for array nullcolumn
get_serde(int) for struct
not get_nested_serdes() for any complex type

throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
"Method get_nested_serdes is not supported for this serde");
}

protected:
bool _return_object_as_string = false;
// This parameter indicates what level the serde belongs to and is mainly used for complex types
Expand Down Expand Up @@ -379,9 +388,6 @@ inline void checkArrowStatus(const arrow::Status& status, const std::string& col
}
}

using DataTypeSerDeSPtr = std::shared_ptr<DataTypeSerDe>;
using DataTypeSerDeSPtrs = std::vector<DataTypeSerDeSPtr>;

DataTypeSerDeSPtrs create_data_type_serdes(
const std::vector<std::shared_ptr<const IDataType>>& types);
DataTypeSerDeSPtrs create_data_type_serdes(const std::vector<SlotDescriptor*>& slots);
Expand Down
2 changes: 2 additions & 0 deletions be/src/vec/data_types/serde/data_type_struct_serde.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ class DataTypeStructSerDe : public DataTypeSerDe {
}
}

virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return elem_serdes_ptrs; }

private:
std::optional<size_t> try_get_position_by_name(const String& name) const;

Expand Down
Loading
Loading