[feature](hive)support hive catalog read json table. (#43469) (#44848)

bp #43469
apache · Dec 4, 2024 · 8efb98e · 8efb98e
1 parent abff86a
commit 8efb98e
Show file tree

Hide file tree

Showing 22 changed files with 830 additions and 158 deletions.
diff --git a/be/src/vec/data_types/serde/data_type_array_serde.h b/be/src/vec/data_types/serde/data_type_array_serde.h
@@ -101,6 +101,8 @@ class DataTypeArraySerDe : public DataTypeSerDe {
         nested_serde->set_return_object_as_string(value);
     }
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {nested_serde}; }
+
 private:
     template <bool is_binary_format>
     Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,

diff --git a/be/src/vec/data_types/serde/data_type_map_serde.h b/be/src/vec/data_types/serde/data_type_map_serde.h
@@ -95,6 +95,10 @@ class DataTypeMapSerDe : public DataTypeSerDe {
         value_serde->set_return_object_as_string(value);
     }
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override {
+        return {key_serde, value_serde};
+    }
+
 private:
     template <bool is_binary_format>
     Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,

diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.h b/be/src/vec/data_types/serde/data_type_nullable_serde.h
@@ -99,6 +99,8 @@ class DataTypeNullableSerDe : public DataTypeSerDe {
                                   int row_num) const override;
     Status read_one_cell_from_json(IColumn& column, const rapidjson::Value& result) const override;
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {nested_serde}; }
+
 private:
     template <bool is_binary_format>
     Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,

diff --git a/be/src/vec/data_types/serde/data_type_serde.h b/be/src/vec/data_types/serde/data_type_serde.h
@@ -98,6 +98,10 @@ class IColumn;
 class Arena;
 class IDataType;
 
+class DataTypeSerDe;
+using DataTypeSerDeSPtr = std::shared_ptr<DataTypeSerDe>;
+using DataTypeSerDeSPtrs = std::vector<DataTypeSerDeSPtr>;
+
 // Deserialize means read from different file format or memory format,
 // for example read from arrow, read from parquet.
 // Serialize means write the column cell or the total column into another
@@ -332,6 +336,11 @@ class DataTypeSerDe {
                                           Arena& mem_pool, int row_num) const;
     virtual Status read_one_cell_from_json(IColumn& column, const rapidjson::Value& result) const;
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const {
+        throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
+                               "Method get_nested_serdes is not supported for this serde");
+    }
+
 protected:
     bool _return_object_as_string = false;
     // This parameter indicates what level the serde belongs to and is mainly used for complex types
@@ -374,9 +383,6 @@ inline void checkArrowStatus(const arrow::Status& status, const std::string& col
     }
 }
 
-using DataTypeSerDeSPtr = std::shared_ptr<DataTypeSerDe>;
-using DataTypeSerDeSPtrs = std::vector<DataTypeSerDeSPtr>;
-
 DataTypeSerDeSPtrs create_data_type_serdes(
         const std::vector<std::shared_ptr<const IDataType>>& types);
 DataTypeSerDeSPtrs create_data_type_serdes(const std::vector<SlotDescriptor*>& slots);

diff --git a/be/src/vec/data_types/serde/data_type_struct_serde.h b/be/src/vec/data_types/serde/data_type_struct_serde.h
@@ -171,6 +171,8 @@ class DataTypeStructSerDe : public DataTypeSerDe {
         }
     }
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return elem_serdes_ptrs; }
+
 private:
     std::optional<size_t> try_get_position_by_name(const String& name) const;
-Original file line number
+Diff line change
@@ Expand Up / @@ -171,6 +171,8 @@ class DataTypeStructSerDe : public DataTypeSerDe { @@
             }
         }
+        virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return elem_serdes_ptrs; }
     private:
         std::optional<size_t> try_get_position_by_name(const String& name) const;
@@ Expand Down @@