From cbba168ebc3a8a7797d1590e7a00e4cb22e632ba Mon Sep 17 00:00:00 2001 From: wyb Date: Mon, 2 Sep 2024 19:48:49 +0800 Subject: [PATCH 1/2] [Enhancement] Support infer struct type from parquet in files() (#50481) Signed-off-by: wyb (cherry picked from commit 37b633948a2948647416e9cb10a63bc157922e2c) # Conflicts: # test/sql/test_across_engine/R/test_sink_nested --- be/src/exec/parquet_schema_builder.cpp | 46 ++++ be/test/exec/parquet_scanner_test.cpp | 18 +- .../sql/test_across_engine/R/test_sink_nested | 199 ++++++++++++++++++ test/sql/test_pipe/R/basic | 2 +- 4 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 test/sql/test_across_engine/R/test_sink_nested diff --git a/be/src/exec/parquet_schema_builder.cpp b/be/src/exec/parquet_schema_builder.cpp index 8a3ac6bc1f9ea..f582520ed9da5 100644 --- a/be/src/exec/parquet_schema_builder.cpp +++ b/be/src/exec/parquet_schema_builder.cpp @@ -23,6 +23,7 @@ static Status get_parquet_type_from_group(const ::parquet::schema::NodePtr& node static Status get_parquet_type_from_primitive(const ::parquet::schema::NodePtr& node, TypeDescriptor* type_desc); static Status get_parquet_type_from_list(const ::parquet::schema::NodePtr& node, TypeDescriptor* type_desc); static Status get_parquet_type_from_map(const ::parquet::schema::NodePtr& node, TypeDescriptor* type_desc); +static Status try_to_infer_struct_type(const ::parquet::schema::NodePtr& node, TypeDescriptor* type_desc); Status get_parquet_type(const ::parquet::schema::NodePtr& node, TypeDescriptor* type_desc) { if (node->is_group()) { @@ -121,6 +122,11 @@ static Status get_parquet_type_from_group(const ::parquet::schema::NodePtr& node return get_parquet_type_from_map(node, type_desc); } + auto st = try_to_infer_struct_type(node, type_desc); + if (st.ok()) { + return Status::OK(); + } + // Treat unsupported types as VARCHAR. *type_desc = TypeDescriptor::create_varchar_type(TypeDescriptor::MAX_VARCHAR_LENGTH); return Status::OK(); @@ -217,4 +223,44 @@ static Status get_parquet_type_from_map(const ::parquet::schema::NodePtr& node, return Status::OK(); } +/* +try to infer struct type from group node. + +parquet does not have struct type, there is no struct definition in parquet. +try to infer like this. +group { + type field0; + type field1; + ... +} +*/ +static Status try_to_infer_struct_type(const ::parquet::schema::NodePtr& node, TypeDescriptor* type_desc) { + // 1st level. + // group name + DCHECK(node->is_group()); + + auto group_node = std::static_pointer_cast<::parquet::schema::GroupNode>(node); + int field_count = group_node->field_count(); + if (field_count == 0) { + return Status::Unknown("unknown type"); + } + + // 2nd level. + // field + std::vector field_names; + std::vector field_types; + field_names.reserve(field_count); + field_types.reserve(field_count); + for (auto i = 0; i < group_node->field_count(); ++i) { + const auto& field = group_node->field(i); + field_names.emplace_back(field->name()); + auto& field_type_desc = field_types.emplace_back(); + RETURN_IF_ERROR(get_parquet_type(field, &field_type_desc)); + } + + *type_desc = TypeDescriptor::create_struct_type(field_names, field_types); + + return Status::OK(); +} + } //namespace starrocks \ No newline at end of file diff --git a/be/test/exec/parquet_scanner_test.cpp b/be/test/exec/parquet_scanner_test.cpp index c5a8ff2d29f47..15b192c123251 100644 --- a/be/test/exec/parquet_scanner_test.cpp +++ b/be/test/exec/parquet_scanner_test.cpp @@ -701,16 +701,26 @@ TEST_F(ParquetScannerTest, get_file_schema) { {"col_json_map_timestamp", TypeDescriptor::create_map_type(TypeDescriptor::from_logical_type(TYPE_DATETIME), TypeDescriptor::from_logical_type(TYPE_INT))}, - {"col_json_struct", TypeDescriptor::create_varchar_type(1048576)}, + {"col_json_struct", + TypeDescriptor::create_struct_type({"s0", "s1"}, {TypeDescriptor::from_logical_type(TYPE_INT), + TypeDescriptor::create_varchar_type(1048576)})}, {"col_json_list_list", TypeDescriptor::create_array_type(TypeDescriptor::create_array_type( TypeDescriptor::from_logical_type(TYPE_INT)))}, {"col_json_map_list", TypeDescriptor::create_map_type( TypeDescriptor::create_varchar_type(1048576), TypeDescriptor::create_array_type(TypeDescriptor::from_logical_type(TYPE_INT)))}, - {"col_json_list_struct", TypeDescriptor::create_array_type(TypeDescriptor::create_varchar_type(1048576))}, - {"col_json_struct_struct", TypeDescriptor::create_varchar_type(1048576)}, - {"col_json_struct_string", TypeDescriptor::create_varchar_type(1048576)}, + {"col_json_list_struct", TypeDescriptor::create_array_type(TypeDescriptor::create_struct_type( + {"s0", "s1"}, {TypeDescriptor::from_logical_type(TYPE_INT), + TypeDescriptor::create_varchar_type(1048576)}))}, + {"col_json_struct_struct", + TypeDescriptor::create_struct_type( + {"s0", "s1"}, + {TypeDescriptor::from_logical_type(TYPE_INT), + TypeDescriptor::create_struct_type({"s2"}, {TypeDescriptor::from_logical_type(TYPE_INT)})})}, + {"col_json_struct_string", + TypeDescriptor::create_struct_type({"s0", "s1"}, {TypeDescriptor::from_logical_type(TYPE_INT), + TypeDescriptor::create_varchar_type(1048576)})}, {"col_json_json_string", TypeDescriptor::create_varchar_type(1048576)}}}, {test_exec_dir + "/test_data/parquet_data/decimal.parquet", {{"col_decimal32", TypeDescriptor::create_decimalv3_type(TYPE_DECIMAL32, 9, 2)}, diff --git a/test/sql/test_across_engine/R/test_sink_nested b/test/sql/test_across_engine/R/test_sink_nested new file mode 100644 index 0000000000000..3a693c0d84500 --- /dev/null +++ b/test/sql/test_across_engine/R/test_sink_nested @@ -0,0 +1,199 @@ +-- name: test_sink_nested_map +shell: ossutil64 cp ./sql/test_across_engine/data/nested_maps.parquet oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_maps.parquet > /dev/null +-- result: +0 + +-- !result +create external catalog hive_sink_${uuid0} PROPERTIES ( + "type"="hive", + "hive.metastore.uris"="${hive_metastore_uris}", + "aws.s3.access_key" = "${oss_ak}", + "aws.s3.secret_key" = "${oss_sk}", + "aws.s3.endpoint" = "${oss_endpoint}" +); +-- result: +-- !result +create database hive_sink_${uuid0}.hive_sink_db_${uuid0}; +-- result: +-- !result +create table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} properties ( + "file_format" = "parquet" +) +as select * from files ( + "path" = "oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_maps.parquet", + "format" = "parquet", + "aws.s3.access_key" = "${oss_ak}", + "aws.s3.secret_key" = "${oss_sk}", + "aws.s3.endpoint" = "${oss_endpoint}" +); +-- result: +-- !result +select * from hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +{"a":{1:1,2:0}} 1 1.0 +{"b":{1:1}} 1 1.0 +{"c":null} 1 1.0 +{"d":{}} 1 1.0 +{"e":{1:1}} 1 1.0 +{"f":{3:1,4:0,5:1}} 1 1.0 +-- !result +spark: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +{"a":{1:true,2:false}} 1 1.0 +{"b":{1:true}} 1 1.0 +{"c":null} 1 1.0 +{"d":{}} 1 1.0 +{"e":{1:true}} 1 1.0 +{"f":{3:true,4:false,5:true}} 1 1.0 +-- !result +hive: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +{"a":{1:true,2:false}} 1 1.0 +{"b":{1:true}} 1 1.0 +{"c":null} 1 1.0 +{"d":{}} 1 1.0 +{"e":{1:true}} 1 1.0 +{"f":{3:true,4:false,5:true}} 1 1.0 +-- !result +drop table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} force; +-- result: +-- !result +drop database hive_sink_${uuid0}.hive_sink_db_${uuid0}; +-- result: +-- !result +drop catalog hive_sink_${uuid0}; +-- result: +-- !result +shell: ossutil64 rm -rf oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_maps.parquet > /dev/null +-- result: +0 + +-- !result +-- name: test_sink_nested_list +shell: ossutil64 cp ./sql/test_across_engine/data/nested_lists.parquet oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_lists.parquet > /dev/null +-- result: +0 + +-- !result +create external catalog hive_sink_${uuid0} PROPERTIES ( + "type"="hive", + "hive.metastore.uris"="${hive_metastore_uris}", + "aws.s3.access_key" = "${oss_ak}", + "aws.s3.secret_key" = "${oss_sk}", + "aws.s3.endpoint" = "${oss_endpoint}" +); +-- result: +-- !result +create database hive_sink_${uuid0}.hive_sink_db_${uuid0}; +-- result: +-- !result +create table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} properties ( + "file_format" = "parquet" +) +as select * from files ( + "path" = "oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_lists.parquet", + "format" = "parquet", + "aws.s3.access_key" = "${oss_ak}", + "aws.s3.secret_key" = "${oss_sk}", + "aws.s3.endpoint" = "${oss_endpoint}" +); +-- result: +-- !result +select * from hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +[[["a","b"],["c"]],[null,["d"]]] 1 +[[["a","b"],["c","d"]],[null,["e"]]] 1 +[[["a","b"],["c","d"],["e"]],[null,["f"]]] 1 +-- !result +trino: select * from hive.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +[[['a', 'b'], ['c']], [None, ['d']]] 1 +[[['a', 'b'], ['c', 'd']], [None, ['e']]] 1 +[[['a', 'b'], ['c', 'd'], ['e']], [None, ['f']]] 1 +-- !result +spark: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +[[["a","b"],["c"]],[null,["d"]]] 1 +[[["a","b"],["c","d"]],[null,["e"]]] 1 +[[["a","b"],["c","d"],["e"]],[null,["f"]]] 1 +-- !result +hive: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +[[["a","b"],["c"]],[null,["d"]]] 1 +[[["a","b"],["c","d"]],[null,["e"]]] 1 +[[["a","b"],["c","d"],["e"]],[null,["f"]]] 1 +-- !result +drop table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} force; +-- result: +-- !result +drop database hive_sink_${uuid0}.hive_sink_db_${uuid0}; +-- result: +-- !result +drop catalog hive_sink_${uuid0}; +-- result: +-- !result +shell: ossutil64 rm -rf oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_lists.parquet > /dev/null +-- result: +0 + +-- !result +-- name: test_sink_nested_struct +shell: ossutil64 cp ./sql/test_across_engine/data/nested_structs.parquet oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_structs.parquet > /dev/null +-- result: +0 + +-- !result +create external catalog hive_sink_${uuid0} PROPERTIES ( + "type"="hive", + "hive.metastore.uris"="${hive_metastore_uris}", + "aws.s3.access_key" = "${oss_ak}", + "aws.s3.secret_key" = "${oss_sk}", + "aws.s3.endpoint" = "${oss_endpoint}" +); +-- result: +-- !result +create database hive_sink_${uuid0}.hive_sink_db_${uuid0}; +-- result: +-- !result +create table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} properties ( + "file_format" = "parquet" +) +as select * from files ( + "path" = "oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_structs.parquet", + "format" = "parquet", + "aws.s3.access_key" = "${oss_ak}", + "aws.s3.secret_key" = "${oss_sk}", + "aws.s3.endpoint" = "${oss_endpoint}" +); +-- result: +-- !result +select * from hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +{"min":190406409000602,"max":190407175004000,"mean":190406671229999,"count":495,"sum":94251302258849568,"variance":0} {"min":115,"max":742,"mean":416,"count":495,"sum":206195,"variance":10374} {"min":737,"max":32150509,"mean":2401239,"count":495,"sum":1188613496,"variance":12977533288261} {"min":1000,"max":35195000,"mean":3519838,"count":495,"sum":1742320297,"variance":24581100553044} {"min":0,"max":0,"mean":0,"count":0,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":0,"sum":0,"variance":0} {"min":0,"max":658.63,"mean":57.76452991452993,"count":351,"sum":20275.350000000006,"variance":6310.500499135526} {"min":0,"max":312.16,"mean":49.40901785714286,"count":336,"sum":16601.43,"variance":3214.842695450431} {"min":0,"max":74136,"mean":6162.133196721318,"count":366,"sum":2255340.7500000023,"variance":104255249.59826614} {"min":0,"max":523800,"mean":19484.146706586827,"count":334,"sum":6507705,"variance":3563198650.906335} {"min":2628,"max":16085,"mean":6698.8,"count":10,"sum":66988,"variance":28540252.400000002} {"min":288,"max":1985,"mean":1285,"count":4,"sum":5140,"variance":509875.3333333333} {"min":0,"max":523800,"mean":18907.566091954024,"count":348,"sum":6579833,"variance":3428378496.7881336} {"min":0,"max":1,"mean":0,"count":495,"sum":452,"variance":0} {"min":0,"max":64749.63000000001,"mean":4951.024888352274,"count":352,"sum":1742760.7607000005,"variance":81195383.98823886} {"min":0,"max":2.8000000000000003,"mean":0.14237550619122732,"count":352,"sum":50.11617817931202,"variance":0.07516922114035923} {"min":0,"max":658063,"mean":1991.3067846607655,"count":339,"sum":675052.9999999995,"variance":1277234044.0126908} {"min":0,"max":300.7,"mean":65.32364985163204,"count":337,"sum":22014.069999999996,"variance":3904.805190507992} {"min":0,"max":18169,"mean":1528.077097505669,"count":441,"sum":673882,"variance":6122348.621315204} {"min":0,"max":121968,"mean":6201.877049689864,"count":352,"sum":2183060.721490832,"variance":142631612.6463931} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":100,"mean":1.2345679012345678,"count":81,"sum":100,"variance":123.45679012345684} {"min":0,"max":4172.084000000002,"mean":42.29786138613863,"count":101,"sum":4272.084000000002,"variance":172355.84886194076} {"min":0,"max":1,"mean":0,"count":495,"sum":208,"variance":0} {"min":0,"max":1,"mean":0,"count":495,"sum":44,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":121968,"mean":8685.222814207653,"count":366,"sum":3178791.550000001,"variance":243347757.98270744} {"min":0,"max":121968,"mean":8841.174394454862,"count":364,"sum":3218187.4795815693,"variance":244563632.41811454} {"min":1,"max":1,"mean":1,"count":495,"sum":495,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} +-- !result +trino: select * from hive.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +(min: 190406409000602, max: 190407175004000, mean: 190406671229999, count: 495, sum: 94251302258849568, variance: 0) (min: 115, max: 742, mean: 416, count: 495, sum: 206195, variance: 10374) (min: 737, max: 32150509, mean: 2401239, count: 495, sum: 1188613496, variance: 12977533288261) (min: 1000, max: 35195000, mean: 3519838, count: 495, sum: 1742320297, variance: 24581100553044) (min: 0.0, max: 0.0, mean: 0.0, count: 0, sum: 0.0, variance: 0.0) (min: 0.0, max: 0.0, mean: 0.0, count: 0, sum: 0.0, variance: 0.0) (min: 0.0, max: 658.63, mean: 57.76452991452993, count: 351, sum: 20275.350000000006, variance: 6310.500499135526) (min: 0.0, max: 312.16, mean: 49.40901785714286, count: 336, sum: 16601.43, variance: 3214.842695450431) (min: 0.0, max: 74136.0, mean: 6162.133196721318, count: 366, sum: 2255340.7500000023, variance: 104255249.59826614) (min: 0.0, max: 523800.0, mean: 19484.146706586827, count: 334, sum: 6507705.0, variance: 3563198650.906335) (min: 2628.0, max: 16085.0, mean: 6698.8, count: 10, sum: 66988.0, variance: 28540252.400000002) (min: 288.0, max: 1985.0, mean: 1285.0, count: 4, sum: 5140.0, variance: 509875.3333333333) (min: 0.0, max: 523800.0, mean: 18907.566091954024, count: 348, sum: 6579833.0, variance: 3428378496.7881336) (min: 0, max: 1, mean: 0, count: 495, sum: 452, variance: 0) (min: 0.0, max: 64749.63000000001, mean: 4951.024888352274, count: 352, sum: 1742760.7607000005, variance: 81195383.98823886) (min: 0.0, max: 2.8000000000000003, mean: 0.14237550619122732, count: 352, sum: 50.11617817931202, variance: 0.07516922114035923) (min: 0.0, max: 658063.0, mean: 1991.3067846607655, count: 339, sum: 675052.9999999995, variance: 1277234044.0126908) (min: 0.0, max: 300.7, mean: 65.32364985163204, count: 337, sum: 22014.069999999996, variance: 3904.805190507992) (min: 0.0, max: 18169.0, mean: 1528.077097505669, count: 441, sum: 673882.0, variance: 6122348.621315204) (min: 0.0, max: 121968.0, mean: 6201.877049689864, count: 352, sum: 2183060.721490832, variance: 142631612.6463931) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0.0, max: 100.0, mean: 1.2345679012345678, count: 81, sum: 100.0, variance: 123.45679012345684) (min: 0.0, max: 4172.084000000002, mean: 42.29786138613863, count: 101, sum: 4272.084000000002, variance: 172355.84886194076) (min: 0, max: 1, mean: 0, count: 495, sum: 208, variance: 0) (min: 0, max: 1, mean: 0, count: 495, sum: 44, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0.0, max: 121968.0, mean: 8685.222814207653, count: 366, sum: 3178791.550000001, variance: 243347757.98270744) (min: 0.0, max: 121968.0, mean: 8841.174394454862, count: 364, sum: 3218187.4795815693, variance: 244563632.41811454) (min: 1, max: 1, mean: 1, count: 495, sum: 495, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) +-- !result +spark: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +{"min":190406409000602,"max":190407175004000,"mean":190406671229999,"count":495,"sum":94251302258849568,"variance":0} {"min":115,"max":742,"mean":416,"count":495,"sum":206195,"variance":10374} {"min":737,"max":32150509,"mean":2401239,"count":495,"sum":1188613496,"variance":12977533288261} {"min":1000,"max":35195000,"mean":3519838,"count":495,"sum":1742320297,"variance":24581100553044} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":658.63,"mean":57.76452991452993,"count":351,"sum":20275.350000000006,"variance":6310.500499135526} {"min":0.0,"max":312.16,"mean":49.40901785714286,"count":336,"sum":16601.43,"variance":3214.842695450431} {"min":0.0,"max":74136.0,"mean":6162.133196721318,"count":366,"sum":2255340.7500000023,"variance":1.0425524959826614E8} {"min":0.0,"max":523800.0,"mean":19484.146706586827,"count":334,"sum":6507705.0,"variance":3.563198650906335E9} {"min":2628.0,"max":16085.0,"mean":6698.8,"count":10,"sum":66988.0,"variance":2.8540252400000002E7} {"min":288.0,"max":1985.0,"mean":1285.0,"count":4,"sum":5140.0,"variance":509875.3333333333} {"min":0.0,"max":523800.0,"mean":18907.566091954024,"count":348,"sum":6579833.0,"variance":3.4283784967881336E9} {"min":0,"max":1,"mean":0,"count":495,"sum":452,"variance":0} {"min":0.0,"max":64749.63000000001,"mean":4951.024888352274,"count":352,"sum":1742760.7607000005,"variance":8.119538398823886E7} {"min":0.0,"max":2.8000000000000003,"mean":0.14237550619122732,"count":352,"sum":50.11617817931202,"variance":0.07516922114035923} {"min":0.0,"max":658063.0,"mean":1991.3067846607655,"count":339,"sum":675052.9999999995,"variance":1.2772340440126908E9} {"min":0.0,"max":300.7,"mean":65.32364985163204,"count":337,"sum":22014.069999999996,"variance":3904.805190507992} {"min":0.0,"max":18169.0,"mean":1528.077097505669,"count":441,"sum":673882.0,"variance":6122348.621315204} {"min":0.0,"max":121968.0,"mean":6201.877049689864,"count":352,"sum":2183060.721490832,"variance":1.426316126463931E8} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":100.0,"mean":1.2345679012345678,"count":81,"sum":100.0,"variance":123.45679012345684} {"min":0.0,"max":4172.084000000002,"mean":42.29786138613863,"count":101,"sum":4272.084000000002,"variance":172355.84886194076} {"min":0,"max":1,"mean":0,"count":495,"sum":208,"variance":0} {"min":0,"max":1,"mean":0,"count":495,"sum":44,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":121968.0,"mean":8685.222814207653,"count":366,"sum":3178791.550000001,"variance":2.4334775798270744E8} {"min":0.0,"max":121968.0,"mean":8841.174394454862,"count":364,"sum":3218187.4795815693,"variance":2.4456363241811454E8} {"min":1,"max":1,"mean":1,"count":495,"sum":495,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} +-- !result +hive: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; +-- result: +{"min":190406409000602,"max":190407175004000,"mean":190406671229999,"count":495,"sum":94251302258849568,"variance":0} {"min":115,"max":742,"mean":416,"count":495,"sum":206195,"variance":10374} {"min":737,"max":32150509,"mean":2401239,"count":495,"sum":1188613496,"variance":12977533288261} {"min":1000,"max":35195000,"mean":3519838,"count":495,"sum":1742320297,"variance":24581100553044} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":658.63,"mean":57.76452991452993,"count":351,"sum":20275.350000000006,"variance":6310.500499135526} {"min":0.0,"max":312.16,"mean":49.40901785714286,"count":336,"sum":16601.43,"variance":3214.842695450431} {"min":0.0,"max":74136.0,"mean":6162.133196721318,"count":366,"sum":2255340.7500000023,"variance":1.0425524959826614E8} {"min":0.0,"max":523800.0,"mean":19484.146706586827,"count":334,"sum":6507705.0,"variance":3.563198650906335E9} {"min":2628.0,"max":16085.0,"mean":6698.8,"count":10,"sum":66988.0,"variance":2.8540252400000002E7} {"min":288.0,"max":1985.0,"mean":1285.0,"count":4,"sum":5140.0,"variance":509875.3333333333} {"min":0.0,"max":523800.0,"mean":18907.566091954024,"count":348,"sum":6579833.0,"variance":3.4283784967881336E9} {"min":0,"max":1,"mean":0,"count":495,"sum":452,"variance":0} {"min":0.0,"max":64749.63000000001,"mean":4951.024888352274,"count":352,"sum":1742760.7607000005,"variance":8.119538398823886E7} {"min":0.0,"max":2.8000000000000003,"mean":0.14237550619122732,"count":352,"sum":50.11617817931202,"variance":0.07516922114035923} {"min":0.0,"max":658063.0,"mean":1991.3067846607655,"count":339,"sum":675052.9999999995,"variance":1.2772340440126908E9} {"min":0.0,"max":300.7,"mean":65.32364985163204,"count":337,"sum":22014.069999999996,"variance":3904.805190507992} {"min":0.0,"max":18169.0,"mean":1528.077097505669,"count":441,"sum":673882.0,"variance":6122348.621315204} {"min":0.0,"max":121968.0,"mean":6201.877049689864,"count":352,"sum":2183060.721490832,"variance":1.426316126463931E8} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":100.0,"mean":1.2345679012345678,"count":81,"sum":100.0,"variance":123.45679012345684} {"min":0.0,"max":4172.084000000002,"mean":42.29786138613863,"count":101,"sum":4272.084000000002,"variance":172355.84886194076} {"min":0,"max":1,"mean":0,"count":495,"sum":208,"variance":0} {"min":0,"max":1,"mean":0,"count":495,"sum":44,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":121968.0,"mean":8685.222814207653,"count":366,"sum":3178791.550000001,"variance":2.4334775798270744E8} {"min":0.0,"max":121968.0,"mean":8841.174394454862,"count":364,"sum":3218187.4795815693,"variance":2.4456363241811454E8} {"min":1,"max":1,"mean":1,"count":495,"sum":495,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} +-- !result +drop table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} force; +-- result: +-- !result +drop database hive_sink_${uuid0}.hive_sink_db_${uuid0}; +-- result: +-- !result +drop catalog hive_sink_${uuid0}; +-- result: +-- !result +shell: ossutil64 rm -rf oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_structs.parquet > /dev/null +-- result: +0 + +-- !result diff --git a/test/sql/test_pipe/R/basic b/test/sql/test_pipe/R/basic index 0a76048cb687e..3eb55c789dd23 100644 --- a/test/sql/test_pipe/R/basic +++ b/test/sql/test_pipe/R/basic @@ -33,7 +33,7 @@ desc t1; -- result: col_int int YES true None col_map map YES false None -col_struct varchar(1048576) YES false None +col_struct struct YES false None -- !result select count(*) from t1; -- result: From b3a4980b4c6032918bc9af5be6fdafddd55f3ca0 Mon Sep 17 00:00:00 2001 From: wyb Date: Mon, 2 Sep 2024 11:53:09 +0000 Subject: [PATCH 2/2] Fix conflict --- .../sql/test_across_engine/R/test_sink_nested | 199 ------------------ 1 file changed, 199 deletions(-) delete mode 100644 test/sql/test_across_engine/R/test_sink_nested diff --git a/test/sql/test_across_engine/R/test_sink_nested b/test/sql/test_across_engine/R/test_sink_nested deleted file mode 100644 index 3a693c0d84500..0000000000000 --- a/test/sql/test_across_engine/R/test_sink_nested +++ /dev/null @@ -1,199 +0,0 @@ --- name: test_sink_nested_map -shell: ossutil64 cp ./sql/test_across_engine/data/nested_maps.parquet oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_maps.parquet > /dev/null --- result: -0 - --- !result -create external catalog hive_sink_${uuid0} PROPERTIES ( - "type"="hive", - "hive.metastore.uris"="${hive_metastore_uris}", - "aws.s3.access_key" = "${oss_ak}", - "aws.s3.secret_key" = "${oss_sk}", - "aws.s3.endpoint" = "${oss_endpoint}" -); --- result: --- !result -create database hive_sink_${uuid0}.hive_sink_db_${uuid0}; --- result: --- !result -create table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} properties ( - "file_format" = "parquet" -) -as select * from files ( - "path" = "oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_maps.parquet", - "format" = "parquet", - "aws.s3.access_key" = "${oss_ak}", - "aws.s3.secret_key" = "${oss_sk}", - "aws.s3.endpoint" = "${oss_endpoint}" -); --- result: --- !result -select * from hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -{"a":{1:1,2:0}} 1 1.0 -{"b":{1:1}} 1 1.0 -{"c":null} 1 1.0 -{"d":{}} 1 1.0 -{"e":{1:1}} 1 1.0 -{"f":{3:1,4:0,5:1}} 1 1.0 --- !result -spark: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -{"a":{1:true,2:false}} 1 1.0 -{"b":{1:true}} 1 1.0 -{"c":null} 1 1.0 -{"d":{}} 1 1.0 -{"e":{1:true}} 1 1.0 -{"f":{3:true,4:false,5:true}} 1 1.0 --- !result -hive: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -{"a":{1:true,2:false}} 1 1.0 -{"b":{1:true}} 1 1.0 -{"c":null} 1 1.0 -{"d":{}} 1 1.0 -{"e":{1:true}} 1 1.0 -{"f":{3:true,4:false,5:true}} 1 1.0 --- !result -drop table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} force; --- result: --- !result -drop database hive_sink_${uuid0}.hive_sink_db_${uuid0}; --- result: --- !result -drop catalog hive_sink_${uuid0}; --- result: --- !result -shell: ossutil64 rm -rf oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_maps.parquet > /dev/null --- result: -0 - --- !result --- name: test_sink_nested_list -shell: ossutil64 cp ./sql/test_across_engine/data/nested_lists.parquet oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_lists.parquet > /dev/null --- result: -0 - --- !result -create external catalog hive_sink_${uuid0} PROPERTIES ( - "type"="hive", - "hive.metastore.uris"="${hive_metastore_uris}", - "aws.s3.access_key" = "${oss_ak}", - "aws.s3.secret_key" = "${oss_sk}", - "aws.s3.endpoint" = "${oss_endpoint}" -); --- result: --- !result -create database hive_sink_${uuid0}.hive_sink_db_${uuid0}; --- result: --- !result -create table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} properties ( - "file_format" = "parquet" -) -as select * from files ( - "path" = "oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_lists.parquet", - "format" = "parquet", - "aws.s3.access_key" = "${oss_ak}", - "aws.s3.secret_key" = "${oss_sk}", - "aws.s3.endpoint" = "${oss_endpoint}" -); --- result: --- !result -select * from hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -[[["a","b"],["c"]],[null,["d"]]] 1 -[[["a","b"],["c","d"]],[null,["e"]]] 1 -[[["a","b"],["c","d"],["e"]],[null,["f"]]] 1 --- !result -trino: select * from hive.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -[[['a', 'b'], ['c']], [None, ['d']]] 1 -[[['a', 'b'], ['c', 'd']], [None, ['e']]] 1 -[[['a', 'b'], ['c', 'd'], ['e']], [None, ['f']]] 1 --- !result -spark: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -[[["a","b"],["c"]],[null,["d"]]] 1 -[[["a","b"],["c","d"]],[null,["e"]]] 1 -[[["a","b"],["c","d"],["e"]],[null,["f"]]] 1 --- !result -hive: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -[[["a","b"],["c"]],[null,["d"]]] 1 -[[["a","b"],["c","d"]],[null,["e"]]] 1 -[[["a","b"],["c","d"],["e"]],[null,["f"]]] 1 --- !result -drop table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} force; --- result: --- !result -drop database hive_sink_${uuid0}.hive_sink_db_${uuid0}; --- result: --- !result -drop catalog hive_sink_${uuid0}; --- result: --- !result -shell: ossutil64 rm -rf oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_lists.parquet > /dev/null --- result: -0 - --- !result --- name: test_sink_nested_struct -shell: ossutil64 cp ./sql/test_across_engine/data/nested_structs.parquet oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_structs.parquet > /dev/null --- result: -0 - --- !result -create external catalog hive_sink_${uuid0} PROPERTIES ( - "type"="hive", - "hive.metastore.uris"="${hive_metastore_uris}", - "aws.s3.access_key" = "${oss_ak}", - "aws.s3.secret_key" = "${oss_sk}", - "aws.s3.endpoint" = "${oss_endpoint}" -); --- result: --- !result -create database hive_sink_${uuid0}.hive_sink_db_${uuid0}; --- result: --- !result -create table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} properties ( - "file_format" = "parquet" -) -as select * from files ( - "path" = "oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_structs.parquet", - "format" = "parquet", - "aws.s3.access_key" = "${oss_ak}", - "aws.s3.secret_key" = "${oss_sk}", - "aws.s3.endpoint" = "${oss_endpoint}" -); --- result: --- !result -select * from hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -{"min":190406409000602,"max":190407175004000,"mean":190406671229999,"count":495,"sum":94251302258849568,"variance":0} {"min":115,"max":742,"mean":416,"count":495,"sum":206195,"variance":10374} {"min":737,"max":32150509,"mean":2401239,"count":495,"sum":1188613496,"variance":12977533288261} {"min":1000,"max":35195000,"mean":3519838,"count":495,"sum":1742320297,"variance":24581100553044} {"min":0,"max":0,"mean":0,"count":0,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":0,"sum":0,"variance":0} {"min":0,"max":658.63,"mean":57.76452991452993,"count":351,"sum":20275.350000000006,"variance":6310.500499135526} {"min":0,"max":312.16,"mean":49.40901785714286,"count":336,"sum":16601.43,"variance":3214.842695450431} {"min":0,"max":74136,"mean":6162.133196721318,"count":366,"sum":2255340.7500000023,"variance":104255249.59826614} {"min":0,"max":523800,"mean":19484.146706586827,"count":334,"sum":6507705,"variance":3563198650.906335} {"min":2628,"max":16085,"mean":6698.8,"count":10,"sum":66988,"variance":28540252.400000002} {"min":288,"max":1985,"mean":1285,"count":4,"sum":5140,"variance":509875.3333333333} {"min":0,"max":523800,"mean":18907.566091954024,"count":348,"sum":6579833,"variance":3428378496.7881336} {"min":0,"max":1,"mean":0,"count":495,"sum":452,"variance":0} {"min":0,"max":64749.63000000001,"mean":4951.024888352274,"count":352,"sum":1742760.7607000005,"variance":81195383.98823886} {"min":0,"max":2.8000000000000003,"mean":0.14237550619122732,"count":352,"sum":50.11617817931202,"variance":0.07516922114035923} {"min":0,"max":658063,"mean":1991.3067846607655,"count":339,"sum":675052.9999999995,"variance":1277234044.0126908} {"min":0,"max":300.7,"mean":65.32364985163204,"count":337,"sum":22014.069999999996,"variance":3904.805190507992} {"min":0,"max":18169,"mean":1528.077097505669,"count":441,"sum":673882,"variance":6122348.621315204} {"min":0,"max":121968,"mean":6201.877049689864,"count":352,"sum":2183060.721490832,"variance":142631612.6463931} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":100,"mean":1.2345679012345678,"count":81,"sum":100,"variance":123.45679012345684} {"min":0,"max":4172.084000000002,"mean":42.29786138613863,"count":101,"sum":4272.084000000002,"variance":172355.84886194076} {"min":0,"max":1,"mean":0,"count":495,"sum":208,"variance":0} {"min":0,"max":1,"mean":0,"count":495,"sum":44,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":121968,"mean":8685.222814207653,"count":366,"sum":3178791.550000001,"variance":243347757.98270744} {"min":0,"max":121968,"mean":8841.174394454862,"count":364,"sum":3218187.4795815693,"variance":244563632.41811454} {"min":1,"max":1,"mean":1,"count":495,"sum":495,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} --- !result -trino: select * from hive.hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -(min: 190406409000602, max: 190407175004000, mean: 190406671229999, count: 495, sum: 94251302258849568, variance: 0) (min: 115, max: 742, mean: 416, count: 495, sum: 206195, variance: 10374) (min: 737, max: 32150509, mean: 2401239, count: 495, sum: 1188613496, variance: 12977533288261) (min: 1000, max: 35195000, mean: 3519838, count: 495, sum: 1742320297, variance: 24581100553044) (min: 0.0, max: 0.0, mean: 0.0, count: 0, sum: 0.0, variance: 0.0) (min: 0.0, max: 0.0, mean: 0.0, count: 0, sum: 0.0, variance: 0.0) (min: 0.0, max: 658.63, mean: 57.76452991452993, count: 351, sum: 20275.350000000006, variance: 6310.500499135526) (min: 0.0, max: 312.16, mean: 49.40901785714286, count: 336, sum: 16601.43, variance: 3214.842695450431) (min: 0.0, max: 74136.0, mean: 6162.133196721318, count: 366, sum: 2255340.7500000023, variance: 104255249.59826614) (min: 0.0, max: 523800.0, mean: 19484.146706586827, count: 334, sum: 6507705.0, variance: 3563198650.906335) (min: 2628.0, max: 16085.0, mean: 6698.8, count: 10, sum: 66988.0, variance: 28540252.400000002) (min: 288.0, max: 1985.0, mean: 1285.0, count: 4, sum: 5140.0, variance: 509875.3333333333) (min: 0.0, max: 523800.0, mean: 18907.566091954024, count: 348, sum: 6579833.0, variance: 3428378496.7881336) (min: 0, max: 1, mean: 0, count: 495, sum: 452, variance: 0) (min: 0.0, max: 64749.63000000001, mean: 4951.024888352274, count: 352, sum: 1742760.7607000005, variance: 81195383.98823886) (min: 0.0, max: 2.8000000000000003, mean: 0.14237550619122732, count: 352, sum: 50.11617817931202, variance: 0.07516922114035923) (min: 0.0, max: 658063.0, mean: 1991.3067846607655, count: 339, sum: 675052.9999999995, variance: 1277234044.0126908) (min: 0.0, max: 300.7, mean: 65.32364985163204, count: 337, sum: 22014.069999999996, variance: 3904.805190507992) (min: 0.0, max: 18169.0, mean: 1528.077097505669, count: 441, sum: 673882.0, variance: 6122348.621315204) (min: 0.0, max: 121968.0, mean: 6201.877049689864, count: 352, sum: 2183060.721490832, variance: 142631612.6463931) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0.0, max: 100.0, mean: 1.2345679012345678, count: 81, sum: 100.0, variance: 123.45679012345684) (min: 0.0, max: 4172.084000000002, mean: 42.29786138613863, count: 101, sum: 4272.084000000002, variance: 172355.84886194076) (min: 0, max: 1, mean: 0, count: 495, sum: 208, variance: 0) (min: 0, max: 1, mean: 0, count: 495, sum: 44, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) (min: 0.0, max: 121968.0, mean: 8685.222814207653, count: 366, sum: 3178791.550000001, variance: 243347757.98270744) (min: 0.0, max: 121968.0, mean: 8841.174394454862, count: 364, sum: 3218187.4795815693, variance: 244563632.41811454) (min: 1, max: 1, mean: 1, count: 495, sum: 495, variance: 0) (min: 0, max: 0, mean: 0, count: 495, sum: 0, variance: 0) --- !result -spark: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -{"min":190406409000602,"max":190407175004000,"mean":190406671229999,"count":495,"sum":94251302258849568,"variance":0} {"min":115,"max":742,"mean":416,"count":495,"sum":206195,"variance":10374} {"min":737,"max":32150509,"mean":2401239,"count":495,"sum":1188613496,"variance":12977533288261} {"min":1000,"max":35195000,"mean":3519838,"count":495,"sum":1742320297,"variance":24581100553044} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":658.63,"mean":57.76452991452993,"count":351,"sum":20275.350000000006,"variance":6310.500499135526} {"min":0.0,"max":312.16,"mean":49.40901785714286,"count":336,"sum":16601.43,"variance":3214.842695450431} {"min":0.0,"max":74136.0,"mean":6162.133196721318,"count":366,"sum":2255340.7500000023,"variance":1.0425524959826614E8} {"min":0.0,"max":523800.0,"mean":19484.146706586827,"count":334,"sum":6507705.0,"variance":3.563198650906335E9} {"min":2628.0,"max":16085.0,"mean":6698.8,"count":10,"sum":66988.0,"variance":2.8540252400000002E7} {"min":288.0,"max":1985.0,"mean":1285.0,"count":4,"sum":5140.0,"variance":509875.3333333333} {"min":0.0,"max":523800.0,"mean":18907.566091954024,"count":348,"sum":6579833.0,"variance":3.4283784967881336E9} {"min":0,"max":1,"mean":0,"count":495,"sum":452,"variance":0} {"min":0.0,"max":64749.63000000001,"mean":4951.024888352274,"count":352,"sum":1742760.7607000005,"variance":8.119538398823886E7} {"min":0.0,"max":2.8000000000000003,"mean":0.14237550619122732,"count":352,"sum":50.11617817931202,"variance":0.07516922114035923} {"min":0.0,"max":658063.0,"mean":1991.3067846607655,"count":339,"sum":675052.9999999995,"variance":1.2772340440126908E9} {"min":0.0,"max":300.7,"mean":65.32364985163204,"count":337,"sum":22014.069999999996,"variance":3904.805190507992} {"min":0.0,"max":18169.0,"mean":1528.077097505669,"count":441,"sum":673882.0,"variance":6122348.621315204} {"min":0.0,"max":121968.0,"mean":6201.877049689864,"count":352,"sum":2183060.721490832,"variance":1.426316126463931E8} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":100.0,"mean":1.2345679012345678,"count":81,"sum":100.0,"variance":123.45679012345684} {"min":0.0,"max":4172.084000000002,"mean":42.29786138613863,"count":101,"sum":4272.084000000002,"variance":172355.84886194076} {"min":0,"max":1,"mean":0,"count":495,"sum":208,"variance":0} {"min":0,"max":1,"mean":0,"count":495,"sum":44,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":121968.0,"mean":8685.222814207653,"count":366,"sum":3178791.550000001,"variance":2.4334775798270744E8} {"min":0.0,"max":121968.0,"mean":8841.174394454862,"count":364,"sum":3218187.4795815693,"variance":2.4456363241811454E8} {"min":1,"max":1,"mean":1,"count":495,"sum":495,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} --- !result -hive: select * from hive_sink_db_${uuid0}.hive_sink_table_${uuid0}; --- result: -{"min":190406409000602,"max":190407175004000,"mean":190406671229999,"count":495,"sum":94251302258849568,"variance":0} {"min":115,"max":742,"mean":416,"count":495,"sum":206195,"variance":10374} {"min":737,"max":32150509,"mean":2401239,"count":495,"sum":1188613496,"variance":12977533288261} {"min":1000,"max":35195000,"mean":3519838,"count":495,"sum":1742320297,"variance":24581100553044} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":0.0,"mean":0.0,"count":0,"sum":0.0,"variance":0.0} {"min":0.0,"max":658.63,"mean":57.76452991452993,"count":351,"sum":20275.350000000006,"variance":6310.500499135526} {"min":0.0,"max":312.16,"mean":49.40901785714286,"count":336,"sum":16601.43,"variance":3214.842695450431} {"min":0.0,"max":74136.0,"mean":6162.133196721318,"count":366,"sum":2255340.7500000023,"variance":1.0425524959826614E8} {"min":0.0,"max":523800.0,"mean":19484.146706586827,"count":334,"sum":6507705.0,"variance":3.563198650906335E9} {"min":2628.0,"max":16085.0,"mean":6698.8,"count":10,"sum":66988.0,"variance":2.8540252400000002E7} {"min":288.0,"max":1985.0,"mean":1285.0,"count":4,"sum":5140.0,"variance":509875.3333333333} {"min":0.0,"max":523800.0,"mean":18907.566091954024,"count":348,"sum":6579833.0,"variance":3.4283784967881336E9} {"min":0,"max":1,"mean":0,"count":495,"sum":452,"variance":0} {"min":0.0,"max":64749.63000000001,"mean":4951.024888352274,"count":352,"sum":1742760.7607000005,"variance":8.119538398823886E7} {"min":0.0,"max":2.8000000000000003,"mean":0.14237550619122732,"count":352,"sum":50.11617817931202,"variance":0.07516922114035923} {"min":0.0,"max":658063.0,"mean":1991.3067846607655,"count":339,"sum":675052.9999999995,"variance":1.2772340440126908E9} {"min":0.0,"max":300.7,"mean":65.32364985163204,"count":337,"sum":22014.069999999996,"variance":3904.805190507992} {"min":0.0,"max":18169.0,"mean":1528.077097505669,"count":441,"sum":673882.0,"variance":6122348.621315204} {"min":0.0,"max":121968.0,"mean":6201.877049689864,"count":352,"sum":2183060.721490832,"variance":1.426316126463931E8} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":100.0,"mean":1.2345679012345678,"count":81,"sum":100.0,"variance":123.45679012345684} {"min":0.0,"max":4172.084000000002,"mean":42.29786138613863,"count":101,"sum":4272.084000000002,"variance":172355.84886194076} {"min":0,"max":1,"mean":0,"count":495,"sum":208,"variance":0} {"min":0,"max":1,"mean":0,"count":495,"sum":44,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} {"min":0.0,"max":121968.0,"mean":8685.222814207653,"count":366,"sum":3178791.550000001,"variance":2.4334775798270744E8} {"min":0.0,"max":121968.0,"mean":8841.174394454862,"count":364,"sum":3218187.4795815693,"variance":2.4456363241811454E8} {"min":1,"max":1,"mean":1,"count":495,"sum":495,"variance":0} {"min":0,"max":0,"mean":0,"count":495,"sum":0,"variance":0} --- !result -drop table hive_sink_${uuid0}.hive_sink_db_${uuid0}.hive_sink_table_${uuid0} force; --- result: --- !result -drop database hive_sink_${uuid0}.hive_sink_db_${uuid0}; --- result: --- !result -drop catalog hive_sink_${uuid0}; --- result: --- !result -shell: ossutil64 rm -rf oss://${oss_bucket}/test_across_engine/test_sink/${uuid0}/nested_structs.parquet > /dev/null --- result: -0 - --- !result