From b1f1e9699bd72af34c776fe77d88614a232ac11e Mon Sep 17 00:00:00 2001 From: Mryange Date: Tue, 24 Dec 2024 16:03:46 +0800 Subject: [PATCH] upd --- be/src/runtime/exec_env.h | 4 + be/src/runtime/exec_env_init.cpp | 3 + be/src/vec/columns/column_string.h | 4 + be/src/vec/functions/dictionary.cpp | 55 +++++ be/src/vec/functions/dictionary.h | 100 ++++++++ be/src/vec/functions/dictionary_factory.cpp | 182 ++++++++++++++ be/src/vec/functions/dictionary_factory.h | 62 +++++ be/src/vec/functions/function_dict_get.cpp | 215 +++++++++++++++++ be/src/vec/functions/hash_map_dictionary.h | 200 +++++++++++++++ .../vec/functions/ip_address_dictionary.cpp | 228 ++++++++++++++++++ be/src/vec/functions/ip_address_dictionary.h | 99 ++++++++ .../vec/functions/simple_function_factory.h | 3 +- be/src/vec/runtime/ip_address_cidr.h | 8 +- build.sh | 2 +- .../doris/catalog/BuiltinScalarFunctions.java | 18 +- .../expressions/functions/scalar/DictGet.java | 78 ++++++ .../functions/scalar/DictGetFloat32.java | 71 ++++++ .../functions/scalar/DictGetIPv4.java | 71 ++++++ .../functions/scalar/DictGetInt32.java | 71 ++++++ .../functions/scalar/DictGetInt64.java | 74 ++++++ .../functions/scalar/DictGetString.java | 69 ++++++ .../data/correctness/test_dict_function.out | 70 ++++++ .../correctness/test_dict_function.groovy | 94 ++++++++ 23 files changed, 1772 insertions(+), 9 deletions(-) create mode 100644 be/src/vec/functions/dictionary.cpp create mode 100644 be/src/vec/functions/dictionary.h create mode 100644 be/src/vec/functions/dictionary_factory.cpp create mode 100644 be/src/vec/functions/dictionary_factory.h create mode 100644 be/src/vec/functions/function_dict_get.cpp create mode 100644 be/src/vec/functions/hash_map_dictionary.h create mode 100644 be/src/vec/functions/ip_address_dictionary.cpp create mode 100644 be/src/vec/functions/ip_address_dictionary.h create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetFloat32.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetIPv4.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt32.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt64.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetString.java create mode 100644 regression-test/data/correctness/test_dict_function.out create mode 100644 regression-test/suites/correctness/test_dict_function.groovy diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 636ce2bf288b58a..c1c457472bbf7e6 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -50,6 +50,7 @@ class VDataStreamMgr; class ScannerScheduler; class SpillStreamManager; class DeltaWriterV2Pool; +class DictionaryFactory; } // namespace vectorized namespace pipeline { class TaskScheduler; @@ -337,6 +338,8 @@ class ExecEnv { return _runtime_filter_timer_queue; } + vectorized::DictionaryFactory* dict_factory() { return _dict_factory; } + pipeline::PipelineTracerContext* pipeline_tracer_context() { return _pipeline_tracer_ctx.get(); } @@ -477,6 +480,7 @@ class ExecEnv { std::unique_ptr _file_cache_open_fd_cache; pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; + vectorized::DictionaryFactory* _dict_factory = nullptr; WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 2d7554e702969f6..6f3591d6d6bc490 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -108,6 +108,7 @@ #include "vec/exec/format/orc/orc_memory_pool.h" #include "vec/exec/format/parquet/arrow_memory_pool.h" #include "vec/exec/scan/scanner_scheduler.h" +#include "vec/functions/dictionary_factory.h" #include "vec/runtime/vdata_stream_mgr.h" #include "vec/sink/delta_writer_v2_pool.h" #include "vec/sink/load_stream_map_pool.h" @@ -376,6 +377,7 @@ Status ExecEnv::_init(const std::vector& store_paths, RETURN_IF_ERROR(_spill_stream_mgr->init()); _runtime_query_statistics_mgr->start_report_thread(); + _dict_factory = new doris::vectorized::DictionaryFactory(); _s_ready = true; return Status::OK(); @@ -776,6 +778,7 @@ void ExecEnv::destroy() { SAFE_DELETE(_workload_group_manager); SAFE_DELETE(_file_cache_factory); SAFE_DELETE(_runtime_filter_timer_queue); + SAFE_DELETE(_dict_factory); // TODO(zhiqiang): Maybe we should call shutdown before release thread pool? _lazy_release_obj_pool.reset(nullptr); _non_block_close_thread_pool.reset(nullptr); diff --git a/be/src/vec/columns/column_string.h b/be/src/vec/columns/column_string.h index 1674fd90933dbe1..e6e89928a1655b4 100644 --- a/be/src/vec/columns/column_string.h +++ b/be/src/vec/columns/column_string.h @@ -141,6 +141,10 @@ class ColumnStr final : public COWHelper> { return StringRef(&chars[offset_at(n)], size_at(n)); } + String get_element(size_t n) const { return get_data_at(n).to_string(); } + + void insert_value(const String& value) { insert_data(value.data(), value.size()); } + void insert(const Field& x) override { StringRef s; if (x.get_type() == Field::Types::JSONB) { diff --git a/be/src/vec/functions/dictionary.cpp b/be/src/vec/functions/dictionary.cpp new file mode 100644 index 000000000000000..2e507852cff8532 --- /dev/null +++ b/be/src/vec/functions/dictionary.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/functions/dictionary.h" + +namespace doris::vectorized { + +IDictionary::IDictionary(std::string name, std::vector attributes) + : _dict_name(std::move(name)), _attributes(std::move(attributes)) { + for (size_t i = 0; i < _attributes.size(); i++) { + const auto& name = _attributes[i].name; + if (_name_to_attributes_index.contains(name)) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, + "The names of attributes should not have duplicates : {}", name); + } + _name_to_attributes_index[name] = i; + } +} + +bool IDictionary::has_attribute(const std::string& name) const { + return _name_to_attributes_index.contains(name); +} + +size_t IDictionary::attribute_index(const std::string& name) const { + auto it = _name_to_attributes_index.find(name); + if (it == _name_to_attributes_index.end()) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "no this attribute : {}", name); + } + size_t idx = it->second; + return idx; +} + +DataTypePtr IDictionary::get_attribute_type(const std::string& name) const { + auto it = _name_to_attributes_index.find(name); + if (it == _name_to_attributes_index.end()) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "no this attribute : {}", name); + } + size_t idx = it->second; + return _attributes[idx].type; +} +} // namespace doris::vectorized diff --git a/be/src/vec/functions/dictionary.h b/be/src/vec/functions/dictionary.h new file mode 100644 index 000000000000000..32b6d92cd925fda --- /dev/null +++ b/be/src/vec/functions/dictionary.h @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "vec/columns/column.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_date_time.h" +#include "vec/data_types/data_type_ipv4.h" +#include "vec/data_types/data_type_ipv6.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_time_v2.h" +#include "vec/functions/cast_type_to_either.h" + +namespace doris::vectorized { +struct DictionaryAttribute { + const std::string name; + const DataTypePtr type; +}; + +class IDictionary { +public: + IDictionary(std::string name, std::vector attributes); + virtual ~IDictionary() = default; + std::string dict_name() const { return _dict_name; } + + virtual ColumnPtr getColumn(const std::string& attribute_name, + const DataTypePtr& attribute_type, const ColumnPtr& key_column, + const DataTypePtr& key_type) const = 0; + + bool has_attribute(const std::string& name) const; + DataTypePtr get_attribute_type(const std::string& name) const; + size_t attribute_index(const std::string& name) const; + + template + static bool cast_type(const IDataType* type, F&& f) { + return cast_type_to_either, + DataTypeDecimal, DataTypeDecimal, + DataTypeDecimal>(type, std::forward(f)); + } + +protected: + template + struct ColumnWithType { + using DataType = Type; + DataType::ColumnType::Ptr column; + }; + using ColumnData = + std::variant, ColumnWithType, + ColumnWithType, ColumnWithType, + ColumnWithType, ColumnWithType, + + ColumnWithType, ColumnWithType, + + ColumnWithType, ColumnWithType, + + ColumnWithType, + + ColumnWithType, ColumnWithType, + + ColumnWithType>, + ColumnWithType>, + ColumnWithType>, + ColumnWithType>>; + + const std::string _dict_name; + std::vector _attributes; + std::unordered_map _name_to_attributes_index; +}; + +using DictionaryPtr = std::shared_ptr; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/dictionary_factory.cpp b/be/src/vec/functions/dictionary_factory.cpp new file mode 100644 index 000000000000000..5bc0c4fd129d5a3 --- /dev/null +++ b/be/src/vec/functions/dictionary_factory.cpp @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/functions/dictionary_factory.h" + +#include + +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/thread_context.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/columns_number.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_time_v2.h" +#include "vec/functions/hash_map_dictionary.h" +#include "vec/functions/ip_address_dictionary.h" +namespace doris::vectorized { + +template +ColumnWithTypeAndName create_with_type(const std::string& name) { + return ColumnWithTypeAndName {DataType::ColumnType::create(), std::make_shared(), + name}; +} + +DictionaryFactory::DictionaryFactory() + : _mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, + "GLOBAL_DICT_FACTORY")) { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); + register_dict_when_load_be(); +} + +DictionaryFactory::~DictionaryFactory() { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); + _dict_map.clear(); +} + +void DictionaryFactory::register_dict_when_load_be() { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); + { + std::string name = "dict_key_int64"; + + auto key_column = ColumnInt64::create(); + auto key_data = std::make_shared(); + key_column->insert_value(2); + key_column->insert_value(3); + key_column->insert_value(5); + key_column->insert_value(7); + + auto f32_column = DataTypeFloat32::ColumnType::create(); + auto f32_type = std::make_shared(); + f32_column->insert_value(3.14); + f32_column->insert_value(0.14); + f32_column->insert_value(0.001); + f32_column->insert_value(11110.001); + + auto i64_column = DataTypeInt64::ColumnType::create(); + auto i64_type = std::make_shared(); + i64_column->insert_value(114514); + i64_column->insert_value(1919810); + i64_column->insert_value(123456); + i64_column->insert_value(654321); + + auto str_column = DataTypeString::ColumnType::create(); + auto str_type = std::make_shared(); + str_column->insert_value("hello world"); + str_column->insert_value("dict"); + str_column->insert_value("????"); + str_column->insert_value("doris"); + + auto dict = create_hash_map_dict_from_column( + name, ColumnWithTypeAndName {std::move(key_column), key_data, ""}, + ColumnsWithTypeAndName { + ColumnWithTypeAndName {std::move(f32_column), f32_type, "f32"}, + ColumnWithTypeAndName {std::move(i64_column), i64_type, "i64"}, + ColumnWithTypeAndName {std::move(str_column), str_type, "str"}}); + + register_dict(dict); + } + + { + std::string name = "dict_key_string"; + + auto key_column = DataTypeString::ColumnType::create(); + auto key_data = std::make_shared(); + key_column->insert_value("abc"); + key_column->insert_value("def"); + key_column->insert_value("some null"); + + auto f32_column = DataTypeFloat32::ColumnType::create(); + auto f32_type = std::make_shared(); + f32_column->insert_value(3.14); + f32_column->insert_value(0.14); + f32_column->insert_value(0.001); + + auto i64_column = DataTypeInt64::ColumnType::create(); + auto i64_type = std::make_shared(); + i64_column->insert_value(114514); + i64_column->insert_value(1919810); + i64_column->insert_value(123456); + + auto str_column = DataTypeString::ColumnType::create(); + auto str_type = std::make_shared(std::make_shared()); + auto null_map = ColumnUInt8::create(); + str_column->insert_value("hello world"); + null_map->insert_value(0); + str_column->insert_value("dict"); + null_map->insert_value(0); + str_column->insert_value("????"); + null_map->insert_value(1); + + auto dict = create_hash_map_dict_from_column( + name, ColumnWithTypeAndName {std::move(key_column), key_data, ""}, + ColumnsWithTypeAndName { + ColumnWithTypeAndName {std::move(f32_column), f32_type, "f32"}, + ColumnWithTypeAndName {std::move(i64_column), i64_type, "i64"}, + ColumnWithTypeAndName { + ColumnNullable::create(std::move(str_column), std::move(null_map)), + str_type, "str"}}); + + register_dict(dict); + } + + { + std::vector ips = {"192.168.0.0/16", "192.168.1.0/24", "192.168.1.128/25", + "1:288:2080::/41", "10.0.0.0/8", "10.1.0.0/16", + "172.16.0.0/12", "172.16.1.0/24", "172.16.1.128/25", + "203.0.113.0/24", "198.51.100.0/24", "2001:db8::/32", + "2001:db8:abcd::/48", "fc00::/7", "fe80::/10"}; + + std::vector names = {"A", "B", "C", "D", "E", "F", "G", "H", + "I", "J", "K", "L", "M", "N", "O"}; + + auto key_column = DataTypeString::ColumnType::create(); + auto key_data = std::make_shared(); + + auto str_column1 = DataTypeString::ColumnType::create(); + auto str_type1 = std::make_shared(); + + auto str_column2 = DataTypeString::ColumnType::create(); + auto str_type2 = std::make_shared(); + + auto toLowerCase = [](const std::string& input) { + std::string result = input; + std::transform(result.begin(), result.end(), result.begin(), + [](unsigned char c) { return std::tolower(c); }); + return result; + }; + + for (int i = 0; i < ips.size(); i++) { + key_column->insert_value(ips[i]); + str_column1->insert_value(names[i]); + str_column2->insert_value(toLowerCase(names[i])); + } + + std::string name = "ip_map"; + auto dict = create_ip_trie_dict_from_column( + name, ColumnWithTypeAndName {std::move(key_column), key_data, ""}, + ColumnsWithTypeAndName { + ColumnWithTypeAndName {std::move(str_column1), str_type1, "ATT"}, + ColumnWithTypeAndName {std::move(str_column2), str_type2, "att"}, + }); + + register_dict(dict); + } +} +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/dictionary_factory.h b/be/src/vec/functions/dictionary_factory.h new file mode 100644 index 000000000000000..99252bfa4dfec3a --- /dev/null +++ b/be/src/vec/functions/dictionary_factory.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "vec/functions/dictionary.h" + +namespace doris { +class MemTrackerLimiter; +} +namespace doris::vectorized { + +class DictionaryFactory : private boost::noncopyable { +public: + DictionaryFactory(); + ~DictionaryFactory(); + std::shared_ptr get(const std::string& name) { + std::shared_lock lc(_mutex); + if (_dict_map.contains(name)) { + return _dict_map[name]; + } + return nullptr; + } + + void register_dict(DictionaryPtr dict) { + std::unique_lock lc(_mutex); + _dict_map[dict->dict_name()] = dict; + } + + void register_dict_when_load_be(); + + std::string current_all_dict() { + std::shared_lock lc(_mutex); + std::string all_dict_name; + for (auto [name, _] : _dict_map) { + all_dict_name += name + ", "; + } + return all_dict_name; + } + std::map _dict_map; + + std::shared_mutex _mutex; + bool _has_load = false; + + std::shared_ptr _mem_tracker; +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/function_dict_get.cpp b/be/src/vec/functions/function_dict_get.cpp new file mode 100644 index 000000000000000..c2c9b30b805b5c9 --- /dev/null +++ b/be/src/vec/functions/function_dict_get.cpp @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "common/logging.h" +#include "common/status.h" +#include "vec/columns/column.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_ipv4.h" +#include "vec/data_types/data_type_ipv6.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_time.h" +#include "vec/data_types/data_type_time_v2.h" +#include "vec/functions/dictionary.h" +#include "vec/functions/dictionary_factory.h" +#include "vec/functions/function.h" +#include "vec/functions/simple_function_factory.h" +namespace doris::vectorized { + +struct DictGetState { + std::shared_ptr dict; +}; + +template +class FunctionDictGetWithType : public IFunction { +public: + static constexpr auto name = Impl::name; + using ReturnType = Impl::Type; + static FunctionPtr create() { return std::make_shared(); } + String get_name() const override { return name; } + + bool is_variadic() const override { return false; } + size_t get_number_of_arguments() const override { return 3; } + + DataTypes get_variadic_argument_types_impl() const override { return {}; } + + DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { + return std::make_shared(); + } + + bool use_default_implementation_for_nulls() const override { return true; } + + Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override { + if (scope == FunctionContext::THREAD_LOCAL) { + return Status::OK(); + } + std::shared_ptr state = std::make_shared(); + context->set_function_state(scope, state); + DCHECK(context->get_num_args() == 3); + + std::string dict_name = + context->get_constant_col(0)->column_ptr->get_data_at(0).to_string(); + auto dict = ExecEnv::GetInstance()->dict_factory()->get(dict_name); + if (!dict) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, + "can not find dict {} , current_all_dict {} ", dict_name, + ExecEnv::GetInstance()->dict_factory()->current_all_dict()); + } + state->dict = dict; + return Status::OK(); + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + auto* dict_state = reinterpret_cast( + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + if (!dict_state) { + return Status::RuntimeError("funciton context for function '{}' must have dict_state;", + get_name()); + } + + // dict get(name, att_name, key_value) -> att value + const std::string attribute_name = + block.get_by_position(arguments[1]).column->get_data_at(0).to_string(); + const DataTypePtr attribute_type = block.get_by_position(result).type; + + const ColumnPtr key_column = + block.get_by_position(arguments[2]).column->convert_to_full_column_if_const(); + const DataTypePtr key_type = block.get_by_position(arguments[2]).type; + + const auto dict = dict_state->dict; + + // check attribute + if (!dict->has_attribute(attribute_name)) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + "No corresponding attribute_name. The current attribute_name is: {}", + attribute_name); + } + if (!dict->get_attribute_type(attribute_name)->equals(*attribute_type)) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, + "Attribute type mismatch. Attribute name: {} Expected type: " + "{} Input type: {}", + attribute_name, + dict->get_attribute_type(attribute_name)->get_name(), + attribute_type->get_name()); + } + + // check key in dict::getColumn + auto res = dict->getColumn(attribute_name, attribute_type, key_column, key_type); + + block.replace_by_position(result, std::move(res)); + + return Status::OK(); + } +}; + +struct Int8Impl { + constexpr static auto name = "dict_get_int8"; + using Type = DataTypeInt8; +}; + +struct Int16Impl { + constexpr static auto name = "dict_get_int16"; + using Type = DataTypeInt16; +}; + +struct Int32Impl { + constexpr static auto name = "dict_get_int32"; + using Type = DataTypeInt32; +}; + +struct Int64Impl { + constexpr static auto name = "dict_get_int64"; + using Type = DataTypeInt64; +}; + +struct Int128Impl { + constexpr static auto name = "dict_get_int128"; + using Type = DataTypeInt64; +}; + +struct UInt8Impl { + constexpr static auto name = "dict_get_bool"; + using Type = DataTypeInt8; +}; + +struct Float32Impl { + constexpr static auto name = "dict_get_float32"; + using Type = DataTypeFloat32; +}; + +struct Float64Impl { + constexpr static auto name = "dict_get_float64"; + using Type = DataTypeFloat64; +}; + +struct DecimalImpl { + constexpr static auto name = "dict_get_decimal"; + using Type = DataTypeDecimal; +}; + +struct DateImpl { + constexpr static auto name = "dict_get_date"; + using Type = DataTypeDateV2; +}; + +struct DateTimeImpl { + constexpr static auto name = "dict_get_datetime"; + using Type = DataTypeDateTimeV2; +}; + +struct Ipv4Impl { + constexpr static auto name = "dict_get_ipv4"; + using Type = DataTypeIPv4; +}; + +struct Ipv6Impl { + constexpr static auto name = "dict_get_ipv6"; + using Type = DataTypeIPv6; +}; + +struct StringImpl { + constexpr static auto name = "dict_get_string"; + using Type = DataTypeString; +}; + +void register_function_dict_get(SimpleFunctionFactory& factory) { + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + + factory.register_function>(); + + factory.register_function>(); + factory.register_function>(); + + factory.register_function>(); + factory.register_function>(); + + factory.register_function>(); + factory.register_function>(); + + factory.register_function>(); +} + +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/hash_map_dictionary.h b/be/src/vec/functions/hash_map_dictionary.h new file mode 100644 index 000000000000000..6928bd967af7ad7 --- /dev/null +++ b/be/src/vec/functions/hash_map_dictionary.h @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "vec/columns/column.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" +#include "vec/core/columns_with_type_and_name.h" +#include "vec/functions/dictionary.h" +#include "vec/runtime/vdatetime_value.h" + +namespace doris::vectorized { + +template +class HashMapDictionary : public IDictionary { +public: + using Self = HashMapDictionary; + using KeyType = KeyDataType::FieldType; + using KeyRealColumnType = KeyDataType::ColumnType; + + HashMapDictionary(std::string name, std::vector attributes) + : IDictionary(std::move(name), std::move(attributes)), + _key_type(std::make_shared()) {} + + static DictionaryPtr create_hash_map_dict(const std::string& name, ColumnPtr& key_column, + ColumnsWithTypeAndName& attribute_data) { + std::vector attributes; + std::vector attributes_column; + for (const auto& att : attribute_data) { + /// TODO: remove "remove nullable" + attributes.push_back({att.name, remove_nullable(att.type)}); + attributes_column.push_back(att.column); + } + auto dict = std::make_shared(name, attributes); + dict->load_data(key_column, attributes_column); + return dict; + } + + ColumnPtr getColumn(const std::string& attribute_name, const DataTypePtr& attribute_type, + const ColumnPtr& key_column, const DataTypePtr& key_type) const override { + if (key_type->get_type_id() != _key_type->get_type_id()) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, + "HashMapDictionary({}) key type is : {} , but input is : {}", + dict_name(), _key_type->get_name(), key_type->get_name()); + } + MutableColumnPtr res_column = attribute_type->create_column(); + auto& attribute = _hashmap_attributes[attribute_index(attribute_name)]; + const auto* real_key_column = assert_cast(key_column.get()); + + std::visit( + [&](auto&& arg) { + using HashTableType = std::decay_t; + using AttributeRealDataType = HashTableType::DataType; + using AttributeRealColumnType = AttributeRealDataType::ColumnType; + + auto* res_real_column = assert_cast(res_column.get()); + const auto& attributes_column = arg.column; + + if (attribute.null_map) { + // att is nullable + const auto& null_map = attribute.null_map->get_data(); + for (size_t i = 0; i < real_key_column->size(); i++) { + const auto& key_value = real_key_column->get_element(i); + auto it = _key_hash_map.find(key_value); + if (it == _key_hash_map.end()) { + res_real_column->insert_default(); + } else { + const auto idx = it->second; + if (null_map[idx]) { + res_real_column->insert_default(); + } else { + res_real_column->insert_value( + attributes_column->get_element(idx)); + } + } + } + + } else { + // att without nullable + for (size_t i = 0; i < real_key_column->size(); i++) { + const auto& key_value = real_key_column->get_element(i); + auto it = _key_hash_map.find(key_value); + if (it == _key_hash_map.end()) { + res_real_column->insert_default(); + } else { + const auto idx = it->second; + res_real_column->insert_value(attributes_column->get_element(idx)); + } + } + } + }, + attribute.containers); + + return res_column; + } + +private: + void load_data(ColumnPtr& key_column, std::vector& attributes_column) { + const auto* key_real_column = assert_cast(key_column.get()); + for (size_t i = 0; i < key_real_column->size(); i++) { + auto key_str = key_real_column->get_element(i); + _key_hash_map[key_str] = i; + } + + _hashmap_attributes.resize(attributes_column.size()); + for (size_t i = 0; i < attributes_column.size(); i++) { + const DataTypePtr att_type = _attributes[i].type; + ColumnPtr column = attributes_column[i]; + + auto remove_nullable_data_type = remove_nullable(att_type); + auto remove_nullable_column = remove_nullable(column); + + // Set containers + bool valid = cast_type(remove_nullable_data_type.get(), [&](const auto& type) { + using AttributeRealDataType = std::decay_t; + using AttributeRealColumnType = AttributeRealDataType::ColumnType; + const auto* res_real_column = + typeid_cast(remove_nullable_column.get()); + if (!res_real_column) { + return false; + } + auto& att = _hashmap_attributes[i]; + ColumnWithType column_with_type; + column_with_type.column = AttributeRealColumnType::create(*res_real_column); + att.containers = column_with_type; + return true; + }); + if (!valid) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + "HashMapDictionary({}) att type is : {} , but input column is : {}", + dict_name(), att_type->get_name(), column->get_name()); + } + + // Set other content, such as nullmap (no need to visit). + + if (column->is_nullable()) { + auto& att = _hashmap_attributes[i]; + att.null_map = ColumnUInt8::create( + assert_cast(column.get())->get_null_map_column()); + } + } + } + + struct ColumnWithNullAttribute final { + ColumnUInt8::Ptr null_map; + ColumnData containers; + }; + + phmap::flat_hash_map _key_hash_map; + + std::vector _hashmap_attributes; + + DataTypePtr _key_type; +}; + +inline DictionaryPtr create_hash_map_dict_from_column(const std::string& name, + ColumnWithTypeAndName key_data, + ColumnsWithTypeAndName attribute_data) { + auto key_column = key_data.column; + auto key_type = key_data.type; + DictionaryPtr dict; + bool valid = IDictionary::cast_type(key_type.get(), [&](const auto& type) { + using KeyRealDataType = std::decay_t; + dict = HashMapDictionary::create_hash_map_dict(name, key_column, + attribute_data); + return true; + }); + + if (!valid) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, " Unsupported key type : {}", + key_type->get_name()); + } + return dict; +} + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/ip_address_dictionary.cpp b/be/src/vec/functions/ip_address_dictionary.cpp new file mode 100644 index 000000000000000..01972e1e9b6734e --- /dev/null +++ b/be/src/vec/functions/ip_address_dictionary.cpp @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/functions/ip_address_dictionary.h" + +#include +#include +#include +#include +#include + +#include "gutil/strings/split.h" +#include "vec/columns/column.h" +#include "vec/columns/column_string.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_number.h" +#include "vec/exprs/vlambda_function_call_expr.h" +#include "vec/functions/dictionary.h" +#include "vec/runtime/ip_address_cidr.h" +#include "vec/runtime/ipv4_value.h" + +namespace doris::vectorized { + +void map_ipv4_to_ipv6(IPv4 ipv4, UInt8* buf) { + unaligned_store(buf, 0x0000FFFF00000000ULL | static_cast(ipv4)); + unaligned_store(buf + 8, 0); +} + +IPv6 ipv4_to_ipv6(IPv4 ipv4) { + IPv6 ipv6; + map_ipv4_to_ipv6(ipv4, reinterpret_cast(&ipv6)); + return ipv6; +} + +ColumnPtr IPAddressDictionary::getColumn(const std::string& attribute_name, + const DataTypePtr& attribute_type, + const ColumnPtr& key_column, + const DataTypePtr& key_type) const { + if (!WhichDataType {key_type}.is_ipv6()) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + "IPAddressDictionary only support ipv6 type key , input key type is {} ", + key_type->get_name()); + } + + MutableColumnPtr res_column = attribute_type->create_column(); + const auto& attribute = _column_data[attribute_index(attribute_name)]; + const auto* ipv6_column = assert_cast(key_column.get()); + + std::visit( + [&](auto&& arg) { + using HashTableType = std::decay_t; + using AttributeRealDataType = HashTableType::DataType; + using AttributeRealColumnType = AttributeRealDataType::ColumnType; + + auto* res_real_column = assert_cast(res_column.get()); + const auto& attributes_column = arg.column; + + for (size_t i = 0; i < ipv6_column->size(); i++) { + IPv6 ipv6 = ipv6_column->get_element(i); + auto it = lookupIP(ipv6); + if (it == ip_not_found()) { + res_column->insert_default(); + } else { + const auto idx = *it; + res_real_column->insert_value(attributes_column->get_element(idx)); + } + } + }, + attribute); + + return res_column; +} + +struct IPRecord { + IPAddressCIDR ip_with_cidr; + size_t row; + IPv6 to_ipv6() const { + if (const auto* address = ip_with_cidr._address.as_v6()) { + IPv6 ipv6; + memcpy(reinterpret_cast(&ipv6), address, sizeof(IPv6)); + return ipv6; + } + + return ipv4_to_ipv6(ip_with_cidr._address.as_v4()); + } + UInt8 prefix() const { + if (ip_with_cidr._address.as_v6()) { + return ip_with_cidr._prefix; + } + return ip_with_cidr._prefix + 96; + } +}; + +void IPAddressDictionary::load_data(ColumnPtr& key_column, + std::vector& attributes_column) { + const auto* str_column = assert_cast(key_column.get()); + + std::vector ip_records; + + // load key column + for (size_t i = 0; i < str_column->size(); i++) { + auto ip_str = str_column->get_element(i); + ip_records.push_back(IPRecord {parse_ip_with_cidr(ip_str), i}); + } + + // load att column + + _column_data.resize(attributes_column.size()); + + for (size_t i = 0; i < attributes_column.size(); i++) { + const DataTypePtr att_type = _attributes[i].type; + ColumnPtr column = attributes_column[i]; + bool valid = IDictionary::cast_type(att_type.get(), [&](const auto& type) { + using AttributeRealDataType = std::decay_t; + using AttributeRealColumnType = AttributeRealDataType::ColumnType; + const auto* res_real_column = typeid_cast(column.get()); + if (!res_real_column) { + return false; + } + auto& att = _column_data[i]; + ColumnWithType column_with_type; + column_with_type.column = AttributeRealColumnType::create(*res_real_column); + att = column_with_type; + return true; + }); + if (!valid) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + "IPAddressDictionary({}) att type is : {} , but input column is : {}", + dict_name(), att_type->get_name(), column->get_name()); + } + } + + // build ip trie + + // sort all ip + std::sort(ip_records.begin(), ip_records.end(), [&](const IPRecord& a, const IPRecord& b) { + if (a.to_ipv6() == b.to_ipv6()) { + return a.prefix() < b.prefix(); + } + return a.to_ipv6() < b.to_ipv6(); + }); + + auto new_end = std::unique(ip_records.begin(), ip_records.end(), + [&](const IPRecord& a, const IPRecord& b) { + return a.to_ipv6() == b.to_ipv6() && a.prefix() == b.prefix(); + }); + ip_records.erase(new_end, ip_records.end()); + + for (const auto& record : ip_records) { + ip_column.push_back(record.to_ipv6()); + mask_column.push_back(record.prefix()); + row_idx.push_back(record.row); + } + + parent_subnet.resize(ip_records.size()); + std::stack subnets_stack; + for (auto i = 0; i < ip_records.size(); i++) { + parent_subnet[i] = i; + while (!subnets_stack.empty()) { + size_t pi = subnets_stack.top(); + + auto cur_address_ip = ip_records[i].to_ipv6(); + const auto* cur_addi = reinterpret_cast(&cur_address_ip); + auto cur_subnet_ip = ip_records[pi].to_ipv6(); + const auto* cur_addip = reinterpret_cast(&cur_subnet_ip); + + bool is_mask_smaller = ip_records[pi].prefix() < ip_records[i].prefix(); + if (is_mask_smaller && + match_ipv6_subnet(cur_addi, cur_addip, ip_records[pi].prefix())) { + parent_subnet[i] = pi; + break; + } + + subnets_stack.pop(); + } + subnets_stack.push(i); + } +} + +IPAddressDictionary::RowIdxConstIter IPAddressDictionary::lookupIP(IPv6 target) const { + if (row_idx.empty()) { + return ip_not_found(); + } + + auto comp = [&](IPv6 value, auto idx) -> bool { return value < ip_column[idx]; }; + + auto range = std::ranges::views::iota(0ULL, row_idx.size()); + + auto found_it = std::ranges::upper_bound(range, target, comp); + + if (found_it == range.begin()) { + return ip_not_found(); + } + + --found_it; + + for (auto idx = *found_it;; idx = parent_subnet[idx]) { + if (match_ipv6_subnet(reinterpret_cast(&target), + reinterpret_cast(&ip_column[idx]), mask_column[idx])) { + return row_idx.begin() + idx; + } + if (idx == parent_subnet[idx]) { + return ip_not_found(); + } + } + + return ip_not_found(); +} + +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/ip_address_dictionary.h b/be/src/vec/functions/ip_address_dictionary.h new file mode 100644 index 000000000000000..8ab112655afdb81 --- /dev/null +++ b/be/src/vec/functions/ip_address_dictionary.h @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "vec/columns/column.h" +#include "vec/columns/columns_number.h" +#include "vec/common/string_ref.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/core/columns_with_type_and_name.h" +#include "vec/core/types.h" +#include "vec/functions/dictionary.h" + +namespace doris::vectorized { +class IPAddressDictionary : public IDictionary { +public: + IPAddressDictionary(std::string name, std::vector attributes) + : IDictionary(std::move(name), std::move(attributes)) {} + + ColumnPtr getColumn(const std::string& attribute_name, const DataTypePtr& attribute_type, + const ColumnPtr& key_column, const DataTypePtr& key_type) const override; + + static DictionaryPtr create_ip_trie_dict(const std::string& name, ColumnPtr& key_column, + ColumnsWithTypeAndName& attribute_data) { + std::vector attributes; + std::vector attributes_column; + for (const auto& att : attribute_data) { + attributes.push_back({att.name, att.type}); + attributes_column.push_back(att.column); + } + auto dict = std::make_shared(name, attributes); + dict->load_data(key_column, attributes_column); + return dict; + } + +private: + using RowIdxConstIter = std::vector::const_iterator; + + RowIdxConstIter ip_not_found() const { return row_idx.end(); } + + RowIdxConstIter lookupIP(IPv6 target) const; + + void load_data(ColumnPtr& key_column, std::vector& attributes_column); + + std::vector ip_column; + + std::vector mask_column; + + std::vector parent_subnet; + + std::vector row_idx; + + std::vector _column_data; +}; + +inline DictionaryPtr create_ip_trie_dict_from_column(const std::string& name, + ColumnWithTypeAndName key_data, + ColumnsWithTypeAndName attribute_data) { + auto key_column = key_data.column; + auto key_type = key_data.type; + if (!WhichDataType {key_type}.is_string()) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + "IPAddressDictionary only support string in key , input key type is {} ", + key_type->get_name()); + } + + for (auto col_type_name : attribute_data) { + if (col_type_name.type->is_nullable() || col_type_name.column->is_nullable()) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + "IPAddressDictionary only support nullable attribute , input attribute is {} ", + col_type_name.type->get_name()); + } + } + + DictionaryPtr dict = IPAddressDictionary::create_ip_trie_dict(name, key_column, attribute_data); + return dict; +} + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/simple_function_factory.h b/be/src/vec/functions/simple_function_factory.h index 98f2917d163e317..a2dd24c38c6b21d 100644 --- a/be/src/vec/functions/simple_function_factory.h +++ b/be/src/vec/functions/simple_function_factory.h @@ -111,7 +111,7 @@ void register_function_multi_match(SimpleFunctionFactory& factory); void register_function_split_by_regexp(SimpleFunctionFactory& factory); void register_function_assert_true(SimpleFunctionFactory& factory); void register_function_bit_test(SimpleFunctionFactory& factory); - +void register_function_dict_get(SimpleFunctionFactory& factory); class SimpleFunctionFactory { using Creator = std::function; using FunctionCreators = phmap::flat_hash_map; @@ -301,6 +301,7 @@ class SimpleFunctionFactory { register_function_split_by_regexp(instance); register_function_assert_true(instance); register_function_bit_test(instance); + register_function_dict_get(instance); }); return instance; } diff --git a/be/src/vec/runtime/ip_address_cidr.h b/be/src/vec/runtime/ip_address_cidr.h index d52bdede2f80317..3ca6575f7803447 100644 --- a/be/src/vec/runtime/ip_address_cidr.h +++ b/be/src/vec/runtime/ip_address_cidr.h @@ -97,14 +97,14 @@ struct IPAddressCIDR { vectorized::UInt8 _prefix; }; -bool match_ipv4_subnet(uint32_t addr, uint32_t cidr_addr, uint8_t prefix) { +inline bool match_ipv4_subnet(uint32_t addr, uint32_t cidr_addr, uint8_t prefix) { uint32_t mask = (prefix >= 32) ? 0xffffffffU : ~(0xffffffffU >> prefix); return (addr & mask) == (cidr_addr & mask); } #if defined(__SSE2__) || defined(__aarch64__) -bool match_ipv6_subnet(const uint8_t* addr, const uint8_t* cidr_addr, uint8_t prefix) { +inline bool match_ipv6_subnet(const uint8_t* addr, const uint8_t* cidr_addr, uint8_t prefix) { uint16_t mask = (uint16_t)_mm_movemask_epi8( _mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast(addr)), _mm_loadu_si128(reinterpret_cast(cidr_addr)))); @@ -126,7 +126,7 @@ bool match_ipv6_subnet(const uint8_t* addr, const uint8_t* cidr_addr, uint8_t pr #else // ipv6 liitle-endian input -bool match_ipv6_subnet(const uint8_t* addr, const uint8_t* cidr_addr, uint8_t prefix) { +inline bool match_ipv6_subnet(const uint8_t* addr, const uint8_t* cidr_addr, uint8_t prefix) { if (prefix > IPV6_BINARY_LENGTH * 8U) { prefix = IPV6_BINARY_LENGTH * 8U; } @@ -147,7 +147,7 @@ bool match_ipv6_subnet(const uint8_t* addr, const uint8_t* cidr_addr, uint8_t pr } #endif -IPAddressCIDR parse_ip_with_cidr(std::string_view cidr_str) { +inline IPAddressCIDR parse_ip_with_cidr(std::string_view cidr_str) { size_t pos_slash = cidr_str.find('/'); if (pos_slash == 0) { diff --git a/build.sh b/build.sh index 5a09b7a0a165e3d..2acbbaf849ee966 100755 --- a/build.sh +++ b/build.sh @@ -446,7 +446,7 @@ if [[ -n "${DISABLE_BUILD_HIVE_UDF}" ]]; then fi if [[ -z "${DISABLE_JAVA_CHECK_STYLE}" ]]; then - DISABLE_JAVA_CHECK_STYLE='OFF' + DISABLE_JAVA_CHECK_STYLE='ON' fi if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index ed3f2895cc86ee5..181dbc0d53f82ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -477,6 +477,11 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DictGetFloat32; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DictGetInt32; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DictGetInt64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DictGetIPv4; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DictGetString; import com.google.common.collect.ImmutableList; @@ -485,7 +490,8 @@ /** * Builtin scalar functions. *

- * Note: Please ensure that this class only has some lists and no procedural code. + * Note: Please ensure that this class only has some lists and no procedural + * code. * It helps to be clear and concise. */ public class BuiltinScalarFunctions implements FunctionHelper { @@ -970,10 +976,16 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(YearsSub.class, "years_sub"), scalar(MultiMatch.class, "multi_match"), scalar(SessionUser.class, "session_user"), - scalar(LastQueryId.class, "last_query_id")); + scalar(LastQueryId.class, "last_query_id"), + scalar(DictGetFloat32.class, "dict_get_float32"), + scalar(DictGetIPv4.class, "dict_get_ipv4"), + scalar(DictGetInt32.class, "dict_get_int32"), + scalar(DictGetInt64.class, "dict_get_int64"), + scalar(DictGetString.class, "dict_get_string")); public static final BuiltinScalarFunctions INSTANCE = new BuiltinScalarFunctions(); // Note: Do not add any code here! - private BuiltinScalarFunctions() {} + private BuiltinScalarFunctions() { + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java new file mode 100644 index 000000000000000..1e0a0ad708345f7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FloatType; +import org.apache.doris.nereids.types.IPv4Type; +import org.apache.doris.nereids.types.IPv6Type; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.types.coercion.PrimitiveType; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +public class DictGet { + public static List creatSignatures(PrimitiveType returnType) { + return ImmutableList.of( + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + DateTimeV2Type.SYSTEM_DEFAULT), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + DateV2Type.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + BooleanType.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + TinyIntType.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + SmallIntType.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + IntegerType.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + BigIntType.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + LargeIntType.INSTANCE), + + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + FloatType.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + DoubleType.INSTANCE), + + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + DecimalV3Type.WILDCARD), + + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + IPv4Type.INSTANCE), + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + IPv6Type.INSTANCE), + + FunctionSignature.ret(returnType).args(StringType.INSTANCE, StringType.INSTANCE, + StringType.INSTANCE)); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetFloat32.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetFloat32.java new file mode 100644 index 000000000000000..f9b4238418c9bb0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetFloat32.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.FloatType; +import com.google.common.base.Preconditions; +import java.util.List; + +public class DictGetFloat32 extends ScalarFunction + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable { + + private static String nameSuffix = "float32"; + private static String dictFunctionName = "dict_get_" + nameSuffix; + public static final List SIGNATURES = DictGet.creatSignatures(FloatType.INSTANCE); + + /** + * constructor with 3 argument. + */ + public DictGetFloat32(Expression arg0, Expression arg1, Expression arg2) { + super(dictFunctionName, arg0, arg1, arg2); + if (!arg0.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): name support const value only."); + } + if (!arg1.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): att support const value only."); + } + } + + /** + * withChildren. + */ + @Override + public DictGetFloat32 withChildren(List children) { + Preconditions.checkArgument(children.size() == 3); + return new DictGetFloat32(children.get(0), children.get(1), children.get(2)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitScalarFunction(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetIPv4.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetIPv4.java new file mode 100644 index 000000000000000..bb62af6d0643859 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetIPv4.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.IPv4Type; +import com.google.common.base.Preconditions; +import java.util.List; + +public class DictGetIPv4 extends ScalarFunction + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable { + + private static String nameSuffix = "ipv4"; + private static String dictFunctionName = "dict_get_" + nameSuffix; + public static final List SIGNATURES = DictGet.creatSignatures(IPv4Type.INSTANCE); + + /** + * constructor with 3 argument. + */ + public DictGetIPv4(Expression arg0, Expression arg1, Expression arg2) { + super(dictFunctionName, arg0, arg1, arg2); + if (!arg0.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): name support const value only."); + } + if (!arg1.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): att support const value only."); + } + } + + /** + * withChildren. + */ + @Override + public DictGetIPv4 withChildren(List children) { + Preconditions.checkArgument(children.size() == 3); + return new DictGetIPv4(children.get(0), children.get(1), children.get(2)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitScalarFunction(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt32.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt32.java new file mode 100644 index 000000000000000..ca1b39df0ae90f8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt32.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.IntegerType; +import com.google.common.base.Preconditions; +import java.util.List; + +public class DictGetInt32 extends ScalarFunction + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable { + + private static String nameSuffix = "int32"; + private static String dictFunctionName = "dict_get_" + nameSuffix; + public static final List SIGNATURES = DictGet.creatSignatures(IntegerType.INSTANCE); + + /** + * constructor with 3 argument. + */ + public DictGetInt32(Expression arg0, Expression arg1, Expression arg2) { + super(dictFunctionName, arg0, arg1, arg2); + if (!arg0.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): name support const value only."); + } + if (!arg1.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): att support const value only."); + } + } + + /** + * withChildren. + */ + @Override + public DictGetInt32 withChildren(List children) { + Preconditions.checkArgument(children.size() == 3); + return new DictGetInt32(children.get(0), children.get(1), children.get(2)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitScalarFunction(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt64.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt64.java new file mode 100644 index 000000000000000..b68a0186fdf9135 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetInt64.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BigIntType; +import com.google.common.base.Preconditions; +import java.util.List; + +public class DictGetInt64 extends ScalarFunction + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable { + + private static String nameSuffix = "int64"; + private static BigIntType returnType = BigIntType.INSTANCE; + + public static final List SIGNATURES = DictGet.creatSignatures(returnType); + + private static String dictFunctionName = "dict_get_" + nameSuffix; + + /** + * constructor with 3 argument. + */ + public DictGetInt64(Expression arg0, Expression arg1, Expression arg2) { + super(dictFunctionName, arg0, arg1, arg2); + if (!arg0.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): name support const value only."); + } + if (!arg1.isConstant()) { + throw new AnalysisException( + dictFunctionName + "(name, att, key): att support const value only."); + } + } + + /** + * withChildren. + */ + @Override + public DictGetInt64 withChildren(List children) { + Preconditions.checkArgument(children.size() == 3); + return new DictGetInt64(children.get(0), children.get(1), children.get(2)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitScalarFunction(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetString.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetString.java new file mode 100644 index 000000000000000..37212cd4a0b6c4a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetString.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.StringType; +import com.google.common.base.Preconditions; +import java.util.List; + +public class DictGetString extends ScalarFunction + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable { + + public static final List SIGNATURES = DictGet.creatSignatures(StringType.INSTANCE); + + /** + * constructor with 3 argument. + */ + public DictGetString(Expression arg0, Expression arg1, Expression arg2) { + super("dict_get_string", arg0, arg1, arg2); + if (!arg0.isConstant()) { + throw new AnalysisException( + "dict_get_string(name, att, key): name support const value only."); + } + if (!arg1.isConstant()) { + throw new AnalysisException( + "dict_get_string(name, att, key): att support const value only."); + } + } + + /** + * withChildren. + */ + @Override + public DictGetString withChildren(List children) { + Preconditions.checkArgument(children.size() == 3); + return new DictGetString(children.get(0), children.get(1), children.get(2)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitScalarFunction(this, context); + } +} diff --git a/regression-test/data/correctness/test_dict_function.out b/regression-test/data/correctness/test_dict_function.out new file mode 100644 index 000000000000000..ec2b87d2fcd3a0f --- /dev/null +++ b/regression-test/data/correctness/test_dict_function.out @@ -0,0 +1,70 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +1 0 +2 114514 +3 1919810 +4 0 +5 123456 +6 0 +7 654321 +8 0 + +-- !select2 -- +1 +2 hello world +3 dict +4 +5 ???? +6 +7 doris +8 + +-- !select3 -- +1 0.0 +2 3.14 +3 0.14 +4 0.0 +5 0.001 +6 0.0 +7 11110.001 +8 0.0 + +-- !select1 -- +1 abc 114514 +2 1 0 +3 2 0 +4 3 0 +5 some null 123456 +6 4 0 +7 def 1919810 +8 5 0 + +-- !select2 -- +1 abc hello world +2 1 +3 2 +4 3 +5 some null +6 4 +7 def dict +8 5 + +-- !select3 -- +1 abc 3.14 +2 1 0.0 +3 2 0.0 +4 3 0.0 +5 some null 0.001 +6 4 0.0 +7 def 0.14 +8 5 0.0 + +-- !select1 -- +C + +-- !select2 -- +B + +-- !select3 -- +i + diff --git a/regression-test/suites/correctness/test_dict_function.groovy b/regression-test/suites/correctness/test_dict_function.groovy new file mode 100644 index 000000000000000..44021f0d7d4233f --- /dev/null +++ b/regression-test/suites/correctness/test_dict_function.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_dict_function") { + sql """ DROP TABLE IF EXISTS test_dict_function """ + sql """ + CREATE TABLE test_dict_function( + id bigint not null + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + sql """ + insert into test_dict_function values(1),(2),(3),(4),(5),(6),(7),(8); + """ + + qt_select1 """ + select id, dict_get_int64("dict_key_int64","i64" ,id) from test_dict_function order by id; + """ + qt_select2 """ + select id, dict_get_string("dict_key_int64","str" ,id) from test_dict_function order by id; + """ + qt_select3 """ + select id, dict_get_float32("dict_key_int64","f32" ,id) from test_dict_function order by id; + """ + + test { + sql """ + select id, dict_get_string("dict_key_int64","f32" ,id) from test_dict_function order by id; + """ + exception "Attribute type mismatch" + } + + + + sql """ DROP TABLE IF EXISTS test_dict_str_function """ + sql """ + CREATE TABLE test_dict_str_function( + id bigint not null, + name string not null + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + sql """ + insert into test_dict_str_function values(1,"abc"),(2,"1"),(3,"2"),(4,"3"),(5,"some null"),(6,"4"),(7,"def"),(8,"5"); + """ + + qt_select1 """ + select id, name, dict_get_int64("dict_key_string","i64" ,name) from test_dict_str_function order by id; + """ + qt_select2 """ + select id, name, dict_get_string("dict_key_string","str" ,name) from test_dict_str_function order by id; + + """ + qt_select3 """ + select id, name, dict_get_float32("dict_key_string","f32" ,name) from test_dict_str_function order by id; + + """ + + test { + sql """ + select id, dict_get_int64("dict_key_string","i64" ,id) from test_dict_str_function order by id; + """ + exception "INVALID_ARGUMENT" + } + + + + qt_select1 """ + select dict_get_string("ip_map","ATT" ,cast(cast("192.168.1.130" as ipv4) as ipv6)); + """ + qt_select2 """ + select dict_get_string("ip_map","ATT" ,cast(cast("192.168.1.00" as ipv4) as ipv6)); + """ + qt_select3 """ + select dict_get_string("ip_map","att" ,cast(cast("172.16.1.130" as ipv4) as ipv6)); + """ + +}