upd

be/src/runtime/exec_env.h

@@ -50,6 +50,7 @@ class VDataStreamMgr;
 class ScannerScheduler;
 class SpillStreamManager;
 class DeltaWriterV2Pool;
+class DictionaryFactory;
 } // namespace vectorized
 namespace pipeline {
 class TaskScheduler;
@@ -337,6 +338,8 @@ class ExecEnv {
         return _runtime_filter_timer_queue;
     }
 
+    vectorized::DictionaryFactory* dict_factory() { return _dict_factory; }
+
     pipeline::PipelineTracerContext* pipeline_tracer_context() {
         return _pipeline_tracer_ctx.get();
     }
@@ -477,6 +480,7 @@ class ExecEnv {
     std::unique_ptr<io::FDCache> _file_cache_open_fd_cache;
 
     pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr;
+    vectorized::DictionaryFactory* _dict_factory = nullptr;
 
     WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr;
 

be/src/runtime/exec_env_init.cpp

@@ -108,6 +108,7 @@
 #include "vec/exec/format/orc/orc_memory_pool.h"
 #include "vec/exec/format/parquet/arrow_memory_pool.h"
 #include "vec/exec/scan/scanner_scheduler.h"
+#include "vec/functions/dictionary_factory.h"
 #include "vec/runtime/vdata_stream_mgr.h"
 #include "vec/sink/delta_writer_v2_pool.h"
 #include "vec/sink/load_stream_map_pool.h"
@@ -376,6 +377,7 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
 
     RETURN_IF_ERROR(_spill_stream_mgr->init());
     _runtime_query_statistics_mgr->start_report_thread();
+    _dict_factory = new doris::vectorized::DictionaryFactory();
     _s_ready = true;
 
     return Status::OK();
@@ -776,6 +778,7 @@ void ExecEnv::destroy() {
     SAFE_DELETE(_workload_group_manager);
     SAFE_DELETE(_file_cache_factory);
     SAFE_DELETE(_runtime_filter_timer_queue);
+    SAFE_DELETE(_dict_factory);
     // TODO(zhiqiang): Maybe we should call shutdown before release thread pool?
     _lazy_release_obj_pool.reset(nullptr);
     _non_block_close_thread_pool.reset(nullptr);

be/src/vec/columns/column_string.h

@@ -141,6 +141,10 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
         return StringRef(&chars[offset_at(n)], size_at(n));
     }
 
+    String get_element(size_t n) const { return get_data_at(n).to_string(); }
+
+    void insert_value(const String& value) { insert_data(value.data(), value.size()); }
+
     void insert(const Field& x) override {
         StringRef s;
         if (x.get_type() == Field::Types::JSONB) {

be/src/vec/functions/dictionary.cpp

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/functions/dictionary.h"
+
+namespace doris::vectorized {
+
+IDictionary::IDictionary(std::string name, std::vector<DictionaryAttribute> attributes)
+        : _dict_name(std::move(name)), _attributes(std::move(attributes)) {
+    for (size_t i = 0; i < _attributes.size(); i++) {
+        const auto& name = _attributes[i].name;
+        if (_name_to_attributes_index.contains(name)) {
+            throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
+                                   "The names of attributes should not have duplicates : {}", name);
+        }
+        _name_to_attributes_index[name] = i;
+    }
+}
+
+bool IDictionary::has_attribute(const std::string& name) const {
+    return _name_to_attributes_index.contains(name);
+}
+
+size_t IDictionary::attribute_index(const std::string& name) const {
+    auto it = _name_to_attributes_index.find(name);
+    if (it == _name_to_attributes_index.end()) {
+        throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "no this attribute : {}", name);
+    }
+    size_t idx = it->second;
+    return idx;
+}
+
+DataTypePtr IDictionary::get_attribute_type(const std::string& name) const {
+    auto it = _name_to_attributes_index.find(name);
+    if (it == _name_to_attributes_index.end()) {
+        throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "no this attribute : {}", name);
+    }
+    size_t idx = it->second;
+    return _attributes[idx].type;
+}
+} // namespace doris::vectorized

be/src/vec/functions/dictionary.h

@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <shared_mutex>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "vec/columns/column.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_ipv4.h"
+#include "vec/data_types/data_type_ipv6.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_time_v2.h"
+#include "vec/functions/cast_type_to_either.h"
+
+namespace doris::vectorized {
+struct DictionaryAttribute {
+    const std::string name;
+    const DataTypePtr type;
+};
+
+class IDictionary {
+public:
+    IDictionary(std::string name, std::vector<DictionaryAttribute> attributes);
+    virtual ~IDictionary() = default;
+    std::string dict_name() const { return _dict_name; }
+
+    virtual ColumnPtr getColumn(const std::string& attribute_name,
+                                const DataTypePtr& attribute_type, const ColumnPtr& key_column,
+                                const DataTypePtr& key_type) const = 0;
+
+    bool has_attribute(const std::string& name) const;
+    DataTypePtr get_attribute_type(const std::string& name) const;
+    size_t attribute_index(const std::string& name) const;
+
+    template <typename F>
+    static bool cast_type(const IDataType* type, F&& f) {
+        return cast_type_to_either<DataTypeUInt8, DataTypeInt8, DataTypeInt16, DataTypeInt32,
+                                   DataTypeInt64, DataTypeInt128, DataTypeFloat32, DataTypeFloat64,
+                                   DataTypeIPv4, DataTypeIPv6, DataTypeString, DataTypeDateV2,
+                                   DataTypeDateTimeV2, DataTypeDecimal<Decimal32>,
+                                   DataTypeDecimal<Decimal64>, DataTypeDecimal<Decimal128V3>,
+                                   DataTypeDecimal<Decimal256>>(type, std::forward<F>(f));
+    }
+
+protected:
+    template <typename Type>
+    struct ColumnWithType {
+        using DataType = Type;
+        DataType::ColumnType::Ptr column;
+    };
+    using ColumnData =
+            std::variant<ColumnWithType<DataTypeUInt8>, ColumnWithType<DataTypeInt8>,
+                         ColumnWithType<DataTypeInt16>, ColumnWithType<DataTypeInt32>,
+                         ColumnWithType<DataTypeInt64>, ColumnWithType<DataTypeInt128>,
+
+                         ColumnWithType<DataTypeFloat32>, ColumnWithType<DataTypeFloat64>,
+
+                         ColumnWithType<DataTypeIPv4>, ColumnWithType<DataTypeIPv6>,
+
+                         ColumnWithType<DataTypeString>,
+
+                         ColumnWithType<DataTypeDateV2>, ColumnWithType<DataTypeDateTimeV2>,
+
+                         ColumnWithType<DataTypeDecimal<Decimal32>>,
+                         ColumnWithType<DataTypeDecimal<Decimal64>>,
+                         ColumnWithType<DataTypeDecimal<Decimal128V3>>,
+                         ColumnWithType<DataTypeDecimal<Decimal256>>>;
+
+    const std::string _dict_name;
+    std::vector<DictionaryAttribute> _attributes;
+    std::unordered_map<std::string, size_t> _name_to_attributes_index;
+};
+
+using DictionaryPtr = std::shared_ptr<IDictionary>;
+
+} // namespace doris::vectorized