diff --git a/be/src/column/column_pool.h b/be/src/column/column_pool.h deleted file mode 100644 index eaf632596b014..0000000000000 --- a/be/src/column/column_pool.h +++ /dev/null @@ -1,453 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include // NOLINT -#include - -#include - -#include "column/binary_column.h" -#include "column/const_column.h" -#include "column/decimalv3_column.h" -#include "column/fixed_length_column.h" -#include "column/object_column.h" -#include "common/compiler_util.h" -#include "common/config.h" -#include "common/type_list.h" -#include "gutil/dynamic_annotations.h" -#include "runtime/current_thread.h" -#include "util/json.h" - -namespace starrocks { - -template -struct ColumnPoolBlockSize { - static const size_t value = 256; -}; - -template <> -struct ColumnPoolBlockSize { - static const size_t value = 128; -}; - -template -int64_t column_bytes(const T* col) { - static_assert(std::is_base_of_v, "must_derived_of_Column"); - return col->memory_usage(); -} - -template -void reset_column(T* col) { - static_assert(std::is_base_of_v, "must_derived_of_Column"); - col->reset_column(); - DCHECK(col->size() == 0); - DCHECK(col->delete_state() == DEL_NOT_SATISFIED); -} - -// Returns the number of bytes freed to tcmalloc. -template -size_t release_column_if_large(T* col, size_t limit) { - auto old_usage = column_bytes(col); - if (old_usage < limit) { - return 0; - } - if constexpr (std::is_same_v) { - auto& bytes = col->get_bytes(); - BinaryColumn::Bytes tmp; - tmp.swap(bytes); - } else { - typename T::Container tmp; - tmp.swap(col->get_data()); - } - auto new_usage = column_bytes(col); - DCHECK_LT(new_usage, old_usage); - return old_usage - new_usage; -} - -template -size_t column_reserved_size(const T* col) { - if constexpr (std::is_same_v) { - const auto& offsets = col->get_offset(); - return offsets.capacity() > 0 ? offsets.capacity() - 1 : 0; - } else { - return col->get_data().capacity(); - } -} - -template -struct ColumnPoolFreeBlock { - int64_t nfree; - int64_t bytes; - T* ptrs[NITEM]; -}; - -struct ColumnPoolInfo { - size_t local_cnt = 0; - size_t central_free_items = 0; - size_t central_free_bytes = 0; -}; - -template -class CACHELINE_ALIGNED ColumnPool { - static constexpr size_t kBlockSize = ColumnPoolBlockSize::value; - - using FreeBlock = ColumnPoolFreeBlock; - using DynamicFreeBlock = ColumnPoolFreeBlock; - - class CACHELINE_ALIGNED LocalPool { - static_assert(std::is_base_of::value, "Must_be_derived_of_Column"); - - public: - explicit LocalPool(ColumnPool* pool) : _pool(pool) { - _curr_free.nfree = 0; - _curr_free.bytes = 0; - } - - ~LocalPool() noexcept { - if (_curr_free.nfree > 0 && !_pool->_push_free_block(_curr_free)) { - for (size_t i = 0; i < _curr_free.nfree; i++) { - ASAN_UNPOISON_MEMORY_REGION(_curr_free.ptrs[i], sizeof(T)); - delete _curr_free.ptrs[i]; - } - } - _pool->_clear_from_destructor_of_local_pool(); - } - - T* get_object() { - if (_curr_free.nfree == 0) { - if (!_pool->_pop_free_block(&_curr_free)) { - return nullptr; - } - } - T* obj = _curr_free.ptrs[--_curr_free.nfree]; - ASAN_UNPOISON_MEMORY_REGION(obj, sizeof(T)); - auto bytes = column_bytes(obj); - _curr_free.bytes -= bytes; - - tls_thread_status.mem_consume(bytes); - _pool->mem_tracker()->release(bytes); - - return obj; - } - - void return_object(T* ptr, size_t chunk_size) { - if (UNLIKELY(column_reserved_size(ptr) > chunk_size)) { - delete ptr; - return; - } - auto bytes = column_bytes(ptr); - if (_curr_free.nfree < kBlockSize) { - ASAN_POISON_MEMORY_REGION(ptr, sizeof(T)); - _curr_free.ptrs[_curr_free.nfree++] = ptr; - _curr_free.bytes += bytes; - - tls_thread_status.mem_release(bytes); - _pool->mem_tracker()->consume(bytes); - - return; - } - if (_pool->_push_free_block(_curr_free)) { - ASAN_POISON_MEMORY_REGION(ptr, sizeof(T)); - _curr_free.nfree = 1; - _curr_free.ptrs[0] = ptr; - _curr_free.bytes = bytes; - - tls_thread_status.mem_release(bytes); - _pool->mem_tracker()->consume(bytes); - - return; - } - delete ptr; - } - - void release_large_columns(size_t limit) { - size_t freed_bytes = 0; - for (size_t i = 0; i < _curr_free.nfree; i++) { - ASAN_UNPOISON_MEMORY_REGION(_curr_free.ptrs[i], sizeof(T)); - freed_bytes += release_column_if_large(_curr_free.ptrs[i], limit); - ASAN_POISON_MEMORY_REGION(_curr_free.ptrs[i], sizeof(T)); - } - if (freed_bytes > 0) { - _curr_free.bytes -= freed_bytes; - tls_thread_status.mem_consume(freed_bytes); - _pool->mem_tracker()->release(freed_bytes); - } - } - - static void delete_local_pool(void* arg) { delete (LocalPool*)arg; } - - private: - ColumnPool* _pool; - FreeBlock _curr_free; - }; - -public: - void set_mem_tracker(std::shared_ptr mem_tracker) { _mem_tracker = std::move(mem_tracker); } - MemTracker* mem_tracker() { return _mem_tracker.get(); } - - static std::enable_if_t, ColumnPool*> singleton() { - static ColumnPool p; - return &p; - } - - template - T* get_column() { - LocalPool* lp = _get_or_new_local_pool(); - if (UNLIKELY(lp == nullptr)) { - return nullptr; - } - T* ptr = lp->get_object(); - if (ptr == nullptr && AllocOnEmpty) { - ptr = new (std::nothrow) T(); - } - return ptr; - } - - void return_column(T* ptr, size_t chunk_size) { - LocalPool* lp = _get_or_new_local_pool(); - if (LIKELY(lp != nullptr)) { - reset_column(ptr); - lp->return_object(ptr, chunk_size); - } else { - delete ptr; - } - } - - // Destroy some objects in the *central* free list. - // Returns the number of bytes freed to tcmalloc. - size_t release_free_columns(double free_ratio) { - free_ratio = std::min(free_ratio, 1.0); - int64_t now = butil::gettimeofday_s(); - std::vector tmp; - if (now - _first_push_time > 3) { - // ^^^^^^^^^^^^^^^^ read without lock by intention. - std::lock_guard l(_free_blocks_lock); - int n = implicit_cast(static_cast::type>( - static_cast(_free_blocks.size()) * (1 - free_ratio))); - tmp.insert(tmp.end(), _free_blocks.begin() + n, _free_blocks.end()); - _free_blocks.resize(n); - } - size_t freed_bytes = 0; - for (DynamicFreeBlock* blk : tmp) { - freed_bytes += blk->bytes; - _release_free_block(blk); - } - - _mem_tracker->release(freed_bytes); - tls_thread_status.mem_consume(freed_bytes); - - return freed_bytes; - } - - // Reduce memory usage on behalf of column if its memory usage is greater - // than or equal to |limit|. - void release_large_columns(size_t limit) { - LocalPool* lp = _local_pool; - if (lp) { - lp->release_large_columns(limit); - } - } - - void clear_columns() { - LocalPool* lp = _local_pool; - if (lp) { - _local_pool = nullptr; - butil::thread_atexit_cancel(LocalPool::delete_local_pool, lp); - delete lp; - } - } - - ColumnPoolInfo describe_column_pool() { - ColumnPoolInfo info; - info.local_cnt = _nlocal.load(std::memory_order_relaxed); - if (_free_blocks.empty()) { - return info; - } - std::lock_guard l(_free_blocks_lock); - for (DynamicFreeBlock* blk : _free_blocks) { - info.central_free_items += blk->nfree; - info.central_free_bytes += blk->bytes; - } - return info; - } - -private: - static void _release_free_block(DynamicFreeBlock* blk) { - for (size_t i = 0; i < blk->nfree; i++) { - T* p = blk->ptrs[i]; - ASAN_UNPOISON_MEMORY_REGION(p, sizeof(T)); - delete p; - } - free(blk); - } - - LocalPool* _get_or_new_local_pool() { - LocalPool* lp = _local_pool; - if (LIKELY(lp != nullptr)) { - return lp; - } - lp = new (std::nothrow) LocalPool(this); - if (nullptr == lp) { - return nullptr; - } - std::lock_guard l(_change_thread_mutex); //avoid race with clear_columns() - _local_pool = lp; - (void)butil::thread_atexit(LocalPool::delete_local_pool, lp); - _nlocal.fetch_add(1, std::memory_order_relaxed); - return lp; - } - - void _clear_from_destructor_of_local_pool() { - _local_pool = nullptr; - - // Do nothing if there are active threads. - if (_nlocal.fetch_sub(1, std::memory_order_relaxed) != 1) { - return; - } - - std::lock_guard l(_change_thread_mutex); // including acquire fence. - // Do nothing if there are active threads. - if (_nlocal.load(std::memory_order_relaxed) != 0) { - return; - } - // All threads exited and we're holding _change_thread_mutex to avoid - // racing with new threads calling get_column(). - - // Clear global free list. - _first_push_time = 0; - release_free_columns(1.0); - } - - bool _push_free_block(const FreeBlock& blk) { - auto* p = (DynamicFreeBlock*)malloc(offsetof(DynamicFreeBlock, ptrs) + sizeof(*blk.ptrs) * blk.nfree); - if (UNLIKELY(p == nullptr)) { - return false; - } - p->nfree = blk.nfree; - p->bytes = blk.bytes; - memcpy(p->ptrs, blk.ptrs, sizeof(*blk.ptrs) * blk.nfree); - std::lock_guard l(_free_blocks_lock); - _first_push_time = _free_blocks.empty() ? butil::gettimeofday_s() : _first_push_time; - _free_blocks.push_back(p); - return true; - } - - bool _pop_free_block(FreeBlock* blk) { - if (_free_blocks.empty()) { - return false; - } - _free_blocks_lock.lock(); - if (_free_blocks.empty()) { - _free_blocks_lock.unlock(); - return false; - } - DynamicFreeBlock* p = _free_blocks.back(); - _free_blocks.pop_back(); - _free_blocks_lock.unlock(); - memcpy(blk->ptrs, p->ptrs, sizeof(*p->ptrs) * p->nfree); - blk->nfree = p->nfree; - blk->bytes = p->bytes; - free(p); - return true; - } - -private: - ColumnPool() { _free_blocks.reserve(32); } - - ~ColumnPool() = default; - - std::shared_ptr _mem_tracker = nullptr; - - static __thread LocalPool* _local_pool; // NOLINT - static std::atomic _nlocal; // NOLINT - static std::mutex _change_thread_mutex; // NOLINT - - mutable std::mutex _free_blocks_lock; - std::vector _free_blocks; - int64_t _first_push_time = 0; -}; - -using ColumnPoolList = - TypeList, ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool>; - -template -__thread typename ColumnPool::LocalPool* ColumnPool::_local_pool = nullptr; // NOLINT - -template -std::atomic ColumnPool::_nlocal = 0; // NOLINT - -template -std::mutex ColumnPool::_change_thread_mutex{}; // NOLINT - -template -inline T* get_column() { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - return ColumnPool::singleton()->template get_column(); -} - -template -inline void return_column(T* ptr, size_t chunk_size) { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - ColumnPool::singleton()->return_column(ptr, chunk_size); -} - -template -inline void release_large_columns(size_t limit) { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - ColumnPool::singleton()->release_large_columns(limit); -} - -template -inline size_t release_free_columns(double ratio) { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - return ColumnPool::singleton()->release_free_columns(ratio); -} - -// NOTE: this is an expensive routine, so it should not be called too often. -template -inline ColumnPoolInfo describe_column_pool() { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - return ColumnPool::singleton()->describe_column_pool(); -} - -// Used in tests. -template -inline void clear_columns() { - ColumnPool::singleton()->clear_columns(); -} - -template -struct HasColumnPool : public std::bool_constant, ColumnPoolList>::value> {}; - -namespace detail { -struct ClearColumnPool { - template - void operator()() { - Pool::singleton()->clear_columns(); - } -}; -} // namespace detail - -inline void TEST_clear_all_columns_this_thread() { - ForEach(detail::ClearColumnPool()); -} - -} // namespace starrocks diff --git a/be/src/common/config.h b/be/src/common/config.h index 6ce587c8042db..41ac3fe3b3625 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -282,8 +282,6 @@ CONF_mBool(enable_bitmap_index_memory_page_cache, "false"); CONF_mBool(enable_zonemap_index_memory_page_cache, "false"); // whether to enable the ordinal index memory cache CONF_mBool(enable_ordinal_index_memory_page_cache, "false"); -// whether to disable column pool -CONF_Bool(disable_column_pool, "true"); CONF_mInt32(base_compaction_check_interval_seconds, "60"); CONF_mInt64(min_base_compaction_num_singleton_deltas, "5"); diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index d1c10514f4eb8..707de05d9defb 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -38,7 +38,6 @@ #include "block_cache/block_cache.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "common/minidump.h" #include "exec/workgroup/work_group.h" @@ -85,36 +84,6 @@ std::atomic k_starrocks_exit = false; // but also waiting for all threads to exit gracefully. std::atomic k_starrocks_exit_quick = false; -class ReleaseColumnPool { -public: - explicit ReleaseColumnPool(double ratio) : _ratio(ratio) {} - - template - void operator()() { - _freed_bytes += Pool::singleton()->release_free_columns(_ratio); - } - - size_t freed_bytes() const { return _freed_bytes; } - -private: - double _ratio; - size_t _freed_bytes = 0; -}; - -void gc_memory(void* arg_this) { - using namespace starrocks; - const static float kFreeRatio = 0.5; - - auto* daemon = static_cast(arg_this); - while (!daemon->stopped()) { - nap_sleep(config::memory_maintenance_sleep_time_s, [daemon] { return daemon->stopped(); }); - - ReleaseColumnPool releaser(kFreeRatio); - ForEach(releaser); - LOG_IF(INFO, releaser.freed_bytes() > 0) << "Released " << releaser.freed_bytes() << " bytes from column pool"; - } -} - /* * This thread will calculate some metrics at a fix interval(15 sec) * 1. push bytes per second @@ -198,15 +167,15 @@ void calculate_metrics(void* arg_this) { LOG(INFO) << fmt::format( "Current memory statistics: process({}), query_pool({}), load({}), " - "metadata({}), compaction({}), schema_change({}), column_pool({}), " + "metadata({}), compaction({}), schema_change({}), " "page_cache({}), update({}), chunk_allocator({}), clone({}), consistency({}), " "datacache({}), jit({})", mem_metrics->process_mem_bytes.value(), mem_metrics->query_mem_bytes.value(), mem_metrics->load_mem_bytes.value(), mem_metrics->metadata_mem_bytes.value(), mem_metrics->compaction_mem_bytes.value(), mem_metrics->schema_change_mem_bytes.value(), - mem_metrics->column_pool_mem_bytes.value(), mem_metrics->storage_page_cache_mem_bytes.value(), - mem_metrics->update_mem_bytes.value(), mem_metrics->chunk_allocator_mem_bytes.value(), - mem_metrics->clone_mem_bytes.value(), mem_metrics->consistency_mem_bytes.value(), datacache_mem_bytes, + mem_metrics->storage_page_cache_mem_bytes.value(), mem_metrics->update_mem_bytes.value(), + mem_metrics->chunk_allocator_mem_bytes.value(), mem_metrics->clone_mem_bytes.value(), + mem_metrics->consistency_mem_bytes.value(), datacache_mem_bytes, mem_metrics->jit_cache_mem_bytes.value()); nap_sleep(15, [daemon] { return daemon->stopped(); }); @@ -307,10 +276,6 @@ void Daemon::init(bool as_cn, const std::vector& paths) { TimezoneUtils::init_time_zones(); - std::thread gc_thread(gc_memory, this); - Thread::set_thread_name(gc_thread, "gc_daemon"); - _daemon_threads.emplace_back(std::move(gc_thread)); - init_starrocks_metrics(paths); if (config::enable_metric_calculator) { diff --git a/be/src/connector/lake_connector.cpp b/be/src/connector/lake_connector.cpp index 7270ecd76c926..75b803ce5eb45 100644 --- a/be/src/connector/lake_connector.cpp +++ b/be/src/connector/lake_connector.cpp @@ -133,8 +133,7 @@ void LakeDataSource::close(RuntimeState* state) { } Status LakeDataSource::get_next(RuntimeState* state, ChunkPtr* chunk) { - chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size(), - _runtime_state->use_column_pool())); + chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size())); auto* chunk_ptr = chunk->get(); do { diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp index d394bab24192f..de5cbeea54934 100644 --- a/be/src/exec/olap_scan_node.cpp +++ b/be/src/exec/olap_scan_node.cpp @@ -20,7 +20,6 @@ #include #include "column/column_access_path.h" -#include "column/column_pool.h" #include "column/type_traits.h" #include "common/compiler_util.h" #include "common/status.h" @@ -159,7 +158,6 @@ Status OlapScanNode::get_next(RuntimeState* state, ChunkPtr* chunk, bool* eos) { RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::GETNEXT)); SCOPED_TIMER(_runtime_profile->total_time_counter()); - bool first_call = !_start; if (!_start && _status.ok()) { Status status = _start_scan(state); _update_status(status); @@ -210,7 +208,7 @@ Status OlapScanNode::get_next(RuntimeState* state, ChunkPtr* chunk, bool* eos) { // is the first time of calling `get_next`, pass the second argument of `_fill_chunk_pool` as // true to ensure that the newly allocated column objects will be returned back into the column // pool. - TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(1, first_call && state->use_column_pool())); + TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(1)); eval_join_runtime_filters(chunk); _num_rows_returned += (*chunk)->num_rows(); COUNTER_SET(_rows_returned_counter, _num_rows_returned); @@ -256,11 +254,6 @@ void OlapScanNode::close(RuntimeState* state) { chunk.reset(); } - if (runtime_state() != nullptr) { - // Reduce the memory usage if the the average string size is greater than 512. - release_large_columns(runtime_state()->chunk_size() * 512); - } - for (const auto& rowsets_per_tablet : _tablet_rowsets) { Rowset::release_readers(rowsets_per_tablet); } @@ -275,10 +268,10 @@ OlapScanNode::~OlapScanNode() { DCHECK(is_closed()); } -void OlapScanNode::_fill_chunk_pool(int count, bool force_column_pool) { +void OlapScanNode::_fill_chunk_pool(int count) { const size_t capacity = runtime_state()->chunk_size(); for (int i = 0; i < count; i++) { - ChunkPtr chunk(ChunkHelper::new_chunk_pooled(*_chunk_schema, capacity, force_column_pool)); + ChunkPtr chunk(ChunkHelper::new_chunk_pooled(*_chunk_schema, capacity)); { std::lock_guard l(_mtx); _chunk_pool.push(std::move(chunk)); @@ -736,7 +729,7 @@ Status OlapScanNode::_start_scan_thread(RuntimeState* state) { COUNTER_SET(_task_concurrency, (int64_t)concurrency); int chunks = _chunks_per_scanner * concurrency; _chunk_pool.reserve(chunks); - TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(chunks, state->use_column_pool())); + TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(chunks)); std::lock_guard l(_mtx); for (int i = 0; i < concurrency; i++) { CHECK(_submit_scanner(_pending_scanners.pop(), true)); diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h index f2a9c1174df27..e087bb27a4304 100644 --- a/be/src/exec/olap_scan_node.h +++ b/be/src/exec/olap_scan_node.h @@ -150,7 +150,7 @@ class OlapScanNode final : public starrocks::ScanNode { void _update_status(const Status& status); Status _get_status(); - void _fill_chunk_pool(int count, bool force_column_pool); + void _fill_chunk_pool(int count); bool _submit_scanner(TabletScanner* scanner, bool blockable); void _close_pending_scanners(); diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.cpp b/be/src/exec/pipeline/scan/olap_chunk_source.cpp index 62e7aa2a49193..eca5e61a932f7 100644 --- a/be/src/exec/pipeline/scan/olap_chunk_source.cpp +++ b/be/src/exec/pipeline/scan/olap_chunk_source.cpp @@ -541,8 +541,7 @@ Status OlapChunkSource::_init_olap_reader(RuntimeState* runtime_state) { } Status OlapChunkSource::_read_chunk(RuntimeState* state, ChunkPtr* chunk) { - chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size(), - _runtime_state->use_column_pool())); + chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size())); auto scope = IOProfiler::scope(IOProfiler::TAG_QUERY, _tablet->tablet_id()); return _read_chunk_from_storage(_runtime_state, (*chunk).get()); } diff --git a/be/src/exec/tablet_scanner.cpp b/be/src/exec/tablet_scanner.cpp index cd537925962b9..1908e98fce8e3 100644 --- a/be/src/exec/tablet_scanner.cpp +++ b/be/src/exec/tablet_scanner.cpp @@ -17,7 +17,6 @@ #include #include -#include "column/column_pool.h" #include "column/vectorized_fwd.h" #include "common/status.h" #include "exec/olap_scan_node.h" @@ -118,8 +117,6 @@ void TabletScanner::close(RuntimeState* state) { _reader.reset(); _predicate_free_pool.clear(); Expr::close(_conjunct_ctxs, state); - // Reduce the memory usage if the the average string size is greater than 512. - release_large_columns(state->chunk_size() * 512); _is_closed = true; } diff --git a/be/src/http/action/memory_metrics_action.cpp b/be/src/http/action/memory_metrics_action.cpp index d442bb85371bc..20b80c783dc32 100644 --- a/be/src/http/action/memory_metrics_action.cpp +++ b/be/src/http/action/memory_metrics_action.cpp @@ -49,7 +49,6 @@ void MemoryMetricsAction::handle(HttpRequest* req) { "bloom_filter_index", "compaction", "schema_change", - "column_pool", "page_cache", "datacache", "update", diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index 14a6108586c28..65e73518ff11a 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -140,9 +140,6 @@ void mem_tracker_handler(MemTracker* mem_tracker, const WebPageHandler::Argument } else if (iter->second == "clone") { start_mem_tracker = GlobalEnv::GetInstance()->clone_mem_tracker(); cur_level = 2; - } else if (iter->second == "column_pool") { - start_mem_tracker = GlobalEnv::GetInstance()->column_pool_mem_tracker(); - cur_level = 2; } else if (iter->second == "page_cache") { start_mem_tracker = GlobalEnv::GetInstance()->page_cache_mem_tracker(); cur_level = 2; diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp index d1f9a9f7dee17..a918c4942e8cc 100644 --- a/be/src/runtime/exec_env.cpp +++ b/be/src/runtime/exec_env.cpp @@ -41,7 +41,6 @@ #include "agent/agent_server.h" #include "agent/master_info.h" #include "block_cache/block_cache.h" -#include "column/column_pool.h" #include "common/config.h" #include "common/configbase.h" #include "common/logging.h" @@ -234,7 +233,6 @@ Status GlobalEnv::_init_mem_tracker() { int64_t compaction_mem_limit = calc_max_compaction_memory(_process_mem_tracker->limit()); _compaction_mem_tracker = regist_tracker(compaction_mem_limit, "compaction", _process_mem_tracker.get()); _schema_change_mem_tracker = regist_tracker(-1, "schema_change", _process_mem_tracker.get()); - _column_pool_mem_tracker = regist_tracker(-1, "column_pool", _process_mem_tracker.get()); _page_cache_mem_tracker = regist_tracker(-1, "page_cache", _process_mem_tracker.get()); _jit_cache_mem_tracker = regist_tracker(-1, "jit_cache", _process_mem_tracker.get()); int32_t update_mem_percent = std::max(std::min(100, config::update_memory_limit_percent), 0); @@ -248,8 +246,6 @@ Status GlobalEnv::_init_mem_tracker() { MemChunkAllocator::init_instance(_chunk_allocator_mem_tracker.get(), config::chunk_reserved_bytes_limit); - SetMemTrackerForColumnPool op(_column_pool_mem_tracker); - ForEach(op); _init_storage_page_cache(); // TODO: move to StorageEngine return Status::OK(); } diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 402ee0b93e457..3cf3cb546c895 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -139,7 +139,6 @@ class GlobalEnv { MemTracker* short_key_index_mem_tracker() { return _short_key_index_mem_tracker.get(); } MemTracker* compaction_mem_tracker() { return _compaction_mem_tracker.get(); } MemTracker* schema_change_mem_tracker() { return _schema_change_mem_tracker.get(); } - MemTracker* column_pool_mem_tracker() { return _column_pool_mem_tracker.get(); } MemTracker* page_cache_mem_tracker() { return _page_cache_mem_tracker.get(); } MemTracker* jit_cache_mem_tracker() { return _jit_cache_mem_tracker.get(); } MemTracker* update_mem_tracker() { return _update_mem_tracker.get(); } @@ -199,9 +198,6 @@ class GlobalEnv { // The memory used for schema change std::shared_ptr _schema_change_mem_tracker; - // The memory used for column pool - std::shared_ptr _column_pool_mem_tracker; - // The memory used for page cache std::shared_ptr _page_cache_mem_tracker; diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 2875ecf5fa9e0..5beac56553b93 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -950,7 +950,6 @@ Status FragmentMgr::exec_external_plan_fragment(const TScanOpenParams& params, c query_options.query_type = TQueryType::EXTERNAL; // For spark sql / flink sql, we dont use page cache. query_options.use_page_cache = false; - query_options.use_column_pool = false; query_options.enable_profile = config::enable_profile_for_external_plan; exec_fragment_params.__set_query_options(query_options); VLOG_ROW << "external exec_plan_fragment params is " diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 45d83eff9ff18..a2dd0bb3f2137 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -287,17 +287,6 @@ bool RuntimeState::use_page_cache() { return true; } -bool RuntimeState::use_column_pool() const { - if (config::disable_column_pool) { - return false; - } - - if (_query_options.__isset.use_column_pool) { - return _query_options.use_column_pool; - } - return true; -} - Status RuntimeState::set_mem_limit_exceeded(MemTracker* tracker, int64_t failed_allocation_size, std::string_view msg) { DCHECK_GE(failed_allocation_size, 0); { diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index c557b9e35c103..f2cae96ffc397 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -131,7 +131,6 @@ class RuntimeState { void set_desc_tbl(DescriptorTbl* desc_tbl) { _desc_tbl = desc_tbl; } int chunk_size() const { return _query_options.batch_size; } void set_chunk_size(int chunk_size) { _query_options.batch_size = chunk_size; } - bool use_column_pool() const; bool abort_on_default_limit_exceeded() const { return _query_options.abort_on_default_limit_exceeded; } int64_t timestamp_ms() const { return _timestamp_us / 1000; } int64_t timestamp_us() const { return _timestamp_us; } diff --git a/be/src/script/script.cpp b/be/src/script/script.cpp index 157839fcffae1..40d30f2d40b78 100644 --- a/be/src/script/script.cpp +++ b/be/src/script/script.cpp @@ -222,7 +222,6 @@ void bind_exec_env(ForeignModule& m) { REG_METHOD(GlobalEnv, metadata_mem_tracker); REG_METHOD(GlobalEnv, compaction_mem_tracker); REG_METHOD(GlobalEnv, schema_change_mem_tracker); - REG_METHOD(GlobalEnv, column_pool_mem_tracker); REG_METHOD(GlobalEnv, page_cache_mem_tracker); REG_METHOD(GlobalEnv, jit_cache_mem_tracker); REG_METHOD(GlobalEnv, update_mem_tracker); diff --git a/be/src/storage/chunk_helper.cpp b/be/src/storage/chunk_helper.cpp index be635e55aa3d2..2679634ac3daa 100644 --- a/be/src/storage/chunk_helper.cpp +++ b/be/src/storage/chunk_helper.cpp @@ -17,7 +17,6 @@ #include "column/array_column.h" #include "column/chunk.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "column/json_column.h" #include "column/map_column.h" #include "column/schema.h" @@ -230,53 +229,34 @@ ColumnId ChunkHelper::max_column_id(const starrocks::Schema& schema) { } template -struct ColumnDeleter { - ColumnDeleter(size_t chunk_size) : chunk_size(chunk_size) {} - void operator()(Column* ptr) const { return_column(down_cast(ptr), chunk_size); } - size_t chunk_size; -}; - -template -inline std::shared_ptr get_column_ptr(size_t chunk_size) { - if constexpr (std::negation_v>) { - return std::make_shared(); - } else { - T* ptr = get_column(); - if (LIKELY(ptr != nullptr)) { - return std::shared_ptr(ptr, ColumnDeleter(chunk_size)); - } else { - return std::make_shared(); - } - } +inline std::shared_ptr get_column_ptr() { + return std::make_shared(); } -template -inline std::shared_ptr> get_decimal_column_ptr(int precision, int scale, size_t chunk_size) { - auto column = get_column_ptr(chunk_size); +template +inline std::shared_ptr> get_decimal_column_ptr(int precision, int scale) { + auto column = get_column_ptr(); column->set_precision(precision); column->set_scale(scale); return column; } -template struct ColumnPtrBuilder { template - ColumnPtr operator()(size_t chunk_size, const Field& field, int precision, int scale) { + ColumnPtr operator()(const Field& field, int precision, int scale) { auto NullableIfNeed = [&](ColumnPtr c) -> ColumnPtr { - return field.is_nullable() - ? NullableColumn::create(std::move(c), get_column_ptr(chunk_size)) - : c; + return field.is_nullable() ? NullableColumn::create(std::move(c), get_column_ptr()) : c; }; if constexpr (ftype == TYPE_ARRAY) { auto elements = NullableColumn::wrap_if_necessary(field.sub_field(0).create_column()); - auto offsets = get_column_ptr(chunk_size); + auto offsets = get_column_ptr(); auto array = ArrayColumn::create(std::move(elements), offsets); return NullableIfNeed(array); } else if constexpr (ftype == TYPE_MAP) { auto keys = NullableColumn::wrap_if_necessary(field.sub_field(0).create_column()); auto values = NullableColumn::wrap_if_necessary(field.sub_field(1).create_column()); - auto offsets = get_column_ptr(chunk_size); + auto offsets = get_column_ptr(); auto map = MapColumn::create(std::move(keys), std::move(values), offsets); return NullableIfNeed(map); } else if constexpr (ftype == TYPE_STRUCT) { @@ -291,33 +271,31 @@ struct ColumnPtrBuilder { } else { switch (ftype) { case TYPE_DECIMAL32: - return NullableIfNeed(get_decimal_column_ptr(precision, scale, chunk_size)); + return NullableIfNeed(get_decimal_column_ptr(precision, scale)); case TYPE_DECIMAL64: - return NullableIfNeed(get_decimal_column_ptr(precision, scale, chunk_size)); + return NullableIfNeed(get_decimal_column_ptr(precision, scale)); case TYPE_DECIMAL128: - return NullableIfNeed(get_decimal_column_ptr(precision, scale, chunk_size)); + return NullableIfNeed(get_decimal_column_ptr(precision, scale)); default: { - return NullableIfNeed(get_column_ptr::ColumnType, force>(chunk_size)); + return NullableIfNeed(get_column_ptr::ColumnType>()); } } } } }; -template -ColumnPtr column_from_pool(const Field& field, size_t chunk_size) { +ColumnPtr column_from_pool(const Field& field) { auto precision = field.type()->precision(); auto scale = field.type()->scale(); - return field_type_dispatch_column(field.type()->type(), ColumnPtrBuilder(), chunk_size, field, precision, - scale); + return field_type_dispatch_column(field.type()->type(), ColumnPtrBuilder(), field, precision, scale); } -Chunk* ChunkHelper::new_chunk_pooled(const Schema& schema, size_t chunk_size, bool force) { +Chunk* ChunkHelper::new_chunk_pooled(const Schema& schema, size_t chunk_size) { Columns columns; columns.reserve(schema.num_fields()); for (size_t i = 0; i < schema.num_fields(); i++) { const FieldPtr& f = schema.field(i); - auto column = force ? column_from_pool(*f, chunk_size) : column_from_pool(*f, chunk_size); + auto column = column_from_pool(*f); column->reserve(chunk_size); columns.emplace_back(std::move(column)); } diff --git a/be/src/storage/chunk_helper.h b/be/src/storage/chunk_helper.h index 94c0a27234cfc..2e90041793df8 100644 --- a/be/src/storage/chunk_helper.h +++ b/be/src/storage/chunk_helper.h @@ -69,7 +69,7 @@ class ChunkHelper { // Create an empty chunk according to the |slots| and reserve it of size |n|. static ChunkUniquePtr new_chunk(const std::vector& slots, size_t n); - static Chunk* new_chunk_pooled(const Schema& schema, size_t n, bool force); + static Chunk* new_chunk_pooled(const Schema& schema, size_t n); // Create a vectorized column from field . // REQUIRE: |type| must be scalar type. diff --git a/be/src/testutil/init_test_env.h b/be/src/testutil/init_test_env.h index 799f6eb4806d2..ce6830b477b0a 100644 --- a/be/src/testutil/init_test_env.h +++ b/be/src/testutil/init_test_env.h @@ -16,7 +16,6 @@ #include "butil/file_util.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "gtest/gtest.h" @@ -110,7 +109,6 @@ int init_test_env(int argc, char** argv) { // clear some trash objects kept in tablet_manager so mem_tracker checks will not fail CHECK(StorageEngine::instance()->tablet_manager()->start_trash_sweep().ok()); (void)butil::DeleteFile(storage_root, true); - TEST_clear_all_columns_this_thread(); // delete engine StorageEngine::instance()->stop(); // destroy exec env diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index 439fb19a864e8..aee876478a4a9 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -40,10 +40,10 @@ #include #include -#include "column/column_pool.h" #include "gutil/strings/split.h" // for string split #include "gutil/strtoint.h" // for atoi64 #include "jemalloc/jemalloc.h" +#include "runtime/mem_tracker.h" #include "runtime/runtime_filter_worker.h" #include "util/metrics.h" @@ -276,7 +276,6 @@ void SystemMetrics::_install_memory_metrics(MetricRegistry* registry) { registry->register_metric("short_key_index_mem_bytes", &_memory_metrics->short_key_index_mem_bytes); registry->register_metric("compaction_mem_bytes", &_memory_metrics->compaction_mem_bytes); registry->register_metric("schema_change_mem_bytes", &_memory_metrics->schema_change_mem_bytes); - registry->register_metric("column_pool_mem_bytes", &_memory_metrics->column_pool_mem_bytes); registry->register_metric("storage_page_cache_mem_bytes", &_memory_metrics->storage_page_cache_mem_bytes); registry->register_metric("jit_cache_mem_bytes", &_memory_metrics->jit_cache_mem_bytes); registry->register_metric("update_mem_bytes", &_memory_metrics->update_mem_bytes); @@ -284,22 +283,6 @@ void SystemMetrics::_install_memory_metrics(MetricRegistry* registry) { registry->register_metric("clone_mem_bytes", &_memory_metrics->clone_mem_bytes); registry->register_metric("consistency_mem_bytes", &_memory_metrics->consistency_mem_bytes); registry->register_metric("datacache_mem_bytes", &_memory_metrics->datacache_mem_bytes); - - registry->register_metric("total_column_pool_bytes", &_memory_metrics->column_pool_total_bytes); - registry->register_metric("local_column_pool_bytes", &_memory_metrics->column_pool_local_bytes); - registry->register_metric("central_column_pool_bytes", &_memory_metrics->column_pool_central_bytes); - registry->register_metric("binary_column_pool_bytes", &_memory_metrics->column_pool_binary_bytes); - registry->register_metric("uint8_column_pool_bytes", &_memory_metrics->column_pool_uint8_bytes); - registry->register_metric("int8_column_pool_bytes", &_memory_metrics->column_pool_int8_bytes); - registry->register_metric("int16_column_pool_bytes", &_memory_metrics->column_pool_int16_bytes); - registry->register_metric("int32_column_pool_bytes", &_memory_metrics->column_pool_int32_bytes); - registry->register_metric("int64_column_pool_bytes", &_memory_metrics->column_pool_int64_bytes); - registry->register_metric("int128_column_pool_bytes", &_memory_metrics->column_pool_int128_bytes); - registry->register_metric("float_column_pool_bytes", &_memory_metrics->column_pool_float_bytes); - registry->register_metric("double_column_pool_bytes", &_memory_metrics->column_pool_double_bytes); - registry->register_metric("decimal_column_pool_bytes", &_memory_metrics->column_pool_decimal_bytes); - registry->register_metric("date_column_pool_bytes", &_memory_metrics->column_pool_date_bytes); - registry->register_metric("datetime_column_pool_bytes", &_memory_metrics->column_pool_datetime_bytes); } void SystemMetrics::_update_memory_metrics() { @@ -362,29 +345,9 @@ void SystemMetrics::_update_memory_metrics() { SET_MEM_METRIC_VALUE(update_mem_tracker, update_mem_bytes) SET_MEM_METRIC_VALUE(chunk_allocator_mem_tracker, chunk_allocator_mem_bytes) SET_MEM_METRIC_VALUE(clone_mem_tracker, clone_mem_bytes) - SET_MEM_METRIC_VALUE(column_pool_mem_tracker, column_pool_mem_bytes) SET_MEM_METRIC_VALUE(consistency_mem_tracker, consistency_mem_bytes) SET_MEM_METRIC_VALUE(datacache_mem_tracker, datacache_mem_bytes) #undef SET_MEM_METRIC_VALUE - -#define UPDATE_COLUMN_POOL_METRIC(var, type) \ - value = describe_column_pool().central_free_bytes; \ - var.set_value(value); - - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_binary_bytes, BinaryColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_uint8_bytes, UInt8Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int8_bytes, Int8Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int16_bytes, Int16Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int32_bytes, Int32Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int64_bytes, Int64Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int128_bytes, Int128Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_float_bytes, FloatColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_double_bytes, DoubleColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_decimal_bytes, DecimalColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_date_bytes, DateColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_datetime_bytes, TimestampColumn) - -#undef UPDATE_COLUMN_POOL_METRIC } void SystemMetrics::_install_disk_metrics(MetricRegistry* registry, const std::set& devices) { diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h index 38652de6f8182..eb96a97160986 100644 --- a/be/src/util/system_metrics.h +++ b/be/src/util/system_metrics.h @@ -62,7 +62,6 @@ class MemoryMetrics { METRIC_DEFINE_INT_GAUGE(short_key_index_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(compaction_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(schema_change_mem_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(storage_page_cache_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(jit_cache_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(update_mem_bytes, MetricUnit::BYTES); @@ -70,23 +69,6 @@ class MemoryMetrics { METRIC_DEFINE_INT_GAUGE(clone_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(consistency_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(datacache_mem_bytes, MetricUnit::BYTES); - - // column pool metrics. - METRIC_DEFINE_INT_GAUGE(column_pool_total_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_local_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_central_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_binary_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_uint8_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int8_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int16_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int32_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int64_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int128_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_float_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_double_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_decimal_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_date_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_datetime_bytes, MetricUnit::BYTES); }; class SystemMetrics { diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 2e6b30a8e1f16..2e51b5e5eb6a3 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -6,7 +6,6 @@ set(EXEC_FILES ./column/binary_column_test.cpp ./column/chunk_test.cpp ./column/column_helper_test.cpp - ./column/column_pool_test.cpp ./column/const_column_test.cpp ./column/date_value_test.cpp ./column/decimalv3_column_test.cpp diff --git a/be/test/column/column_pool_test.cpp b/be/test/column/column_pool_test.cpp deleted file mode 100644 index e10a8bb9afd88..0000000000000 --- a/be/test/column/column_pool_test.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "column/column_pool.h" - -#include "gtest/gtest.h" - -namespace starrocks { - -class ColumnPoolTest : public ::testing::Test { -protected: - void SetUp() override { TEST_clear_all_columns_this_thread(); } - void TearDown() override { TEST_clear_all_columns_this_thread(); } -}; - -// NOLINTNEXTLINE -TEST_F(ColumnPoolTest, mem_statistics) { - auto* pool = ColumnPool::singleton(); - std::string str(20, '1'); - auto* c1 = get_column(); - c1->reserve(config::vector_chunk_size); - for (size_t i = 0; i < config::vector_chunk_size; i++) { - c1->append_string(str); - } - - pool->return_column(c1, config::vector_chunk_size); - size_t usage_1 = pool->mem_tracker()->consumption(); - - pool->release_large_columns(config::vector_chunk_size * 10); - size_t usage_2 = pool->mem_tracker()->consumption(); - - ASSERT_EQ(usage_1 - usage_2, 81920); -} - -// NOLINTNEXTLINE -TEST_F(ColumnPoolTest, single_thread) { - auto c1 = get_column(); - ASSERT_EQ(0u, c1->get_data().capacity()); - - c1->reserve(3); - c1->append_datum(Datum((int32_t)1)); - c1->append_datum(Datum((int32_t)2)); - c1->set_delete_state(DEL_PARTIAL_SATISFIED); - return_column(c1, config::vector_chunk_size); - - auto c2 = get_column(); - ASSERT_EQ(c1, c2); - ASSERT_EQ(0u, c2->size()); - ASSERT_EQ(DEL_NOT_SATISFIED, c2->delete_state()); - ASSERT_EQ(3, c2->get_data().capacity()); - - auto c3 = get_column(); - ASSERT_NE(c2, c3); - - auto c4 = get_column(); - ASSERT_NE(c3, c4); - - auto c5 = get_column(); - ASSERT_NE(c4, c5); - - auto c6 = get_column(); - ASSERT_NE(c5, c6); - - return_column(c6, config::vector_chunk_size); - return_column(c5, config::vector_chunk_size); - - auto c7 = get_column(); - auto c8 = get_column(); - ASSERT_EQ(c5, c7); - ASSERT_EQ(c6, c8); - - return_column(c8, config::vector_chunk_size); - return_column(c7, config::vector_chunk_size); - - delete c2; - delete c3; - delete c4; -} - -} // namespace starrocks diff --git a/be/test/storage/aggregate_iterator_test.cpp b/be/test/storage/aggregate_iterator_test.cpp index 922a26e949670..daa0ed77a814f 100644 --- a/be/test/storage/aggregate_iterator_test.cpp +++ b/be/test/storage/aggregate_iterator_test.cpp @@ -14,7 +14,7 @@ #include "storage/aggregate_iterator.h" -#include "column/column_pool.h" +#include "common/config.h" #include "gtest/gtest.h" #include "storage/aggregate_type.h" #include "storage/vector_chunk_iterator.h" @@ -36,7 +36,7 @@ static std::vector row(const Chunk& chunk, size_t row_id) { class AggregateIteratorTest : public testing::Test { protected: void SetUp() override {} - void TearDown() override { TEST_clear_all_columns_this_thread(); } + void TearDown() override {} }; // NOLINTNEXTLINE diff --git a/be/test/storage/projection_iterator_test.cpp b/be/test/storage/projection_iterator_test.cpp index 03081ec6200c7..e090f18d8c6f8 100644 --- a/be/test/storage/projection_iterator_test.cpp +++ b/be/test/storage/projection_iterator_test.cpp @@ -18,8 +18,8 @@ #include "column/chunk.h" #include "column/column.h" -#include "column/column_pool.h" #include "column/datum.h" +#include "common/config.h" #include "storage/chunk_helper.h" namespace starrocks { @@ -67,7 +67,7 @@ class VectorIterator final : public ChunkIterator { class ProjectionIteratorTest : public testing::Test { protected: void SetUp() override {} - void TearDown() override { TEST_clear_all_columns_this_thread(); } + void TearDown() override {} }; // NOLINTNEXTLINE diff --git a/be/test/storage/publish_version_manager_test.cpp b/be/test/storage/publish_version_manager_test.cpp index 9f23978ec7e1a..9af7839ab3519 100644 --- a/be/test/storage/publish_version_manager_test.cpp +++ b/be/test/storage/publish_version_manager_test.cpp @@ -21,7 +21,6 @@ #include "agent/publish_version.h" #include "butil/file_util.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "fs/fs_util.h" diff --git a/be/test/storage/publish_version_task_test.cpp b/be/test/storage/publish_version_task_test.cpp index 1e40784d656db..9b6b1a683e875 100644 --- a/be/test/storage/publish_version_task_test.cpp +++ b/be/test/storage/publish_version_task_test.cpp @@ -19,7 +19,6 @@ #include "agent/publish_version.h" #include "butil/file_util.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "fs/fs_util.h" diff --git a/be/test/storage/task/engine_storage_migration_task_test.cpp b/be/test/storage/task/engine_storage_migration_task_test.cpp index f9cc364bc2293..78ff38624ac45 100644 --- a/be/test/storage/task/engine_storage_migration_task_test.cpp +++ b/be/test/storage/task/engine_storage_migration_task_test.cpp @@ -17,7 +17,6 @@ #include #include "butil/file_util.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "fs/fs_util.h" @@ -663,7 +662,6 @@ int main(int argc, char** argv) { // clear some trash objects kept in tablet_manager so mem_tracker checks will not fail starrocks::StorageEngine::instance()->tablet_manager()->start_trash_sweep(); starrocks::fs::remove_all(storage_root.value()); - starrocks::TEST_clear_all_columns_this_thread(); // delete engine engine->stop(); delete engine; diff --git a/be/test/storage/union_iterator_test.cpp b/be/test/storage/union_iterator_test.cpp index 0808efe321aff..b53285efb632c 100644 --- a/be/test/storage/union_iterator_test.cpp +++ b/be/test/storage/union_iterator_test.cpp @@ -20,9 +20,9 @@ #include #include "column/chunk.h" -#include "column/column_pool.h" #include "column/fixed_length_column.h" #include "column/schema.h" +#include "common/config.h" #include "storage/chunk_helper.h" namespace starrocks { @@ -30,7 +30,7 @@ namespace starrocks { class UnionIteratorTest : public testing::Test { protected: void SetUp() override {} - void TearDown() override { TEST_clear_all_columns_this_thread(); } + void TearDown() override {} // return chunk with single column of type int32_t. class IntIterator final : public ChunkIterator { diff --git a/gensrc/thrift/InternalService.thrift b/gensrc/thrift/InternalService.thrift index dd0f3d5fb372a..6dc55fa7bd01a 100644 --- a/gensrc/thrift/InternalService.thrift +++ b/gensrc/thrift/InternalService.thrift @@ -281,7 +281,7 @@ struct TQueryOptions { 103: optional i32 interleaving_group_size; 104: optional TOverflowMode overflow_mode = TOverflowMode.OUTPUT_NULL; - 105: optional bool use_column_pool = true; + 105: optional bool use_column_pool = true; // Deprecated // Deprecated 106: optional bool enable_agg_spill_preaggregation; 107: optional i64 global_runtime_filter_build_max_size;