From 37d72ac1ccf5e9cdd8d4a69bbb06da2c0e5048ce Mon Sep 17 00:00:00 2001 From: trueeyu Date: Fri, 20 Sep 2024 20:15:09 +0800 Subject: [PATCH] [Refactor] Remove column pool (#51105) Why I'm doing: ColumnPool is designed to address performance issues caused by frequent cross thread memory alloc/free. Tcmalloc has a CentrlHeap, which results in high lock contention overhead. Currently it has been already been replaced by Jemalloc, which is now an independent arena and no longer has this issue. In version 3.2, ColumnPool` has been disabled by default, and there have been no performance issues in performance testing or user production environments. So in main branch, it can be safely removed. What I'm doing: Remove column pool Signed-off-by: trueeyu --- be/src/column/column_pool.h | 453 ------------------ be/src/common/config.h | 2 - be/src/common/daemon.cpp | 43 +- be/src/connector/lake_connector.cpp | 3 +- be/src/exec/olap_scan_node.cpp | 15 +- be/src/exec/olap_scan_node.h | 2 +- .../exec/pipeline/scan/olap_chunk_source.cpp | 3 +- be/src/exec/tablet_scanner.cpp | 3 - be/src/http/action/memory_metrics_action.cpp | 1 - be/src/http/default_path_handlers.cpp | 3 - be/src/runtime/exec_env.cpp | 4 - be/src/runtime/exec_env.h | 4 - be/src/runtime/fragment_mgr.cpp | 1 - be/src/runtime/runtime_state.cpp | 11 - be/src/runtime/runtime_state.h | 1 - be/src/script/script.cpp | 1 - be/src/storage/chunk_helper.cpp | 56 +-- be/src/storage/chunk_helper.h | 2 +- be/src/testutil/init_test_env.h | 2 - be/src/util/system_metrics.cpp | 39 +- be/src/util/system_metrics.h | 18 - be/test/CMakeLists.txt | 1 - be/test/column/column_pool_test.cpp | 91 ---- be/test/storage/aggregate_iterator_test.cpp | 4 +- be/test/storage/projection_iterator_test.cpp | 4 +- .../storage/publish_version_manager_test.cpp | 1 - be/test/storage/publish_version_task_test.cpp | 1 - .../engine_storage_migration_task_test.cpp | 2 - be/test/storage/union_iterator_test.cpp | 4 +- gensrc/thrift/InternalService.thrift | 2 +- 30 files changed, 37 insertions(+), 740 deletions(-) delete mode 100644 be/src/column/column_pool.h delete mode 100644 be/test/column/column_pool_test.cpp diff --git a/be/src/column/column_pool.h b/be/src/column/column_pool.h deleted file mode 100644 index eaf632596b014..0000000000000 --- a/be/src/column/column_pool.h +++ /dev/null @@ -1,453 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include // NOLINT -#include - -#include - -#include "column/binary_column.h" -#include "column/const_column.h" -#include "column/decimalv3_column.h" -#include "column/fixed_length_column.h" -#include "column/object_column.h" -#include "common/compiler_util.h" -#include "common/config.h" -#include "common/type_list.h" -#include "gutil/dynamic_annotations.h" -#include "runtime/current_thread.h" -#include "util/json.h" - -namespace starrocks { - -template -struct ColumnPoolBlockSize { - static const size_t value = 256; -}; - -template <> -struct ColumnPoolBlockSize { - static const size_t value = 128; -}; - -template -int64_t column_bytes(const T* col) { - static_assert(std::is_base_of_v, "must_derived_of_Column"); - return col->memory_usage(); -} - -template -void reset_column(T* col) { - static_assert(std::is_base_of_v, "must_derived_of_Column"); - col->reset_column(); - DCHECK(col->size() == 0); - DCHECK(col->delete_state() == DEL_NOT_SATISFIED); -} - -// Returns the number of bytes freed to tcmalloc. -template -size_t release_column_if_large(T* col, size_t limit) { - auto old_usage = column_bytes(col); - if (old_usage < limit) { - return 0; - } - if constexpr (std::is_same_v) { - auto& bytes = col->get_bytes(); - BinaryColumn::Bytes tmp; - tmp.swap(bytes); - } else { - typename T::Container tmp; - tmp.swap(col->get_data()); - } - auto new_usage = column_bytes(col); - DCHECK_LT(new_usage, old_usage); - return old_usage - new_usage; -} - -template -size_t column_reserved_size(const T* col) { - if constexpr (std::is_same_v) { - const auto& offsets = col->get_offset(); - return offsets.capacity() > 0 ? offsets.capacity() - 1 : 0; - } else { - return col->get_data().capacity(); - } -} - -template -struct ColumnPoolFreeBlock { - int64_t nfree; - int64_t bytes; - T* ptrs[NITEM]; -}; - -struct ColumnPoolInfo { - size_t local_cnt = 0; - size_t central_free_items = 0; - size_t central_free_bytes = 0; -}; - -template -class CACHELINE_ALIGNED ColumnPool { - static constexpr size_t kBlockSize = ColumnPoolBlockSize::value; - - using FreeBlock = ColumnPoolFreeBlock; - using DynamicFreeBlock = ColumnPoolFreeBlock; - - class CACHELINE_ALIGNED LocalPool { - static_assert(std::is_base_of::value, "Must_be_derived_of_Column"); - - public: - explicit LocalPool(ColumnPool* pool) : _pool(pool) { - _curr_free.nfree = 0; - _curr_free.bytes = 0; - } - - ~LocalPool() noexcept { - if (_curr_free.nfree > 0 && !_pool->_push_free_block(_curr_free)) { - for (size_t i = 0; i < _curr_free.nfree; i++) { - ASAN_UNPOISON_MEMORY_REGION(_curr_free.ptrs[i], sizeof(T)); - delete _curr_free.ptrs[i]; - } - } - _pool->_clear_from_destructor_of_local_pool(); - } - - T* get_object() { - if (_curr_free.nfree == 0) { - if (!_pool->_pop_free_block(&_curr_free)) { - return nullptr; - } - } - T* obj = _curr_free.ptrs[--_curr_free.nfree]; - ASAN_UNPOISON_MEMORY_REGION(obj, sizeof(T)); - auto bytes = column_bytes(obj); - _curr_free.bytes -= bytes; - - tls_thread_status.mem_consume(bytes); - _pool->mem_tracker()->release(bytes); - - return obj; - } - - void return_object(T* ptr, size_t chunk_size) { - if (UNLIKELY(column_reserved_size(ptr) > chunk_size)) { - delete ptr; - return; - } - auto bytes = column_bytes(ptr); - if (_curr_free.nfree < kBlockSize) { - ASAN_POISON_MEMORY_REGION(ptr, sizeof(T)); - _curr_free.ptrs[_curr_free.nfree++] = ptr; - _curr_free.bytes += bytes; - - tls_thread_status.mem_release(bytes); - _pool->mem_tracker()->consume(bytes); - - return; - } - if (_pool->_push_free_block(_curr_free)) { - ASAN_POISON_MEMORY_REGION(ptr, sizeof(T)); - _curr_free.nfree = 1; - _curr_free.ptrs[0] = ptr; - _curr_free.bytes = bytes; - - tls_thread_status.mem_release(bytes); - _pool->mem_tracker()->consume(bytes); - - return; - } - delete ptr; - } - - void release_large_columns(size_t limit) { - size_t freed_bytes = 0; - for (size_t i = 0; i < _curr_free.nfree; i++) { - ASAN_UNPOISON_MEMORY_REGION(_curr_free.ptrs[i], sizeof(T)); - freed_bytes += release_column_if_large(_curr_free.ptrs[i], limit); - ASAN_POISON_MEMORY_REGION(_curr_free.ptrs[i], sizeof(T)); - } - if (freed_bytes > 0) { - _curr_free.bytes -= freed_bytes; - tls_thread_status.mem_consume(freed_bytes); - _pool->mem_tracker()->release(freed_bytes); - } - } - - static void delete_local_pool(void* arg) { delete (LocalPool*)arg; } - - private: - ColumnPool* _pool; - FreeBlock _curr_free; - }; - -public: - void set_mem_tracker(std::shared_ptr mem_tracker) { _mem_tracker = std::move(mem_tracker); } - MemTracker* mem_tracker() { return _mem_tracker.get(); } - - static std::enable_if_t, ColumnPool*> singleton() { - static ColumnPool p; - return &p; - } - - template - T* get_column() { - LocalPool* lp = _get_or_new_local_pool(); - if (UNLIKELY(lp == nullptr)) { - return nullptr; - } - T* ptr = lp->get_object(); - if (ptr == nullptr && AllocOnEmpty) { - ptr = new (std::nothrow) T(); - } - return ptr; - } - - void return_column(T* ptr, size_t chunk_size) { - LocalPool* lp = _get_or_new_local_pool(); - if (LIKELY(lp != nullptr)) { - reset_column(ptr); - lp->return_object(ptr, chunk_size); - } else { - delete ptr; - } - } - - // Destroy some objects in the *central* free list. - // Returns the number of bytes freed to tcmalloc. - size_t release_free_columns(double free_ratio) { - free_ratio = std::min(free_ratio, 1.0); - int64_t now = butil::gettimeofday_s(); - std::vector tmp; - if (now - _first_push_time > 3) { - // ^^^^^^^^^^^^^^^^ read without lock by intention. - std::lock_guard l(_free_blocks_lock); - int n = implicit_cast(static_cast::type>( - static_cast(_free_blocks.size()) * (1 - free_ratio))); - tmp.insert(tmp.end(), _free_blocks.begin() + n, _free_blocks.end()); - _free_blocks.resize(n); - } - size_t freed_bytes = 0; - for (DynamicFreeBlock* blk : tmp) { - freed_bytes += blk->bytes; - _release_free_block(blk); - } - - _mem_tracker->release(freed_bytes); - tls_thread_status.mem_consume(freed_bytes); - - return freed_bytes; - } - - // Reduce memory usage on behalf of column if its memory usage is greater - // than or equal to |limit|. - void release_large_columns(size_t limit) { - LocalPool* lp = _local_pool; - if (lp) { - lp->release_large_columns(limit); - } - } - - void clear_columns() { - LocalPool* lp = _local_pool; - if (lp) { - _local_pool = nullptr; - butil::thread_atexit_cancel(LocalPool::delete_local_pool, lp); - delete lp; - } - } - - ColumnPoolInfo describe_column_pool() { - ColumnPoolInfo info; - info.local_cnt = _nlocal.load(std::memory_order_relaxed); - if (_free_blocks.empty()) { - return info; - } - std::lock_guard l(_free_blocks_lock); - for (DynamicFreeBlock* blk : _free_blocks) { - info.central_free_items += blk->nfree; - info.central_free_bytes += blk->bytes; - } - return info; - } - -private: - static void _release_free_block(DynamicFreeBlock* blk) { - for (size_t i = 0; i < blk->nfree; i++) { - T* p = blk->ptrs[i]; - ASAN_UNPOISON_MEMORY_REGION(p, sizeof(T)); - delete p; - } - free(blk); - } - - LocalPool* _get_or_new_local_pool() { - LocalPool* lp = _local_pool; - if (LIKELY(lp != nullptr)) { - return lp; - } - lp = new (std::nothrow) LocalPool(this); - if (nullptr == lp) { - return nullptr; - } - std::lock_guard l(_change_thread_mutex); //avoid race with clear_columns() - _local_pool = lp; - (void)butil::thread_atexit(LocalPool::delete_local_pool, lp); - _nlocal.fetch_add(1, std::memory_order_relaxed); - return lp; - } - - void _clear_from_destructor_of_local_pool() { - _local_pool = nullptr; - - // Do nothing if there are active threads. - if (_nlocal.fetch_sub(1, std::memory_order_relaxed) != 1) { - return; - } - - std::lock_guard l(_change_thread_mutex); // including acquire fence. - // Do nothing if there are active threads. - if (_nlocal.load(std::memory_order_relaxed) != 0) { - return; - } - // All threads exited and we're holding _change_thread_mutex to avoid - // racing with new threads calling get_column(). - - // Clear global free list. - _first_push_time = 0; - release_free_columns(1.0); - } - - bool _push_free_block(const FreeBlock& blk) { - auto* p = (DynamicFreeBlock*)malloc(offsetof(DynamicFreeBlock, ptrs) + sizeof(*blk.ptrs) * blk.nfree); - if (UNLIKELY(p == nullptr)) { - return false; - } - p->nfree = blk.nfree; - p->bytes = blk.bytes; - memcpy(p->ptrs, blk.ptrs, sizeof(*blk.ptrs) * blk.nfree); - std::lock_guard l(_free_blocks_lock); - _first_push_time = _free_blocks.empty() ? butil::gettimeofday_s() : _first_push_time; - _free_blocks.push_back(p); - return true; - } - - bool _pop_free_block(FreeBlock* blk) { - if (_free_blocks.empty()) { - return false; - } - _free_blocks_lock.lock(); - if (_free_blocks.empty()) { - _free_blocks_lock.unlock(); - return false; - } - DynamicFreeBlock* p = _free_blocks.back(); - _free_blocks.pop_back(); - _free_blocks_lock.unlock(); - memcpy(blk->ptrs, p->ptrs, sizeof(*p->ptrs) * p->nfree); - blk->nfree = p->nfree; - blk->bytes = p->bytes; - free(p); - return true; - } - -private: - ColumnPool() { _free_blocks.reserve(32); } - - ~ColumnPool() = default; - - std::shared_ptr _mem_tracker = nullptr; - - static __thread LocalPool* _local_pool; // NOLINT - static std::atomic _nlocal; // NOLINT - static std::mutex _change_thread_mutex; // NOLINT - - mutable std::mutex _free_blocks_lock; - std::vector _free_blocks; - int64_t _first_push_time = 0; -}; - -using ColumnPoolList = - TypeList, ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool, ColumnPool, - ColumnPool, ColumnPool>; - -template -__thread typename ColumnPool::LocalPool* ColumnPool::_local_pool = nullptr; // NOLINT - -template -std::atomic ColumnPool::_nlocal = 0; // NOLINT - -template -std::mutex ColumnPool::_change_thread_mutex{}; // NOLINT - -template -inline T* get_column() { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - return ColumnPool::singleton()->template get_column(); -} - -template -inline void return_column(T* ptr, size_t chunk_size) { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - ColumnPool::singleton()->return_column(ptr, chunk_size); -} - -template -inline void release_large_columns(size_t limit) { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - ColumnPool::singleton()->release_large_columns(limit); -} - -template -inline size_t release_free_columns(double ratio) { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - return ColumnPool::singleton()->release_free_columns(ratio); -} - -// NOTE: this is an expensive routine, so it should not be called too often. -template -inline ColumnPoolInfo describe_column_pool() { - static_assert(InList, ColumnPoolList>::value, "Cannot use column pool"); - return ColumnPool::singleton()->describe_column_pool(); -} - -// Used in tests. -template -inline void clear_columns() { - ColumnPool::singleton()->clear_columns(); -} - -template -struct HasColumnPool : public std::bool_constant, ColumnPoolList>::value> {}; - -namespace detail { -struct ClearColumnPool { - template - void operator()() { - Pool::singleton()->clear_columns(); - } -}; -} // namespace detail - -inline void TEST_clear_all_columns_this_thread() { - ForEach(detail::ClearColumnPool()); -} - -} // namespace starrocks diff --git a/be/src/common/config.h b/be/src/common/config.h index 6ce587c8042db..41ac3fe3b3625 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -282,8 +282,6 @@ CONF_mBool(enable_bitmap_index_memory_page_cache, "false"); CONF_mBool(enable_zonemap_index_memory_page_cache, "false"); // whether to enable the ordinal index memory cache CONF_mBool(enable_ordinal_index_memory_page_cache, "false"); -// whether to disable column pool -CONF_Bool(disable_column_pool, "true"); CONF_mInt32(base_compaction_check_interval_seconds, "60"); CONF_mInt64(min_base_compaction_num_singleton_deltas, "5"); diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index d1c10514f4eb8..707de05d9defb 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -38,7 +38,6 @@ #include "block_cache/block_cache.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "common/minidump.h" #include "exec/workgroup/work_group.h" @@ -85,36 +84,6 @@ std::atomic k_starrocks_exit = false; // but also waiting for all threads to exit gracefully. std::atomic k_starrocks_exit_quick = false; -class ReleaseColumnPool { -public: - explicit ReleaseColumnPool(double ratio) : _ratio(ratio) {} - - template - void operator()() { - _freed_bytes += Pool::singleton()->release_free_columns(_ratio); - } - - size_t freed_bytes() const { return _freed_bytes; } - -private: - double _ratio; - size_t _freed_bytes = 0; -}; - -void gc_memory(void* arg_this) { - using namespace starrocks; - const static float kFreeRatio = 0.5; - - auto* daemon = static_cast(arg_this); - while (!daemon->stopped()) { - nap_sleep(config::memory_maintenance_sleep_time_s, [daemon] { return daemon->stopped(); }); - - ReleaseColumnPool releaser(kFreeRatio); - ForEach(releaser); - LOG_IF(INFO, releaser.freed_bytes() > 0) << "Released " << releaser.freed_bytes() << " bytes from column pool"; - } -} - /* * This thread will calculate some metrics at a fix interval(15 sec) * 1. push bytes per second @@ -198,15 +167,15 @@ void calculate_metrics(void* arg_this) { LOG(INFO) << fmt::format( "Current memory statistics: process({}), query_pool({}), load({}), " - "metadata({}), compaction({}), schema_change({}), column_pool({}), " + "metadata({}), compaction({}), schema_change({}), " "page_cache({}), update({}), chunk_allocator({}), clone({}), consistency({}), " "datacache({}), jit({})", mem_metrics->process_mem_bytes.value(), mem_metrics->query_mem_bytes.value(), mem_metrics->load_mem_bytes.value(), mem_metrics->metadata_mem_bytes.value(), mem_metrics->compaction_mem_bytes.value(), mem_metrics->schema_change_mem_bytes.value(), - mem_metrics->column_pool_mem_bytes.value(), mem_metrics->storage_page_cache_mem_bytes.value(), - mem_metrics->update_mem_bytes.value(), mem_metrics->chunk_allocator_mem_bytes.value(), - mem_metrics->clone_mem_bytes.value(), mem_metrics->consistency_mem_bytes.value(), datacache_mem_bytes, + mem_metrics->storage_page_cache_mem_bytes.value(), mem_metrics->update_mem_bytes.value(), + mem_metrics->chunk_allocator_mem_bytes.value(), mem_metrics->clone_mem_bytes.value(), + mem_metrics->consistency_mem_bytes.value(), datacache_mem_bytes, mem_metrics->jit_cache_mem_bytes.value()); nap_sleep(15, [daemon] { return daemon->stopped(); }); @@ -307,10 +276,6 @@ void Daemon::init(bool as_cn, const std::vector& paths) { TimezoneUtils::init_time_zones(); - std::thread gc_thread(gc_memory, this); - Thread::set_thread_name(gc_thread, "gc_daemon"); - _daemon_threads.emplace_back(std::move(gc_thread)); - init_starrocks_metrics(paths); if (config::enable_metric_calculator) { diff --git a/be/src/connector/lake_connector.cpp b/be/src/connector/lake_connector.cpp index 7270ecd76c926..75b803ce5eb45 100644 --- a/be/src/connector/lake_connector.cpp +++ b/be/src/connector/lake_connector.cpp @@ -133,8 +133,7 @@ void LakeDataSource::close(RuntimeState* state) { } Status LakeDataSource::get_next(RuntimeState* state, ChunkPtr* chunk) { - chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size(), - _runtime_state->use_column_pool())); + chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size())); auto* chunk_ptr = chunk->get(); do { diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp index d394bab24192f..de5cbeea54934 100644 --- a/be/src/exec/olap_scan_node.cpp +++ b/be/src/exec/olap_scan_node.cpp @@ -20,7 +20,6 @@ #include #include "column/column_access_path.h" -#include "column/column_pool.h" #include "column/type_traits.h" #include "common/compiler_util.h" #include "common/status.h" @@ -159,7 +158,6 @@ Status OlapScanNode::get_next(RuntimeState* state, ChunkPtr* chunk, bool* eos) { RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::GETNEXT)); SCOPED_TIMER(_runtime_profile->total_time_counter()); - bool first_call = !_start; if (!_start && _status.ok()) { Status status = _start_scan(state); _update_status(status); @@ -210,7 +208,7 @@ Status OlapScanNode::get_next(RuntimeState* state, ChunkPtr* chunk, bool* eos) { // is the first time of calling `get_next`, pass the second argument of `_fill_chunk_pool` as // true to ensure that the newly allocated column objects will be returned back into the column // pool. - TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(1, first_call && state->use_column_pool())); + TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(1)); eval_join_runtime_filters(chunk); _num_rows_returned += (*chunk)->num_rows(); COUNTER_SET(_rows_returned_counter, _num_rows_returned); @@ -256,11 +254,6 @@ void OlapScanNode::close(RuntimeState* state) { chunk.reset(); } - if (runtime_state() != nullptr) { - // Reduce the memory usage if the the average string size is greater than 512. - release_large_columns(runtime_state()->chunk_size() * 512); - } - for (const auto& rowsets_per_tablet : _tablet_rowsets) { Rowset::release_readers(rowsets_per_tablet); } @@ -275,10 +268,10 @@ OlapScanNode::~OlapScanNode() { DCHECK(is_closed()); } -void OlapScanNode::_fill_chunk_pool(int count, bool force_column_pool) { +void OlapScanNode::_fill_chunk_pool(int count) { const size_t capacity = runtime_state()->chunk_size(); for (int i = 0; i < count; i++) { - ChunkPtr chunk(ChunkHelper::new_chunk_pooled(*_chunk_schema, capacity, force_column_pool)); + ChunkPtr chunk(ChunkHelper::new_chunk_pooled(*_chunk_schema, capacity)); { std::lock_guard l(_mtx); _chunk_pool.push(std::move(chunk)); @@ -736,7 +729,7 @@ Status OlapScanNode::_start_scan_thread(RuntimeState* state) { COUNTER_SET(_task_concurrency, (int64_t)concurrency); int chunks = _chunks_per_scanner * concurrency; _chunk_pool.reserve(chunks); - TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(chunks, state->use_column_pool())); + TRY_CATCH_BAD_ALLOC(_fill_chunk_pool(chunks)); std::lock_guard l(_mtx); for (int i = 0; i < concurrency; i++) { CHECK(_submit_scanner(_pending_scanners.pop(), true)); diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h index f2a9c1174df27..e087bb27a4304 100644 --- a/be/src/exec/olap_scan_node.h +++ b/be/src/exec/olap_scan_node.h @@ -150,7 +150,7 @@ class OlapScanNode final : public starrocks::ScanNode { void _update_status(const Status& status); Status _get_status(); - void _fill_chunk_pool(int count, bool force_column_pool); + void _fill_chunk_pool(int count); bool _submit_scanner(TabletScanner* scanner, bool blockable); void _close_pending_scanners(); diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.cpp b/be/src/exec/pipeline/scan/olap_chunk_source.cpp index 62e7aa2a49193..eca5e61a932f7 100644 --- a/be/src/exec/pipeline/scan/olap_chunk_source.cpp +++ b/be/src/exec/pipeline/scan/olap_chunk_source.cpp @@ -541,8 +541,7 @@ Status OlapChunkSource::_init_olap_reader(RuntimeState* runtime_state) { } Status OlapChunkSource::_read_chunk(RuntimeState* state, ChunkPtr* chunk) { - chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size(), - _runtime_state->use_column_pool())); + chunk->reset(ChunkHelper::new_chunk_pooled(_prj_iter->output_schema(), _runtime_state->chunk_size())); auto scope = IOProfiler::scope(IOProfiler::TAG_QUERY, _tablet->tablet_id()); return _read_chunk_from_storage(_runtime_state, (*chunk).get()); } diff --git a/be/src/exec/tablet_scanner.cpp b/be/src/exec/tablet_scanner.cpp index cd537925962b9..1908e98fce8e3 100644 --- a/be/src/exec/tablet_scanner.cpp +++ b/be/src/exec/tablet_scanner.cpp @@ -17,7 +17,6 @@ #include #include -#include "column/column_pool.h" #include "column/vectorized_fwd.h" #include "common/status.h" #include "exec/olap_scan_node.h" @@ -118,8 +117,6 @@ void TabletScanner::close(RuntimeState* state) { _reader.reset(); _predicate_free_pool.clear(); Expr::close(_conjunct_ctxs, state); - // Reduce the memory usage if the the average string size is greater than 512. - release_large_columns(state->chunk_size() * 512); _is_closed = true; } diff --git a/be/src/http/action/memory_metrics_action.cpp b/be/src/http/action/memory_metrics_action.cpp index d442bb85371bc..20b80c783dc32 100644 --- a/be/src/http/action/memory_metrics_action.cpp +++ b/be/src/http/action/memory_metrics_action.cpp @@ -49,7 +49,6 @@ void MemoryMetricsAction::handle(HttpRequest* req) { "bloom_filter_index", "compaction", "schema_change", - "column_pool", "page_cache", "datacache", "update", diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index 14a6108586c28..65e73518ff11a 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -140,9 +140,6 @@ void mem_tracker_handler(MemTracker* mem_tracker, const WebPageHandler::Argument } else if (iter->second == "clone") { start_mem_tracker = GlobalEnv::GetInstance()->clone_mem_tracker(); cur_level = 2; - } else if (iter->second == "column_pool") { - start_mem_tracker = GlobalEnv::GetInstance()->column_pool_mem_tracker(); - cur_level = 2; } else if (iter->second == "page_cache") { start_mem_tracker = GlobalEnv::GetInstance()->page_cache_mem_tracker(); cur_level = 2; diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp index d1f9a9f7dee17..a918c4942e8cc 100644 --- a/be/src/runtime/exec_env.cpp +++ b/be/src/runtime/exec_env.cpp @@ -41,7 +41,6 @@ #include "agent/agent_server.h" #include "agent/master_info.h" #include "block_cache/block_cache.h" -#include "column/column_pool.h" #include "common/config.h" #include "common/configbase.h" #include "common/logging.h" @@ -234,7 +233,6 @@ Status GlobalEnv::_init_mem_tracker() { int64_t compaction_mem_limit = calc_max_compaction_memory(_process_mem_tracker->limit()); _compaction_mem_tracker = regist_tracker(compaction_mem_limit, "compaction", _process_mem_tracker.get()); _schema_change_mem_tracker = regist_tracker(-1, "schema_change", _process_mem_tracker.get()); - _column_pool_mem_tracker = regist_tracker(-1, "column_pool", _process_mem_tracker.get()); _page_cache_mem_tracker = regist_tracker(-1, "page_cache", _process_mem_tracker.get()); _jit_cache_mem_tracker = regist_tracker(-1, "jit_cache", _process_mem_tracker.get()); int32_t update_mem_percent = std::max(std::min(100, config::update_memory_limit_percent), 0); @@ -248,8 +246,6 @@ Status GlobalEnv::_init_mem_tracker() { MemChunkAllocator::init_instance(_chunk_allocator_mem_tracker.get(), config::chunk_reserved_bytes_limit); - SetMemTrackerForColumnPool op(_column_pool_mem_tracker); - ForEach(op); _init_storage_page_cache(); // TODO: move to StorageEngine return Status::OK(); } diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 402ee0b93e457..3cf3cb546c895 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -139,7 +139,6 @@ class GlobalEnv { MemTracker* short_key_index_mem_tracker() { return _short_key_index_mem_tracker.get(); } MemTracker* compaction_mem_tracker() { return _compaction_mem_tracker.get(); } MemTracker* schema_change_mem_tracker() { return _schema_change_mem_tracker.get(); } - MemTracker* column_pool_mem_tracker() { return _column_pool_mem_tracker.get(); } MemTracker* page_cache_mem_tracker() { return _page_cache_mem_tracker.get(); } MemTracker* jit_cache_mem_tracker() { return _jit_cache_mem_tracker.get(); } MemTracker* update_mem_tracker() { return _update_mem_tracker.get(); } @@ -199,9 +198,6 @@ class GlobalEnv { // The memory used for schema change std::shared_ptr _schema_change_mem_tracker; - // The memory used for column pool - std::shared_ptr _column_pool_mem_tracker; - // The memory used for page cache std::shared_ptr _page_cache_mem_tracker; diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 2875ecf5fa9e0..5beac56553b93 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -950,7 +950,6 @@ Status FragmentMgr::exec_external_plan_fragment(const TScanOpenParams& params, c query_options.query_type = TQueryType::EXTERNAL; // For spark sql / flink sql, we dont use page cache. query_options.use_page_cache = false; - query_options.use_column_pool = false; query_options.enable_profile = config::enable_profile_for_external_plan; exec_fragment_params.__set_query_options(query_options); VLOG_ROW << "external exec_plan_fragment params is " diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 45d83eff9ff18..a2dd0bb3f2137 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -287,17 +287,6 @@ bool RuntimeState::use_page_cache() { return true; } -bool RuntimeState::use_column_pool() const { - if (config::disable_column_pool) { - return false; - } - - if (_query_options.__isset.use_column_pool) { - return _query_options.use_column_pool; - } - return true; -} - Status RuntimeState::set_mem_limit_exceeded(MemTracker* tracker, int64_t failed_allocation_size, std::string_view msg) { DCHECK_GE(failed_allocation_size, 0); { diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index c557b9e35c103..f2cae96ffc397 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -131,7 +131,6 @@ class RuntimeState { void set_desc_tbl(DescriptorTbl* desc_tbl) { _desc_tbl = desc_tbl; } int chunk_size() const { return _query_options.batch_size; } void set_chunk_size(int chunk_size) { _query_options.batch_size = chunk_size; } - bool use_column_pool() const; bool abort_on_default_limit_exceeded() const { return _query_options.abort_on_default_limit_exceeded; } int64_t timestamp_ms() const { return _timestamp_us / 1000; } int64_t timestamp_us() const { return _timestamp_us; } diff --git a/be/src/script/script.cpp b/be/src/script/script.cpp index 157839fcffae1..40d30f2d40b78 100644 --- a/be/src/script/script.cpp +++ b/be/src/script/script.cpp @@ -222,7 +222,6 @@ void bind_exec_env(ForeignModule& m) { REG_METHOD(GlobalEnv, metadata_mem_tracker); REG_METHOD(GlobalEnv, compaction_mem_tracker); REG_METHOD(GlobalEnv, schema_change_mem_tracker); - REG_METHOD(GlobalEnv, column_pool_mem_tracker); REG_METHOD(GlobalEnv, page_cache_mem_tracker); REG_METHOD(GlobalEnv, jit_cache_mem_tracker); REG_METHOD(GlobalEnv, update_mem_tracker); diff --git a/be/src/storage/chunk_helper.cpp b/be/src/storage/chunk_helper.cpp index be635e55aa3d2..2679634ac3daa 100644 --- a/be/src/storage/chunk_helper.cpp +++ b/be/src/storage/chunk_helper.cpp @@ -17,7 +17,6 @@ #include "column/array_column.h" #include "column/chunk.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "column/json_column.h" #include "column/map_column.h" #include "column/schema.h" @@ -230,53 +229,34 @@ ColumnId ChunkHelper::max_column_id(const starrocks::Schema& schema) { } template -struct ColumnDeleter { - ColumnDeleter(size_t chunk_size) : chunk_size(chunk_size) {} - void operator()(Column* ptr) const { return_column(down_cast(ptr), chunk_size); } - size_t chunk_size; -}; - -template -inline std::shared_ptr get_column_ptr(size_t chunk_size) { - if constexpr (std::negation_v>) { - return std::make_shared(); - } else { - T* ptr = get_column(); - if (LIKELY(ptr != nullptr)) { - return std::shared_ptr(ptr, ColumnDeleter(chunk_size)); - } else { - return std::make_shared(); - } - } +inline std::shared_ptr get_column_ptr() { + return std::make_shared(); } -template -inline std::shared_ptr> get_decimal_column_ptr(int precision, int scale, size_t chunk_size) { - auto column = get_column_ptr(chunk_size); +template +inline std::shared_ptr> get_decimal_column_ptr(int precision, int scale) { + auto column = get_column_ptr(); column->set_precision(precision); column->set_scale(scale); return column; } -template struct ColumnPtrBuilder { template - ColumnPtr operator()(size_t chunk_size, const Field& field, int precision, int scale) { + ColumnPtr operator()(const Field& field, int precision, int scale) { auto NullableIfNeed = [&](ColumnPtr c) -> ColumnPtr { - return field.is_nullable() - ? NullableColumn::create(std::move(c), get_column_ptr(chunk_size)) - : c; + return field.is_nullable() ? NullableColumn::create(std::move(c), get_column_ptr()) : c; }; if constexpr (ftype == TYPE_ARRAY) { auto elements = NullableColumn::wrap_if_necessary(field.sub_field(0).create_column()); - auto offsets = get_column_ptr(chunk_size); + auto offsets = get_column_ptr(); auto array = ArrayColumn::create(std::move(elements), offsets); return NullableIfNeed(array); } else if constexpr (ftype == TYPE_MAP) { auto keys = NullableColumn::wrap_if_necessary(field.sub_field(0).create_column()); auto values = NullableColumn::wrap_if_necessary(field.sub_field(1).create_column()); - auto offsets = get_column_ptr(chunk_size); + auto offsets = get_column_ptr(); auto map = MapColumn::create(std::move(keys), std::move(values), offsets); return NullableIfNeed(map); } else if constexpr (ftype == TYPE_STRUCT) { @@ -291,33 +271,31 @@ struct ColumnPtrBuilder { } else { switch (ftype) { case TYPE_DECIMAL32: - return NullableIfNeed(get_decimal_column_ptr(precision, scale, chunk_size)); + return NullableIfNeed(get_decimal_column_ptr(precision, scale)); case TYPE_DECIMAL64: - return NullableIfNeed(get_decimal_column_ptr(precision, scale, chunk_size)); + return NullableIfNeed(get_decimal_column_ptr(precision, scale)); case TYPE_DECIMAL128: - return NullableIfNeed(get_decimal_column_ptr(precision, scale, chunk_size)); + return NullableIfNeed(get_decimal_column_ptr(precision, scale)); default: { - return NullableIfNeed(get_column_ptr::ColumnType, force>(chunk_size)); + return NullableIfNeed(get_column_ptr::ColumnType>()); } } } } }; -template -ColumnPtr column_from_pool(const Field& field, size_t chunk_size) { +ColumnPtr column_from_pool(const Field& field) { auto precision = field.type()->precision(); auto scale = field.type()->scale(); - return field_type_dispatch_column(field.type()->type(), ColumnPtrBuilder(), chunk_size, field, precision, - scale); + return field_type_dispatch_column(field.type()->type(), ColumnPtrBuilder(), field, precision, scale); } -Chunk* ChunkHelper::new_chunk_pooled(const Schema& schema, size_t chunk_size, bool force) { +Chunk* ChunkHelper::new_chunk_pooled(const Schema& schema, size_t chunk_size) { Columns columns; columns.reserve(schema.num_fields()); for (size_t i = 0; i < schema.num_fields(); i++) { const FieldPtr& f = schema.field(i); - auto column = force ? column_from_pool(*f, chunk_size) : column_from_pool(*f, chunk_size); + auto column = column_from_pool(*f); column->reserve(chunk_size); columns.emplace_back(std::move(column)); } diff --git a/be/src/storage/chunk_helper.h b/be/src/storage/chunk_helper.h index 94c0a27234cfc..2e90041793df8 100644 --- a/be/src/storage/chunk_helper.h +++ b/be/src/storage/chunk_helper.h @@ -69,7 +69,7 @@ class ChunkHelper { // Create an empty chunk according to the |slots| and reserve it of size |n|. static ChunkUniquePtr new_chunk(const std::vector& slots, size_t n); - static Chunk* new_chunk_pooled(const Schema& schema, size_t n, bool force); + static Chunk* new_chunk_pooled(const Schema& schema, size_t n); // Create a vectorized column from field . // REQUIRE: |type| must be scalar type. diff --git a/be/src/testutil/init_test_env.h b/be/src/testutil/init_test_env.h index 799f6eb4806d2..ce6830b477b0a 100644 --- a/be/src/testutil/init_test_env.h +++ b/be/src/testutil/init_test_env.h @@ -16,7 +16,6 @@ #include "butil/file_util.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "gtest/gtest.h" @@ -110,7 +109,6 @@ int init_test_env(int argc, char** argv) { // clear some trash objects kept in tablet_manager so mem_tracker checks will not fail CHECK(StorageEngine::instance()->tablet_manager()->start_trash_sweep().ok()); (void)butil::DeleteFile(storage_root, true); - TEST_clear_all_columns_this_thread(); // delete engine StorageEngine::instance()->stop(); // destroy exec env diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index 439fb19a864e8..aee876478a4a9 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -40,10 +40,10 @@ #include #include -#include "column/column_pool.h" #include "gutil/strings/split.h" // for string split #include "gutil/strtoint.h" // for atoi64 #include "jemalloc/jemalloc.h" +#include "runtime/mem_tracker.h" #include "runtime/runtime_filter_worker.h" #include "util/metrics.h" @@ -276,7 +276,6 @@ void SystemMetrics::_install_memory_metrics(MetricRegistry* registry) { registry->register_metric("short_key_index_mem_bytes", &_memory_metrics->short_key_index_mem_bytes); registry->register_metric("compaction_mem_bytes", &_memory_metrics->compaction_mem_bytes); registry->register_metric("schema_change_mem_bytes", &_memory_metrics->schema_change_mem_bytes); - registry->register_metric("column_pool_mem_bytes", &_memory_metrics->column_pool_mem_bytes); registry->register_metric("storage_page_cache_mem_bytes", &_memory_metrics->storage_page_cache_mem_bytes); registry->register_metric("jit_cache_mem_bytes", &_memory_metrics->jit_cache_mem_bytes); registry->register_metric("update_mem_bytes", &_memory_metrics->update_mem_bytes); @@ -284,22 +283,6 @@ void SystemMetrics::_install_memory_metrics(MetricRegistry* registry) { registry->register_metric("clone_mem_bytes", &_memory_metrics->clone_mem_bytes); registry->register_metric("consistency_mem_bytes", &_memory_metrics->consistency_mem_bytes); registry->register_metric("datacache_mem_bytes", &_memory_metrics->datacache_mem_bytes); - - registry->register_metric("total_column_pool_bytes", &_memory_metrics->column_pool_total_bytes); - registry->register_metric("local_column_pool_bytes", &_memory_metrics->column_pool_local_bytes); - registry->register_metric("central_column_pool_bytes", &_memory_metrics->column_pool_central_bytes); - registry->register_metric("binary_column_pool_bytes", &_memory_metrics->column_pool_binary_bytes); - registry->register_metric("uint8_column_pool_bytes", &_memory_metrics->column_pool_uint8_bytes); - registry->register_metric("int8_column_pool_bytes", &_memory_metrics->column_pool_int8_bytes); - registry->register_metric("int16_column_pool_bytes", &_memory_metrics->column_pool_int16_bytes); - registry->register_metric("int32_column_pool_bytes", &_memory_metrics->column_pool_int32_bytes); - registry->register_metric("int64_column_pool_bytes", &_memory_metrics->column_pool_int64_bytes); - registry->register_metric("int128_column_pool_bytes", &_memory_metrics->column_pool_int128_bytes); - registry->register_metric("float_column_pool_bytes", &_memory_metrics->column_pool_float_bytes); - registry->register_metric("double_column_pool_bytes", &_memory_metrics->column_pool_double_bytes); - registry->register_metric("decimal_column_pool_bytes", &_memory_metrics->column_pool_decimal_bytes); - registry->register_metric("date_column_pool_bytes", &_memory_metrics->column_pool_date_bytes); - registry->register_metric("datetime_column_pool_bytes", &_memory_metrics->column_pool_datetime_bytes); } void SystemMetrics::_update_memory_metrics() { @@ -362,29 +345,9 @@ void SystemMetrics::_update_memory_metrics() { SET_MEM_METRIC_VALUE(update_mem_tracker, update_mem_bytes) SET_MEM_METRIC_VALUE(chunk_allocator_mem_tracker, chunk_allocator_mem_bytes) SET_MEM_METRIC_VALUE(clone_mem_tracker, clone_mem_bytes) - SET_MEM_METRIC_VALUE(column_pool_mem_tracker, column_pool_mem_bytes) SET_MEM_METRIC_VALUE(consistency_mem_tracker, consistency_mem_bytes) SET_MEM_METRIC_VALUE(datacache_mem_tracker, datacache_mem_bytes) #undef SET_MEM_METRIC_VALUE - -#define UPDATE_COLUMN_POOL_METRIC(var, type) \ - value = describe_column_pool().central_free_bytes; \ - var.set_value(value); - - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_binary_bytes, BinaryColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_uint8_bytes, UInt8Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int8_bytes, Int8Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int16_bytes, Int16Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int32_bytes, Int32Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int64_bytes, Int64Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_int128_bytes, Int128Column) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_float_bytes, FloatColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_double_bytes, DoubleColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_decimal_bytes, DecimalColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_date_bytes, DateColumn) - UPDATE_COLUMN_POOL_METRIC(_memory_metrics->column_pool_datetime_bytes, TimestampColumn) - -#undef UPDATE_COLUMN_POOL_METRIC } void SystemMetrics::_install_disk_metrics(MetricRegistry* registry, const std::set& devices) { diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h index 38652de6f8182..eb96a97160986 100644 --- a/be/src/util/system_metrics.h +++ b/be/src/util/system_metrics.h @@ -62,7 +62,6 @@ class MemoryMetrics { METRIC_DEFINE_INT_GAUGE(short_key_index_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(compaction_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(schema_change_mem_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(storage_page_cache_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(jit_cache_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(update_mem_bytes, MetricUnit::BYTES); @@ -70,23 +69,6 @@ class MemoryMetrics { METRIC_DEFINE_INT_GAUGE(clone_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(consistency_mem_bytes, MetricUnit::BYTES); METRIC_DEFINE_INT_GAUGE(datacache_mem_bytes, MetricUnit::BYTES); - - // column pool metrics. - METRIC_DEFINE_INT_GAUGE(column_pool_total_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_local_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_central_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_binary_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_uint8_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int8_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int16_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int32_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int64_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_int128_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_float_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_double_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_decimal_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_date_bytes, MetricUnit::BYTES); - METRIC_DEFINE_INT_GAUGE(column_pool_datetime_bytes, MetricUnit::BYTES); }; class SystemMetrics { diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 2e6b30a8e1f16..2e51b5e5eb6a3 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -6,7 +6,6 @@ set(EXEC_FILES ./column/binary_column_test.cpp ./column/chunk_test.cpp ./column/column_helper_test.cpp - ./column/column_pool_test.cpp ./column/const_column_test.cpp ./column/date_value_test.cpp ./column/decimalv3_column_test.cpp diff --git a/be/test/column/column_pool_test.cpp b/be/test/column/column_pool_test.cpp deleted file mode 100644 index e10a8bb9afd88..0000000000000 --- a/be/test/column/column_pool_test.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "column/column_pool.h" - -#include "gtest/gtest.h" - -namespace starrocks { - -class ColumnPoolTest : public ::testing::Test { -protected: - void SetUp() override { TEST_clear_all_columns_this_thread(); } - void TearDown() override { TEST_clear_all_columns_this_thread(); } -}; - -// NOLINTNEXTLINE -TEST_F(ColumnPoolTest, mem_statistics) { - auto* pool = ColumnPool::singleton(); - std::string str(20, '1'); - auto* c1 = get_column(); - c1->reserve(config::vector_chunk_size); - for (size_t i = 0; i < config::vector_chunk_size; i++) { - c1->append_string(str); - } - - pool->return_column(c1, config::vector_chunk_size); - size_t usage_1 = pool->mem_tracker()->consumption(); - - pool->release_large_columns(config::vector_chunk_size * 10); - size_t usage_2 = pool->mem_tracker()->consumption(); - - ASSERT_EQ(usage_1 - usage_2, 81920); -} - -// NOLINTNEXTLINE -TEST_F(ColumnPoolTest, single_thread) { - auto c1 = get_column(); - ASSERT_EQ(0u, c1->get_data().capacity()); - - c1->reserve(3); - c1->append_datum(Datum((int32_t)1)); - c1->append_datum(Datum((int32_t)2)); - c1->set_delete_state(DEL_PARTIAL_SATISFIED); - return_column(c1, config::vector_chunk_size); - - auto c2 = get_column(); - ASSERT_EQ(c1, c2); - ASSERT_EQ(0u, c2->size()); - ASSERT_EQ(DEL_NOT_SATISFIED, c2->delete_state()); - ASSERT_EQ(3, c2->get_data().capacity()); - - auto c3 = get_column(); - ASSERT_NE(c2, c3); - - auto c4 = get_column(); - ASSERT_NE(c3, c4); - - auto c5 = get_column(); - ASSERT_NE(c4, c5); - - auto c6 = get_column(); - ASSERT_NE(c5, c6); - - return_column(c6, config::vector_chunk_size); - return_column(c5, config::vector_chunk_size); - - auto c7 = get_column(); - auto c8 = get_column(); - ASSERT_EQ(c5, c7); - ASSERT_EQ(c6, c8); - - return_column(c8, config::vector_chunk_size); - return_column(c7, config::vector_chunk_size); - - delete c2; - delete c3; - delete c4; -} - -} // namespace starrocks diff --git a/be/test/storage/aggregate_iterator_test.cpp b/be/test/storage/aggregate_iterator_test.cpp index 922a26e949670..daa0ed77a814f 100644 --- a/be/test/storage/aggregate_iterator_test.cpp +++ b/be/test/storage/aggregate_iterator_test.cpp @@ -14,7 +14,7 @@ #include "storage/aggregate_iterator.h" -#include "column/column_pool.h" +#include "common/config.h" #include "gtest/gtest.h" #include "storage/aggregate_type.h" #include "storage/vector_chunk_iterator.h" @@ -36,7 +36,7 @@ static std::vector row(const Chunk& chunk, size_t row_id) { class AggregateIteratorTest : public testing::Test { protected: void SetUp() override {} - void TearDown() override { TEST_clear_all_columns_this_thread(); } + void TearDown() override {} }; // NOLINTNEXTLINE diff --git a/be/test/storage/projection_iterator_test.cpp b/be/test/storage/projection_iterator_test.cpp index 03081ec6200c7..e090f18d8c6f8 100644 --- a/be/test/storage/projection_iterator_test.cpp +++ b/be/test/storage/projection_iterator_test.cpp @@ -18,8 +18,8 @@ #include "column/chunk.h" #include "column/column.h" -#include "column/column_pool.h" #include "column/datum.h" +#include "common/config.h" #include "storage/chunk_helper.h" namespace starrocks { @@ -67,7 +67,7 @@ class VectorIterator final : public ChunkIterator { class ProjectionIteratorTest : public testing::Test { protected: void SetUp() override {} - void TearDown() override { TEST_clear_all_columns_this_thread(); } + void TearDown() override {} }; // NOLINTNEXTLINE diff --git a/be/test/storage/publish_version_manager_test.cpp b/be/test/storage/publish_version_manager_test.cpp index 9f23978ec7e1a..9af7839ab3519 100644 --- a/be/test/storage/publish_version_manager_test.cpp +++ b/be/test/storage/publish_version_manager_test.cpp @@ -21,7 +21,6 @@ #include "agent/publish_version.h" #include "butil/file_util.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "fs/fs_util.h" diff --git a/be/test/storage/publish_version_task_test.cpp b/be/test/storage/publish_version_task_test.cpp index 1e40784d656db..9b6b1a683e875 100644 --- a/be/test/storage/publish_version_task_test.cpp +++ b/be/test/storage/publish_version_task_test.cpp @@ -19,7 +19,6 @@ #include "agent/publish_version.h" #include "butil/file_util.h" #include "column/column_helper.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "fs/fs_util.h" diff --git a/be/test/storage/task/engine_storage_migration_task_test.cpp b/be/test/storage/task/engine_storage_migration_task_test.cpp index f9cc364bc2293..78ff38624ac45 100644 --- a/be/test/storage/task/engine_storage_migration_task_test.cpp +++ b/be/test/storage/task/engine_storage_migration_task_test.cpp @@ -17,7 +17,6 @@ #include #include "butil/file_util.h" -#include "column/column_pool.h" #include "common/config.h" #include "exec/pipeline/query_context.h" #include "fs/fs_util.h" @@ -663,7 +662,6 @@ int main(int argc, char** argv) { // clear some trash objects kept in tablet_manager so mem_tracker checks will not fail starrocks::StorageEngine::instance()->tablet_manager()->start_trash_sweep(); starrocks::fs::remove_all(storage_root.value()); - starrocks::TEST_clear_all_columns_this_thread(); // delete engine engine->stop(); delete engine; diff --git a/be/test/storage/union_iterator_test.cpp b/be/test/storage/union_iterator_test.cpp index 0808efe321aff..b53285efb632c 100644 --- a/be/test/storage/union_iterator_test.cpp +++ b/be/test/storage/union_iterator_test.cpp @@ -20,9 +20,9 @@ #include #include "column/chunk.h" -#include "column/column_pool.h" #include "column/fixed_length_column.h" #include "column/schema.h" +#include "common/config.h" #include "storage/chunk_helper.h" namespace starrocks { @@ -30,7 +30,7 @@ namespace starrocks { class UnionIteratorTest : public testing::Test { protected: void SetUp() override {} - void TearDown() override { TEST_clear_all_columns_this_thread(); } + void TearDown() override {} // return chunk with single column of type int32_t. class IntIterator final : public ChunkIterator { diff --git a/gensrc/thrift/InternalService.thrift b/gensrc/thrift/InternalService.thrift index dd0f3d5fb372a..6dc55fa7bd01a 100644 --- a/gensrc/thrift/InternalService.thrift +++ b/gensrc/thrift/InternalService.thrift @@ -281,7 +281,7 @@ struct TQueryOptions { 103: optional i32 interleaving_group_size; 104: optional TOverflowMode overflow_mode = TOverflowMode.OUTPUT_NULL; - 105: optional bool use_column_pool = true; + 105: optional bool use_column_pool = true; // Deprecated // Deprecated 106: optional bool enable_agg_spill_preaggregation; 107: optional i64 global_runtime_filter_build_max_size;