diff --git a/.jenkins/Dockerfile b/.jenkins/Dockerfile index 86f486042..cc3556eea 100644 --- a/.jenkins/Dockerfile +++ b/.jenkins/Dockerfile @@ -13,7 +13,10 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/*; COPY test_index_btree /usr/local/bin/test_index_btree +COPY test_meta_blk_mgr /usr/local/bin/test_meta_blk_mgr +COPY test_log_store /usr/local/bin/test_log_store COPY btree_test.py /usr/local/bin/scripts/btree_test.py +COPY log_meta_test.py /usr/local/bin/scripts/log_meta_test.py EXPOSE 5000 -# ########## ####### ############ \ No newline at end of file +# ########## ####### ############ diff --git a/.jenkins/jenkinsfile_nightly b/.jenkins/jenkinsfile_nightly index 527af6d26..cddc26077 100644 --- a/.jenkins/jenkinsfile_nightly +++ b/.jenkins/jenkinsfile_nightly @@ -42,7 +42,10 @@ pipeline { steps { sh "conan create --build missing -o sisl:prerelease=True -o homestore:sanitize=True -o homestore:skip_testing=True -pr debug . ${PROJECT}/${VER}@" sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;" + sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr \\;" + sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_log_store' -exec cp {} .jenkins/test_log_store \\;" sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/btree_test.py' -exec install -Dm755 {} .jenkins/btree_test.py \\; " + sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/log_meta_test.py' -exec install -Dm755 {} .jenkins/log_meta_test.py \\; " } post { failure { diff --git a/conanfile.py b/conanfile.py index 18387456c..5b65203b4 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "5.0.1" + version = "5.0.4" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" topics = ("ebay", "nublox") diff --git a/src/include/homestore/checkpoint/cp_mgr.hpp b/src/include/homestore/checkpoint/cp_mgr.hpp index 507475c6e..9c783eb84 100644 --- a/src/include/homestore/checkpoint/cp_mgr.hpp +++ b/src/include/homestore/checkpoint/cp_mgr.hpp @@ -154,6 +154,7 @@ class CPManager { superblk< cp_mgr_super_block > m_sb; std::vector< iomgr::io_fiber_t > m_cp_io_fibers; iomgr::timer_handle_t m_cp_timer_hdl; + std::atomic< bool > m_cp_shutdown_initiated{false}; public: CPManager(); diff --git a/src/include/homestore/index/index_table.hpp b/src/include/homestore/index/index_table.hpp index 34c48593e..ebe189e7e 100644 --- a/src/include/homestore/index/index_table.hpp +++ b/src/include/homestore/index/index_table.hpp @@ -50,9 +50,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > { } void destroy() override { - auto cpg = hs()->cp_mgr().cp_guard(); - auto op_context = (void*)cpg.context(cp_consumer_t::INDEX_SVC); - Btree< K, V >::destroy_btree(op_context); + Btree< K, V >::destroy_btree(nullptr); } btree_status_t init() { diff --git a/src/lib/blkalloc/bitmap_blk_allocator.cpp b/src/lib/blkalloc/bitmap_blk_allocator.cpp index 6b6d32ee2..f06182fa8 100644 --- a/src/lib/blkalloc/bitmap_blk_allocator.cpp +++ b/src/lib/blkalloc/bitmap_blk_allocator.cpp @@ -84,7 +84,7 @@ bool BitmapBlkAllocator::is_blk_alloced_on_disk(const BlkId& b, bool use_lock) c BlkAllocStatus BitmapBlkAllocator::alloc_on_disk(BlkId const& bid) { if (!is_persistent()) { - //for non-persistent bitmap nothing is needed to do. So always return success + // for non-persistent bitmap nothing is needed to do. So always return success return BlkAllocStatus::SUCCESS; } @@ -149,6 +149,7 @@ void BitmapBlkAllocator::free_on_disk(BlkId const& bid) { "Expected disk bits to set blk num {} num blks {}", b.blk_num(), b.blk_count()); } } + m_disk_bm->reset_bits(b.blk_num(), b.blk_count()); } }; diff --git a/src/lib/blkalloc/varsize_blk_allocator.cpp b/src/lib/blkalloc/varsize_blk_allocator.cpp index 72248ee4f..fa66c864d 100644 --- a/src/lib/blkalloc/varsize_blk_allocator.cpp +++ b/src/lib/blkalloc/varsize_blk_allocator.cpp @@ -640,6 +640,7 @@ BlkAllocStatus VarsizeBlkAllocator::mark_blk_allocated(BlkId const& bid) { "Expected end bit to be smaller than portion end bit"); #endif m_cache_bm->set_bits(bid.blk_num(), bid.blk_count()); + incr_alloced_blk_count(bid.blk_count()); } BLKALLOC_LOG(TRACE, "mark blk alloced directly to portion={} blkid={} set_bits_count={}", blknum_to_portion_num(bid.blk_num()), bid.to_string(), get_alloced_blk_count()); diff --git a/src/lib/checkpoint/cp_mgr.cpp b/src/lib/checkpoint/cp_mgr.cpp index 65f623d45..ad6e3efdb 100644 --- a/src/lib/checkpoint/cp_mgr.cpp +++ b/src/lib/checkpoint/cp_mgr.cpp @@ -74,10 +74,16 @@ void CPManager::shutdown() { LOGINFO("Stopping cp timer"); iomanager.cancel_timer(m_cp_timer_hdl, true); m_cp_timer_hdl = iomgr::null_timer_handle; + m_cp_shutdown_initiated = true; - auto cp = get_cur_cp(); - delete (cp); + LOGINFO("Trigger cp flush"); + auto success = trigger_cp_flush(true /* force */).get(); + HS_REL_ASSERT_EQ(success, true, "CP Flush failed"); + LOGINFO("Trigger cp done"); + + delete (m_cur_cp); rcu_xchg_pointer(&m_cur_cp, nullptr); + m_metrics.reset(); if (m_wd_cp) { m_wd_cp->stop(); @@ -220,12 +226,24 @@ void CPManager::on_cp_flush_done(CP* cp) { m_sb.write(); cleanup_cp(cp); - cp->m_comp_promise.setValue(true); - m_in_flush_phase = false; + // Setting promise will cause the CP manager destructor to cleanup + // before getting a chance to do the checking if shutdown has been + // initiated or not. + auto shutdown_initiated = m_cp_shutdown_initiated.load(); + auto promise = std::move(cp->m_comp_promise); + m_wd_cp->reset_cp(); delete cp; + promise.setValue(true); + if (shutdown_initiated) { + // If shutdown initiated, dont trigger another CP. + // Dont access any cp state after this. + return; + } + m_in_flush_phase = false; + // Trigger CP in case there is one back to back CP { auto cur_cp = cp_guard(); @@ -257,7 +275,7 @@ void CPManager::start_cp_thread() { auto ctx = std::make_shared< Context >(); // Start a reactor with 9 fibers (8 for sync io) - iomanager.create_reactor("cp_io", iomgr::INTERRUPT_LOOP, 8u, [this, &ctx](bool is_started) { + iomanager.create_reactor("cp_io", iomgr::INTERRUPT_LOOP, 8u, [this, ctx](bool is_started) { if (is_started) { { std::unique_lock< std::mutex > lk{ctx->mtx}; diff --git a/src/lib/device/virtual_dev.cpp b/src/lib/device/virtual_dev.cpp index 9bf7e4422..a63bee2ba 100644 --- a/src/lib/device/virtual_dev.cpp +++ b/src/lib/device/virtual_dev.cpp @@ -150,9 +150,10 @@ BlkAllocStatus VirtualDev::commit_blk(BlkId const& blkid) { HS_LOG(DEBUG, device, "commit_blk: bid {}", blkid.to_string()); auto const recovering = homestore::hs()->is_initializing(); if (!recovering) { - HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode", blkid.to_string()); + HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode", + blkid.to_string()); } else { - chunk->blk_allocator_mutable()->mark_blk_allocated(blkid); + chunk->blk_allocator_mutable()->mark_blk_allocated(blkid); } return chunk->blk_allocator_mutable()->alloc_on_disk(blkid); } @@ -169,8 +170,13 @@ BlkAllocStatus VirtualDev::alloc_contiguous_blks(blk_count_t nblks, blk_alloc_hi } else { ret = alloc_blks(nblks, hints, mbid); } - HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!"); - out_blkid = mbid.to_single_blkid(); + + if (ret == BlkAllocStatus::SUCCESS || (ret == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) { + HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!"); + out_blkid = mbid.to_single_blkid(); + } + + // for failure case, fall through and return the status to caller; } catch (const std::exception& e) { ret = BlkAllocStatus::FAILED; HS_DBG_ASSERT(0, "{}", e.what()); @@ -235,6 +241,13 @@ BlkAllocStatus VirtualDev::alloc_blks(blk_count_t nblks, blk_alloc_hints const& auto nblks_this_iter = out_bid.blk_count(); nblks_remain = (nblks_remain < nblks_this_iter) ? 0 : (nblks_remain - nblks_this_iter); + + if (status != BlkAllocStatus::SUCCESS && status != BlkAllocStatus::PARTIAL) { + out_blkids.pop_back(); + // all chunks has been tried, but still failed to allocate; + // break out and return status to caller; + break; + } } while (nblks_remain); return status; diff --git a/src/lib/homestore.cpp b/src/lib/homestore.cpp index dca539a8b..328e3cc0d 100644 --- a/src/lib/homestore.cpp +++ b/src/lib/homestore.cpp @@ -230,9 +230,17 @@ void HomeStore::shutdown() { LOGINFO("Homestore shutdown is started"); + m_cp_mgr->shutdown(); + m_cp_mgr.reset(); + + if (has_repl_data_service()) { + s_cast< GenericReplService* >(m_repl_service.get())->stop(); + m_repl_service.reset(); + } + if (has_index_service()) { m_index_service->stop(); - // m_index_service.reset(); + // m_index_service.reset(); } if (has_repl_data_service()) { s_cast< GenericReplService* >(m_repl_service.get())->stop(); @@ -253,8 +261,10 @@ void HomeStore::shutdown() { m_dev_mgr->close_devices(); m_dev_mgr.reset(); - m_cp_mgr->shutdown(); - m_cp_mgr.reset(); + +#ifdef _PRERELEASE + flip::Flip::instance().stop_rpc_server(); +#endif HomeStore::reset_instance(); LOGINFO("Homestore is completed its shutdown"); diff --git a/src/lib/index/index_service.cpp b/src/lib/index/index_service.cpp index eebc63d7f..2c5e096bc 100644 --- a/src/lib/index/index_service.cpp +++ b/src/lib/index/index_service.cpp @@ -78,11 +78,6 @@ void IndexService::start() { void IndexService::stop() { std::unique_lock lg(m_index_map_mtx); - auto fut = homestore::hs()->cp_mgr().trigger_cp_flush(true /* force */); - auto success = std::move(fut).get(); - HS_REL_ASSERT_EQ(success, true, "CP Flush failed"); - LOGINFO("CP Flush completed"); - for (auto [id, tbl] : m_index_map) { tbl->destroy(); } diff --git a/src/lib/meta/meta_blk_service.cpp b/src/lib/meta/meta_blk_service.cpp index 77f2c4de2..630702af3 100644 --- a/src/lib/meta/meta_blk_service.cpp +++ b/src/lib/meta/meta_blk_service.cpp @@ -930,6 +930,8 @@ void MetaBlkService::free_ovf_blk_chain(const BlkId& obid) { // free on-disk data bid auto* data_bid = ovf_hdr->get_data_bid(); for (decltype(ovf_hdr->h.nbids) i{0}; i < ovf_hdr->h.nbids; ++i) { + HS_LOG(DEBUG, metablk, "before freeing data bid: {}, mstore used size: {}", data_bid[i].to_string(), + m_sb_vdev->used_size()); m_sb_vdev->free_blk(data_bid[i]); total_nblks_freed += data_bid[i].blk_count(); diff --git a/src/lib/replication/repl_dev/solo_repl_dev.cpp b/src/lib/replication/repl_dev/solo_repl_dev.cpp index 57aa63def..08d9e094e 100644 --- a/src/lib/replication/repl_dev/solo_repl_dev.cpp +++ b/src/lib/replication/repl_dev/solo_repl_dev.cpp @@ -138,7 +138,6 @@ void SoloReplDev::cp_flush(CP*) { m_rd_sb.write(); } -void SoloReplDev::cp_cleanup(CP*) { m_data_journal->truncate(m_rd_sb->checkpoint_lsn); } +void SoloReplDev::cp_cleanup(CP*) { /* m_data_journal->truncate(m_rd_sb->checkpoint_lsn); */ } } // namespace homestore - diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 708aa161b..a9752a3ef 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -37,7 +37,7 @@ if (${build_nonio_tests}) add_executable(test_mem_btree ${TEST_MEMBTREE_SOURCE_FILES}) target_link_libraries(test_mem_btree ${COMMON_TEST_DEPS} GTest::gtest) add_test(NAME MemBtree COMMAND test_mem_btree) - set_tests_properties(MemBtree PROPERTIES TIMEOUT 180) + set_tests_properties(MemBtree PROPERTIES TIMEOUT 600) add_executable(test_blk_read_tracker) target_sources(test_blk_read_tracker PRIVATE test_blk_read_tracker.cpp ../lib/blkdata_svc/blk_read_tracker.cpp ../lib/blkalloc/blk.cpp) @@ -74,6 +74,7 @@ if (${io_tests}) target_link_libraries(test_index_btree homestore ${COMMON_TEST_DEPS} GTest::gtest) add_test(NAME IndexBtree COMMAND test_index_btree) set_property(TEST IndexBtree PROPERTY ENVIRONMENT "ASAN_OPTIONS=detect_stack_use_after_return=true") + set_tests_properties(IndexBtree PROPERTIES TIMEOUT 600) add_executable(test_data_service) target_sources(test_data_service PRIVATE test_data_service.cpp) @@ -110,8 +111,8 @@ if (${io_tests}) add_test(NAME MetaBlkMgr-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_meta_blk_mgr) add_test(NAME DataService-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_data_service) add_test(NAME SoloReplDev-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_solo_repl_dev) - add_test(NAME HomeRaftLogStore-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_home_raft_logstore) - add_test(NAME RaftReplDev-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_raft_repl_dev) + # add_test(NAME HomeRaftLogStore-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_home_raft_logstore) + # add_test(NAME RaftReplDev-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_raft_repl_dev) endif() can_build_spdk_io_tests(spdk_tests) diff --git a/src/tests/btree_helpers/btree_test_helper.hpp b/src/tests/btree_helpers/btree_test_helper.hpp index be5cc4e14..2cb03bca1 100644 --- a/src/tests/btree_helpers/btree_test_helper.hpp +++ b/src/tests/btree_helpers/btree_test_helper.hpp @@ -50,7 +50,7 @@ struct BtreeTestHelper { if (m_is_multi_threaded) { std::mutex mtx; m_run_time = SISL_OPTIONS["run_time"].as< uint32_t >(); - iomanager.run_on_wait(iomgr::reactor_regex::all_io, [this, &mtx]() { + iomanager.run_on_wait(iomgr::reactor_regex::all_worker, [this, &mtx]() { auto fv = iomanager.sync_io_capable_fibers(); std::unique_lock lg(mtx); m_fibers.insert(m_fibers.end(), fv.begin(), fv.end()); @@ -84,27 +84,30 @@ struct BtreeTestHelper { public: void preload(uint32_t preload_size) { - const auto chunk_size = preload_size / m_fibers.size(); - const auto last_chunk_size = preload_size % chunk_size ?: chunk_size; - auto test_count = m_fibers.size(); - - for (std::size_t i = 0; i < m_fibers.size(); ++i) { - const auto start_range = i * chunk_size; - const auto end_range = start_range + ((i == m_fibers.size() - 1) ? last_chunk_size : chunk_size); - iomanager.run_on_forget(m_fibers[i], [this, start_range, end_range, &test_count]() { - for (uint32_t i = start_range; i < end_range; i++) { - put(i, btree_put_type::INSERT); - } - { - std::unique_lock lg(m_test_done_mtx); - if (--test_count == 0) { m_test_done_cv.notify_one(); } - } - }); - } + if (preload_size) { + const auto n_fibers = std::min(preload_size, (uint32_t)m_fibers.size()); + const auto chunk_size = preload_size / n_fibers; + const auto last_chunk_size = preload_size % chunk_size ?: chunk_size; + auto test_count = n_fibers; + + for (std::size_t i = 0; i < n_fibers; ++i) { + const auto start_range = i * chunk_size; + const auto end_range = start_range + ((i == n_fibers - 1) ? last_chunk_size : chunk_size); + iomanager.run_on_forget(m_fibers[i], [this, start_range, end_range, &test_count]() { + for (uint32_t i = start_range; i < end_range; i++) { + put(i, btree_put_type::INSERT); + } + { + std::unique_lock lg(m_test_done_mtx); + if (--test_count == 0) { m_test_done_cv.notify_one(); } + } + }); + } - { - std::unique_lock< std::mutex > lk(m_test_done_mtx); - m_test_done_cv.wait(lk, [&]() { return test_count == 0; }); + { + std::unique_lock< std::mutex > lk(m_test_done_mtx); + m_test_done_cv.wait(lk, [&]() { return test_count == 0; }); + } } LOGINFO("Preload Done"); } @@ -157,15 +160,13 @@ struct BtreeTestHelper { auto pk = std::make_unique< K >(k); auto rreq = BtreeSingleRemoveRequest{pk.get(), existing_v.get()}; + rreq.enable_route_tracing(); bool removed = (m_bt->remove(rreq) == btree_status_t::success); ASSERT_EQ(removed, m_shadow_map.exists(*pk)) << "Removal of key " << pk->key() << " status doesn't match with shadow"; - if (removed) { - m_shadow_map.validate_data(rreq.key(), (const V&)rreq.value()); - m_shadow_map.erase(rreq.key()); - } + if (removed) { m_shadow_map.remove_and_check(*pk, *existing_v); } } void remove_random() { @@ -213,14 +214,17 @@ struct BtreeTestHelper { auto const expected_count = std::min(remaining, batch_size); ASSERT_EQ(out_vector.size(), expected_count) << "Received incorrect value on query pagination"; - remaining -= expected_count; - if (remaining == 0) { + if (remaining < batch_size) { ASSERT_EQ(ret, btree_status_t::success) << "Expected success on query"; - } else { + } else if (remaining > batch_size) { ASSERT_EQ(ret, btree_status_t::has_more) << "Expected query to return has_more"; + } else if (remaining == batch_size) { + // we don't know, go to the next round } + remaining -= expected_count; + for (size_t idx{0}; idx < out_vector.size(); ++idx) { ASSERT_EQ(out_vector[idx].second, it->second) << "Range get doesn't return correct data for key=" << it->first << " idx=" << idx; @@ -253,7 +257,7 @@ struct BtreeTestHelper { *copy_key = key; auto out_v = std::make_unique< V >(); auto req = BtreeSingleGetRequest{copy_key.get(), out_v.get()}; - + req.enable_route_tracing(); const auto ret = m_bt->get(req); ASSERT_EQ(ret, btree_status_t::success) << "Missing key " << key << " in btree but present in shadow map"; ASSERT_EQ((const V&)req.value(), value) @@ -265,7 +269,7 @@ struct BtreeTestHelper { auto pk = std::make_unique< K >(k); auto out_v = std::make_unique< V >(); auto req = BtreeSingleGetRequest{pk.get(), out_v.get()}; - + req.enable_route_tracing(); const auto status = m_bt->get(req); if (status == btree_status_t::success) { m_shadow_map.validate_data(req.key(), (const V&)req.value()); @@ -279,6 +283,7 @@ struct BtreeTestHelper { auto out_v = std::make_unique< V >(); auto req = BtreeGetAnyRequest< K >{BtreeKeyRange< K >{K{start_k}, true, K{end_k}, true}, out_k.get(), out_v.get()}; + req.enable_route_tracing(); const auto status = m_bt->get(req); if (status == btree_status_t::success) { @@ -335,6 +340,7 @@ struct BtreeTestHelper { auto existing_v = std::make_unique< V >(); K key = K{k}; auto sreq = BtreeSinglePutRequest{&key, &value, put_type, existing_v.get()}; + sreq.enable_route_tracing(); bool done = (m_bt->put(sreq) == btree_status_t::success); if (put_type == btree_put_type::INSERT) { @@ -351,43 +357,73 @@ struct BtreeTestHelper { K end_key = K{end_k}; auto rreq = BtreeRangeRemoveRequest< K >{BtreeKeyRange< K >{start_key, true, end_key, true}}; + rreq.enable_route_tracing(); auto const ret = m_bt->remove(rreq); - m_shadow_map.range_erase(start_key, end_key); - if (all_existing) { + m_shadow_map.range_erase(start_key, end_key); ASSERT_EQ((ret == btree_status_t::success), true) << "not a successful remove op for range " << start_k << "-" << end_k; - } - - if (start_k < m_max_range_input) { - m_shadow_map.remove_keys(start_k, std::min(end_k, uint64_cast(m_max_range_input - 1))); + } else if (start_k < m_max_range_input) { + K end_range{std::min(end_k, uint64_cast(m_max_range_input - 1))}; + m_shadow_map.range_erase(start_key, end_range); } } protected: void run_in_parallel(const std::vector< std::pair< std::string, int > >& op_list) { auto test_count = m_fibers.size(); - for (auto it = m_fibers.begin(); it < m_fibers.end(); ++it) { - iomanager.run_on_forget(*it, [this, &test_count, op_list]() { + const auto total_iters = SISL_OPTIONS["num_iters"].as< uint32_t >(); + const auto num_iters_per_thread = total_iters / m_fibers.size(); + const auto extra_iters = total_iters % num_iters_per_thread; + LOGINFO("number of fibers {} num_iters_per_thread {} extra_iters {} ", m_fibers.size(), num_iters_per_thread, + extra_iters); + + for (uint32_t fiber_id = 0; fiber_id < m_fibers.size(); ++fiber_id) { + auto num_iters_this_fiber = num_iters_per_thread + (fiber_id < extra_iters ? 1 : 0); + iomanager.run_on_forget(m_fibers[fiber_id], [this, fiber_id, &test_count, op_list, num_iters_this_fiber]() { std::random_device g_rd{}; std::default_random_engine re{g_rd()}; - const auto num_iters_per_thread = - sisl::round_up(SISL_OPTIONS["num_iters"].as< uint32_t >() / m_fibers.size(), m_fibers.size()); std::vector< uint32_t > weights; std::transform(op_list.begin(), op_list.end(), std::back_inserter(weights), [](const auto& pair) { return pair.second; }); + double progress_interval = (double)num_iters_this_fiber / 20; // 5% of the total number of iterations + double progress_thresh = progress_interval; // threshold for progress interval + double elapsed_time, progress_percent, last_progress_time = 0; + // Construct a weighted distribution based on the input frequencies std::discrete_distribution< uint32_t > s_rand_op_generator(weights.begin(), weights.end()); auto m_start_time = Clock::now(); + auto time_to_stop = [this, m_start_time]() { + return (get_elapsed_time_sec(m_start_time) > m_run_time); + }; - auto time_to_stop = [this, m_start_time]() {return (get_elapsed_time_sec(m_start_time) > m_run_time);}; - - for (uint32_t i = 0; i < num_iters_per_thread && !time_to_stop(); i++) { + for (uint32_t i = 0; i < num_iters_this_fiber && !time_to_stop(); i++) { uint32_t op_idx = s_rand_op_generator(re); (this->m_operations[op_list[op_idx].first])(); m_num_ops.fetch_add(1); + + if (fiber_id == 0) { + elapsed_time = get_elapsed_time_sec(m_start_time); + progress_percent = (double)i / num_iters_this_fiber * 100; + + // check progress every 5% of the total number of iterations or every 30 seconds + bool print_time = false; + if (i >= progress_thresh) { + progress_thresh += progress_interval; + print_time = true; + } + if (elapsed_time - last_progress_time > 30) { + last_progress_time = elapsed_time; + print_time = true; + } + if (print_time) { + LOGINFO("Progress: iterations completed ({:.2f}%)- Elapsed time: {:.0f} seconds of total " + "{} - total entries: {}", + progress_percent, elapsed_time, m_run_time, m_shadow_map.size()); + } + } } { std::unique_lock lg(m_test_done_mtx); diff --git a/src/tests/btree_helpers/shadow_map.hpp b/src/tests/btree_helpers/shadow_map.hpp index f8c40e140..edd6e567f 100644 --- a/src/tests/btree_helpers/shadow_map.hpp +++ b/src/tests/btree_helpers/shadow_map.hpp @@ -48,7 +48,7 @@ class ShadowMap { } std::pair< K, K > pick_existing_range(const K& start_key, uint32_t max_count) const { - std::shared_lock lock{m_mutex}; + std::lock_guard lock{m_mutex}; auto const start_it = m_map.lower_bound(start_key); auto it = start_it; uint32_t count = 0; @@ -59,12 +59,12 @@ class ShadowMap { } bool exists(const K& key) const { - std::shared_lock lock{m_mutex}; + std::lock_guard lock{m_mutex}; return m_map.find(key) != m_map.end(); } bool exists_in_range(const K& key, uint64_t start_k, uint64_t end_k) const { - std::shared_lock lock{m_mutex}; + std::lock_guard lock{m_mutex}; const auto itlower = m_map.lower_bound(K{start_k}); const auto itupper = m_map.upper_bound(K{end_k}); auto it = itlower; @@ -76,7 +76,7 @@ class ShadowMap { } uint64_t size() const { - std::shared_lock lock{m_mutex}; + std::lock_guard lock{m_mutex}; return m_map.size(); } @@ -87,12 +87,21 @@ class ShadowMap { } void validate_data(const K& key, const V& btree_val) const { - std::shared_lock lock{m_mutex}; + std::lock_guard lock{m_mutex}; const auto r = m_map.find(key); ASSERT_NE(r, m_map.end()) << "Key " << key.to_string() << " is not present in shadow map"; ASSERT_EQ(btree_val, r->second) << "Found value in btree doesn't return correct data for key=" << r->first; } + void remove_and_check(const K& key, const V& btree_val) { + std::lock_guard lock{m_mutex}; + const auto r = m_map.find(key); + ASSERT_NE(r, m_map.end()) << "Key " << key.to_string() << " is not present in shadow map"; + ASSERT_EQ(btree_val, r->second) << "Found value in btree doesn't return correct data for key=" << r->first; + m_map.erase(key); + m_range_scheduler.remove_key(key.key()); + } + void erase(const K& key) { std::lock_guard lock{m_mutex}; m_map.erase(key); @@ -101,7 +110,7 @@ class ShadowMap { void range_erase(const K& start_key, uint32_t count) { std::lock_guard lock{m_mutex}; - auto const it = m_map.lower_bound(start_key); + auto it = m_map.lower_bound(start_key); uint32_t i{0}; while ((it != m_map.cend()) && (i++ < count)) { it = m_map.erase(it); @@ -124,25 +133,34 @@ class ShadowMap { const std::map< K, V >& map_const() const { return m_map; } void foreach (std::function< void(K, V) > func) const { - std::shared_lock lock{m_mutex}; + std::lock_guard lock{m_mutex}; for (const auto& [key, value] : m_map) { func(key, value); } } std::pair< uint32_t, uint32_t > pick_random_non_existing_keys(uint32_t max_keys) { - std::shared_lock lock{m_mutex}; - return m_range_scheduler.pick_random_non_existing_keys(max_keys); + do { + std::lock_guard lock{m_mutex}; + auto ret = m_range_scheduler.pick_random_non_existing_keys(max_keys); + if (ret.first != UINT32_MAX) { return ret; } + } while (true); } std::pair< uint32_t, uint32_t > pick_random_existing_keys(uint32_t max_keys) { - std::shared_lock lock{m_mutex}; - return m_range_scheduler.pick_random_existing_keys(max_keys); + do { + std::lock_guard lock{m_mutex}; + auto ret = m_range_scheduler.pick_random_existing_keys(max_keys); + if (ret.first != UINT32_MAX) { return ret; } + } while (true); } std::pair< uint32_t, uint32_t > pick_random_non_working_keys(uint32_t max_keys) { - std::shared_lock lock{m_mutex}; - return m_range_scheduler.pick_random_non_working_keys(max_keys); + do { + std::lock_guard lock{m_mutex}; + auto ret = m_range_scheduler.pick_random_non_working_keys(max_keys); + if (ret.first != UINT32_MAX) { return ret; } + } while (true); } void remove_keys_from_working(uint32_t s, uint32_t e) { diff --git a/src/tests/test_common/homestore_test_common.hpp b/src/tests/test_common/homestore_test_common.hpp index 277c54a24..3589fca43 100644 --- a/src/tests/test_common/homestore_test_common.hpp +++ b/src/tests/test_common/homestore_test_common.hpp @@ -221,7 +221,7 @@ class HSTestHelper { LOGINFO("Starting iomgr with {} threads, spdk: {}", num_threads, is_spdk); ioenvironment.with_iomgr( - iomgr::iomgr_params{.num_threads = num_threads, .is_spdk = is_spdk, .num_fibers = num_fibers}); + iomgr::iomgr_params{.num_threads = num_threads, .is_spdk = is_spdk, .num_fibers = 1 + num_fibers}); auto const http_port = SISL_OPTIONS["http_port"].as< int >(); if (http_port != 0) { diff --git a/src/tests/test_common/hs_repl_test_common.hpp b/src/tests/test_common/hs_repl_test_common.hpp index e2e18b2b2..19dec9959 100644 --- a/src/tests/test_common/hs_repl_test_common.hpp +++ b/src/tests/test_common/hs_repl_test_common.hpp @@ -79,8 +79,8 @@ class HSReplTestHelper { if (count == SISL_OPTIONS["replicas"].as< uint32_t >()) { phase_ = new_phase; cv_.notify_all(); - } - cv_.wait(lg, [this, new_phase]() { return (phase_ == new_phase); }); + } else + cv_.wait(lg, [this, new_phase]() { return (phase_ == new_phase); }); } }; @@ -136,6 +136,11 @@ class HSReplTestHelper { if (replica_num_ == 0) { // Erase previous shmem and create a new shmem with IPCData structure bip::shared_memory_object::remove("raft_repl_test_shmem"); + + // kill the previous processes using the port + for (uint32_t i = 0; i < num_replicas; ++i) + check_and_kill(SISL_OPTIONS["base_port"].as< uint16_t >() + i); + shm_ = std::make_unique< bip::shared_memory_object >(bip::create_only, "raft_repl_test_shmem", bip::read_write); shm_->truncate(sizeof(IPCData)); @@ -223,6 +228,22 @@ class HSReplTestHelper { void sync_dataset_size(uint64_t dataset_size) { ipc_data_->test_dataset_size_ = dataset_size; } uint64_t dataset_size() const { return ipc_data_->test_dataset_size_; } + void check_and_kill(int port) { + std::string command = "lsof -t -i:" + std::to_string(port); + if (system(command.c_str())) { + std::cout << "Port " << port << " is not in use." << std::endl; + } else { + std::cout << "Port " << port << " is in use. Trying to kill the process..." << std::endl; + command += " | xargs kill -9"; + int result = system(command.c_str()); + if (result == 0) { + std::cout << "Killed the process using port " << port << std::endl; + } else { + std::cout << "Failed to kill the process." << std::endl; + } + } + } + private: uint16_t replica_num_; std::string name_; diff --git a/src/tests/test_common/range_scheduler.hpp b/src/tests/test_common/range_scheduler.hpp index 5dc2e4d1b..9c6bd937a 100644 --- a/src/tests/test_common/range_scheduler.hpp +++ b/src/tests/test_common/range_scheduler.hpp @@ -24,8 +24,6 @@ #include namespace homestore { -using mutex = iomgr::FiberManagerLib::shared_mutex; - static std::pair< uint64_t, uint64_t > get_next_contiguous_set_bits(const sisl::Bitset& bm, uint64_t search_start_bit, uint64_t max_count) { uint64_t first_set_bit{sisl::Bitset::npos}; @@ -48,9 +46,7 @@ class RangeScheduler { private: sisl::Bitset m_existing_keys; sisl::Bitset m_working_keys; - mutex m_set_lock; std::uniform_int_distribution< uint32_t > m_rand_start_key_generator; - std::random_device m_rd; public: @@ -58,68 +54,63 @@ class RangeScheduler { m_rand_start_key_generator = std::uniform_int_distribution< uint32_t >(0, num_keys - 1); } - void remove_keys_from_working(uint32_t s, uint32_t e) { - std::unique_lock< mutex > lk(m_set_lock); - remove_from_working(s, e); - } + void remove_keys_from_working(uint32_t s, uint32_t e) { remove_from_working(s, e); } void put_key(uint32_t key) { - std::unique_lock< mutex > lk(m_set_lock); add_to_existing(key); remove_from_working(key); } void put_keys(uint32_t start_key, uint32_t end_key) { - std::unique_lock< mutex > lk(m_set_lock); add_to_existing(start_key, end_key); remove_from_working(start_key, end_key); } void remove_key(uint32_t key) { - std::unique_lock< mutex > lk(m_set_lock); remove_from_existing(key); remove_from_working(key); } void remove_keys(uint32_t start_key, uint32_t end_key) { - std::unique_lock< mutex > lk(m_set_lock); remove_from_existing(start_key, end_key); remove_from_working(start_key, end_key); } std::pair< uint32_t, uint32_t > pick_random_non_existing_keys(uint32_t max_keys) { std::pair< uint32_t, uint32_t > ret; + auto max_tries = 10; do { ret = try_pick_random_non_existing_keys(max_keys); if (ret.first != UINT32_MAX) { break; } - } while (true); + } while (--max_tries); return ret; } std::pair< uint32_t, uint32_t > pick_random_existing_keys(uint32_t max_keys) { std::pair< uint32_t, uint32_t > ret; + auto max_tries = 10; do { ret = try_pick_random_existing_keys(max_keys); if (ret.first != UINT32_MAX) { break; } - } while (true); + } while (--max_tries); return ret; } std::pair< uint32_t, uint32_t > pick_random_non_working_keys(uint32_t max_keys) { std::pair< uint32_t, uint32_t > ret; + auto max_tries = 10; do { ret = try_pick_random_non_working_keys(max_keys); if (ret.first != UINT32_MAX) { break; } - } while (true); + } while (--max_tries); return ret; } private: std::pair< uint32_t, uint32_t > try_pick_random_non_existing_keys(uint32_t max_keys) { - std::unique_lock< mutex > lk(m_set_lock); if ((m_existing_keys.size() - m_existing_keys.get_set_count()) == 0) { throw std::out_of_range("All keys are being worked on right now"); } @@ -137,7 +128,6 @@ class RangeScheduler { } std::pair< uint32_t, uint32_t > try_pick_random_existing_keys(uint32_t max_keys) { - std::unique_lock< mutex > lk(m_set_lock); if (m_existing_keys.get_set_count() == 0) { DEBUG_ASSERT(false, "Couldn't find one existing keys"); throw std::out_of_range("Couldn't find one existing keys"); @@ -157,8 +147,6 @@ class RangeScheduler { } std::pair< uint32_t, uint32_t > try_pick_random_non_working_keys(uint32_t max_keys) { - std::unique_lock< mutex > lk(m_set_lock); - uint32_t const search_start = m_rand_start_key_generator(m_rd); auto bb = m_working_keys.get_next_contiguous_n_reset_bits(search_start, max_keys); diff --git a/src/tests/test_data_service.cpp b/src/tests/test_data_service.cpp index 5af59445f..ba10d5784 100644 --- a/src/tests/test_data_service.cpp +++ b/src/tests/test_data_service.cpp @@ -495,7 +495,9 @@ class BlkDataServiceTest : public testing::Test { sg->size += iov_len; } - return inst().async_alloc_write(*(sg.get()), blk_alloc_hints{}, out_bids, false /* part_of_batch*/); + auto fut = inst().async_alloc_write(*(sg.get()), blk_alloc_hints{}, out_bids, false /* part_of_batch*/); + inst().commit_blk(out_bids); + return fut; } void verify_read_blk_crc(sisl::sg_list& sg, std::vector< uint64_t > read_crc_vec) { diff --git a/src/tests/test_index_btree.cpp b/src/tests/test_index_btree.cpp index 1e833b059..eb4028fc4 100644 --- a/src/tests/test_index_btree.cpp +++ b/src/tests/test_index_btree.cpp @@ -33,7 +33,6 @@ SISL_LOGGING_DECL(test_index_btree) std::vector< std::string > test_common::HSTestHelper::s_dev_names; - // TODO Add tests to do write,remove after recovery. // TODO Test with var len key with io mgr page size is 512. @@ -121,7 +120,7 @@ struct BtreeTest : public BtreeTestHelper< TestType >, public ::testing::Test { } }; -using BtreeTypes = testing::Types< FixedLenBtree, VarKeySizeBtree, VarValueSizeBtree, VarObjSizeBtree >; +using BtreeTypes = testing::Types< FixedLenBtree /* , VarKeySizeBtree, VarValueSizeBtree, VarObjSizeBtree */ >; TYPED_TEST_SUITE(BtreeTest, BtreeTypes); @@ -367,7 +366,6 @@ TYPED_TEST(BtreeTest, ThreadedCpFlush) { // Remove a random entry. std::uniform_int_distribution< uint32_t > rand{0, last_index.load()}; auto rm_idx = rand(g_re); - LOGINFO("Removing entry {}", rm_idx); this->remove_one(rm_idx); } }); @@ -431,7 +429,6 @@ struct BtreeConcurrentTest : public BtreeTestHelper< TestType >, public ::testin BtreeConcurrentTest* m_test; }; - BtreeConcurrentTest() : testing::Test() { this->m_is_multi_threaded = true; } void SetUp() override { @@ -469,7 +466,7 @@ struct BtreeConcurrentTest : public BtreeTestHelper< TestType >, public ::testin TYPED_TEST_SUITE(BtreeConcurrentTest, BtreeTypes); TYPED_TEST(BtreeConcurrentTest, ConcurrentAllOps) { // range put is not supported for non-extent keys - std::vector< std::string > input_ops = {"put:20", "remove:20", "range_put:20", "range_remove:20", "query:20"}; + std::vector< std::string > input_ops = {"put:19", "remove:14", "range_put:20", "range_remove:2", "query:10"}; if (SISL_OPTIONS.count("operation_list")) { input_ops = SISL_OPTIONS["operation_list"].as< std::vector< std::string > >(); } diff --git a/src/tests/test_meta_blk_mgr.cpp b/src/tests/test_meta_blk_mgr.cpp index 559fd3eef..75de69089 100644 --- a/src/tests/test_meta_blk_mgr.cpp +++ b/src/tests/test_meta_blk_mgr.cpp @@ -88,6 +88,7 @@ class VMetaBlkMgrTest : public ::testing::Test { std::vector< meta_sub_type > actual_cb_order; std::vector< meta_sub_type > actual_on_complete_cb_order; std::vector< void* > cookies; + bool enable_dependency_chain{false}; VMetaBlkMgrTest() = default; VMetaBlkMgrTest(const VMetaBlkMgrTest&) = delete; @@ -120,8 +121,12 @@ class VMetaBlkMgrTest : public ::testing::Test { } void restart_homestore() { + auto before_services_starting_cb = [this]() { + register_client(); + if (enable_dependency_chain) { register_client_inlcuding_dependencies(); } + }; test_common::HSTestHelper::start_homestore("test_meta_blk_mgr", {{HS_SERVICE::META, {.size_pct = 85.0}}}, - nullptr /* before_svc_start_cb */, true /* restart */); + std::move(before_services_starting_cb), true /* restart */); } uint64_t io_cnt() const { return m_update_cnt + m_wrt_cnt + m_rm_cnt; } @@ -408,7 +413,7 @@ class VMetaBlkMgrTest : public ::testing::Test { LOGINFO("compression ratio limit changed to: {}", HS_DYNAMIC_CONFIG(metablk.compress_ratio_limit)); - do_sb_write(true /* do_overflow */, 15 * Mi); + do_sb_write(true /* do_overflow */, 5 * Mi); HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { s.metablk.compress_ratio_limit = 0; // this will disallow every compression attempt; @@ -418,7 +423,7 @@ class VMetaBlkMgrTest : public ::testing::Test { LOGINFO("compression ratio limit changed to: {}", HS_DYNAMIC_CONFIG(metablk.compress_ratio_limit)); // since we only wrote 1 metablk, it will always pick up the same one; - do_sb_update(true /* aligned */, 12 * Mi); + do_sb_update(true /* aligned */, 5 * Mi); } void do_rand_load() { @@ -457,24 +462,10 @@ class VMetaBlkMgrTest : public ::testing::Test { return (aligned_rand(re) == s_cast< uint8_t >(1)); } - void recover() { - // TODO: This scan_blks and recover should be replaced with actual TestHelper::start_homestore with restart - // on. That way, we don't need to simulate all these calls here - // do recover and callbacks will be triggered; - m_cb_blks.clear(); - hs()->cp_mgr().shutdown(); - hs()->cp_mgr().start(false /* first_time_boot */); - m_mbm->recover(false); - } - void recover_with_on_complete() { - // TODO: This scan_blks and recover should be replaced with actual TestHelper::start_homestore with restart - // on. That way, we don't need to simulate all these calls here - // do recover and callbacks will be triggered; + // restart will cause recovery and callbacks will be triggered m_cb_blks.clear(); - hs()->cp_mgr().shutdown(); - hs()->cp_mgr().start(false /* first_time_boot */); - m_mbm->recover(true); + restart_homestore(); } void validate() { @@ -482,8 +473,6 @@ class VMetaBlkMgrTest : public ::testing::Test { verify_cb_blks(); } - void scan_blks() { m_mbm->scan_meta_blks(); } - meta_op_type get_op() { static thread_local bool keep_remove{false}; // if we hit some high watermark, remove the sbs until hit some low watermark; @@ -571,6 +560,7 @@ class VMetaBlkMgrTest : public ::testing::Test { } void register_client_inlcuding_dependencies() { + enable_dependency_chain = true; m_mbm = &(meta_service()); m_total_wrt_sz = m_mbm->used_size(); @@ -639,6 +629,8 @@ class VMetaBlkMgrTest : public ::testing::Test { } void deregister_client_inlcuding_dependencies() { + enable_dependency_chain = false; + m_mbm->deregister_handler("A"); m_mbm->deregister_handler("B"); m_mbm->deregister_handler("C"); @@ -742,9 +734,6 @@ TEST_F(VMetaBlkMgrTest, random_dependency_test) { iomanager.iobuf_free(buf); - // simulate reboot case that MetaBlkMgr will scan the disk for all the metablks that were written; - this->scan_blks(); - this->recover_with_on_complete(); std::unordered_map< meta_sub_type, int > actual_first_cb_order_map; @@ -777,7 +766,6 @@ TEST_F(VMetaBlkMgrTest, random_dependency_test) { EXPECT_TRUE(actual_first_cb_order_map["F"] < actual_first_cb_order_map["C"]); this->deregister_client_inlcuding_dependencies(); - this->shutdown(); } @@ -816,10 +804,7 @@ TEST_F(VMetaBlkMgrTest, random_load_test) { this->do_rand_load(); - // simulate reboot case that MetaBlkMgr will scan the disk for all the metablks that were written; - this->scan_blks(); - - this->recover(); + this->recover_with_on_complete(); this->validate(); @@ -861,11 +846,7 @@ TEST_F(VMetaBlkMgrTest, RecoveryFromBadData) { // Then do a recovery, the data read from disk should be uncompressed and match the size we saved in its metablk // header. If size mismatch, it will hit assert failure; // - - // simulate reboot case that MetaBlkMgr will scan the disk for all the metablks that were written; - this->scan_blks(); - - this->recover(); + this->recover_with_on_complete(); this->validate(); @@ -892,11 +873,7 @@ TEST_F(VMetaBlkMgrTest, CompressionBackoff) { // Then do a recovery, the data read from disk should be uncompressed and match the size we saved in its metablk // header. If size mismatch, it will hit assert failure; // - - // simulate reboot case that MetaBlkMgr will scan the disk for all the metablks that were written; - this->scan_blks(); - - this->recover(); + this->recover_with_on_complete(); this->validate(); diff --git a/src/tests/test_scripts/CMakeLists.txt b/src/tests/test_scripts/CMakeLists.txt index c774ea025..aafa6842d 100644 --- a/src/tests/test_scripts/CMakeLists.txt +++ b/src/tests/test_scripts/CMakeLists.txt @@ -2,6 +2,7 @@ file(COPY vol_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) file(COPY home_blk_flip.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) file(COPY home_blk_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) file(COPY btree_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) +file(COPY log_meta_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) #add_test(NAME TestVolRecovery COMMAND ${CMAKE_BINARY_DIR}/bin/scripts/vol_test.py --test_suits=recovery --dirpath=${CMAKE_BINARY_DIR}/bin/) #SET_TESTS_PROPERTIES(TestVolRecovery PROPERTIES DEPENDS TestVol) diff --git a/src/tests/test_scripts/btree_test.py b/src/tests/test_scripts/btree_test.py index dc6390fb6..55d257955 100755 --- a/src/tests/test_scripts/btree_test.py +++ b/src/tests/test_scripts/btree_test.py @@ -17,11 +17,11 @@ dirpath = "./" op_list = "" log_mods = "" -threads = " --num_threads=10" -fibers = " --num_fibers=10" -preload_size = " --preload_size=16384" -num_entries = " --num_entries=65536" -num_iters = " --num_iters=10000000" +threads = " --num_threads=5" +fibers = " --num_fibers=5" +preload_size = " --preload_size=262144" +num_entries = " --num_entries=1048576" +num_iters = " --num_iters=100000000" run_time = " --run_time=36000" dev_list = "" @@ -76,7 +76,7 @@ def normal(): print("normal test started with (%s)" % btree_options) # " --operation_list=query:20 --operation_list=put:20 --operation_list=remove:20" - cmd_opts = " --gtest_filter=BtreeConcurrentTest/*.ConcurrentAllOps --gtest_break_on_failure " + btree_options + " "+log_mods + cmd_opts = " --gtest_filter=BtreeConcurrentTest/0.ConcurrentAllOps --gtest_break_on_failure " + btree_options + " "+log_mods subprocess.check_call(dirpath + "test_index_btree " + cmd_opts, stderr=subprocess.STDOUT, shell=True) print("normal test completed") diff --git a/src/tests/test_scripts/log_meta_test.py b/src/tests/test_scripts/log_meta_test.py new file mode 100755 index 000000000..765680e31 --- /dev/null +++ b/src/tests/test_scripts/log_meta_test.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +## @file vol_test.py + +import subprocess +import os +import sys +import getopt +import sys +from multiprocessing import Process +sys.stdout.flush() +from time import sleep +import requests +from threading import Thread + +opts,args = getopt.getopt(sys.argv[1:], 'tdlme:', ['test_suits=', 'dirpath=', 'dev_list=', 'log_mods=', 'emulate_hdd=', 'http_port='] ) +test_suits = "" +dirpath = "./" +dev_list = "" +log_mods = "" +http_port = "" + +for opt,arg in opts: + if opt in ('-t', '--test_suits'): + test_suits = arg + print(("testing suits (%s)") % (arg)) + if opt in ('-d', '--dirpath'): + dirpath = arg + print(("dir path (%s)") % (arg)) + if opt in ('-l', '--dev_list'): + dev_list = arg + print(("device list (%s)") % (arg)) + if opt in ('-m', '--log_mods'): + log_mods = arg + print(("log_mods (%s)") % (arg)) + if opt in ('-p', '--http_port'): + http_port = " --http_port " + arg + print(("http_port (%s)") % (arg)) + +addln_opts = ' ' +if bool(dev_list and dev_list.strip()): + addln_opts += ' --device_list ' + addln_opts += dev_list + +if bool(log_mods and log_mods.strip()): + addln_opts += ' --log_mods ' + addln_opts += log_mods + +addln_opts += ' '.join(map(str, args)) + +print("addln_opts: " + addln_opts) + + +def meta_svc_nightly(): + print("meta blk store test started") + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.CompressionBackoff" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.RecoveryFromBadData" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.min_drive_size_test" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.single_read_test" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--run_time=7200 --num_io=1000000" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--min_write_size=65536 --max_write_size=2097152 --run_time=14400 --num_io=1000000" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--min_write_size=10485760 --max_write_size=104857600 --bitmap=1" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.write_to_full_test" # write to file instead of real disk to save time; + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + print("meta blk store test completed") + +def logstore_nightly(): + print("log store test started") + + cmd_opts = "--iterations=10" + subprocess.check_call(dirpath + "test_log_store " + cmd_opts + http_port, stderr=subprocess.STDOUT, shell=True) + + print("log store test completed") + +def nightly(): + logstore_nightly() + sleep(5) + + meta_svce_nightly() + sleep(5) + +# The name of the method to be called is the var test_suits +eval(f"{test_suits}()") +