diff --git a/.jenkins/Dockerfile b/.jenkins/Dockerfile index 86f486042..cc3556eea 100644 --- a/.jenkins/Dockerfile +++ b/.jenkins/Dockerfile @@ -13,7 +13,10 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/*; COPY test_index_btree /usr/local/bin/test_index_btree +COPY test_meta_blk_mgr /usr/local/bin/test_meta_blk_mgr +COPY test_log_store /usr/local/bin/test_log_store COPY btree_test.py /usr/local/bin/scripts/btree_test.py +COPY log_meta_test.py /usr/local/bin/scripts/log_meta_test.py EXPOSE 5000 -# ########## ####### ############ \ No newline at end of file +# ########## ####### ############ diff --git a/.jenkins/jenkinsfile_nightly b/.jenkins/jenkinsfile_nightly index 527af6d26..05b9a9d88 100644 --- a/.jenkins/jenkinsfile_nightly +++ b/.jenkins/jenkinsfile_nightly @@ -42,7 +42,10 @@ pipeline { steps { sh "conan create --build missing -o sisl:prerelease=True -o homestore:sanitize=True -o homestore:skip_testing=True -pr debug . ${PROJECT}/${VER}@" sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;" + sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr\\;" + sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_log_store' -exec cp {} .jenkins/test_log_store\\;" sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/btree_test.py' -exec install -Dm755 {} .jenkins/btree_test.py \\; " + sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/log_meta_test.py' -exec install -Dm755 {} .jenkins/log_meta_test.py \\; " } post { failure { diff --git a/src/lib/device/virtual_dev.cpp b/src/lib/device/virtual_dev.cpp index 9bf7e4422..a63bee2ba 100644 --- a/src/lib/device/virtual_dev.cpp +++ b/src/lib/device/virtual_dev.cpp @@ -150,9 +150,10 @@ BlkAllocStatus VirtualDev::commit_blk(BlkId const& blkid) { HS_LOG(DEBUG, device, "commit_blk: bid {}", blkid.to_string()); auto const recovering = homestore::hs()->is_initializing(); if (!recovering) { - HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode", blkid.to_string()); + HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode", + blkid.to_string()); } else { - chunk->blk_allocator_mutable()->mark_blk_allocated(blkid); + chunk->blk_allocator_mutable()->mark_blk_allocated(blkid); } return chunk->blk_allocator_mutable()->alloc_on_disk(blkid); } @@ -169,8 +170,13 @@ BlkAllocStatus VirtualDev::alloc_contiguous_blks(blk_count_t nblks, blk_alloc_hi } else { ret = alloc_blks(nblks, hints, mbid); } - HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!"); - out_blkid = mbid.to_single_blkid(); + + if (ret == BlkAllocStatus::SUCCESS || (ret == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) { + HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!"); + out_blkid = mbid.to_single_blkid(); + } + + // for failure case, fall through and return the status to caller; } catch (const std::exception& e) { ret = BlkAllocStatus::FAILED; HS_DBG_ASSERT(0, "{}", e.what()); @@ -235,6 +241,13 @@ BlkAllocStatus VirtualDev::alloc_blks(blk_count_t nblks, blk_alloc_hints const& auto nblks_this_iter = out_bid.blk_count(); nblks_remain = (nblks_remain < nblks_this_iter) ? 0 : (nblks_remain - nblks_this_iter); + + if (status != BlkAllocStatus::SUCCESS && status != BlkAllocStatus::PARTIAL) { + out_blkids.pop_back(); + // all chunks has been tried, but still failed to allocate; + // break out and return status to caller; + break; + } } while (nblks_remain); return status; diff --git a/src/tests/test_meta_blk_mgr.cpp b/src/tests/test_meta_blk_mgr.cpp index 4b009bb78..75de69089 100644 --- a/src/tests/test_meta_blk_mgr.cpp +++ b/src/tests/test_meta_blk_mgr.cpp @@ -413,7 +413,7 @@ class VMetaBlkMgrTest : public ::testing::Test { LOGINFO("compression ratio limit changed to: {}", HS_DYNAMIC_CONFIG(metablk.compress_ratio_limit)); - do_sb_write(true /* do_overflow */, 15 * Mi); + do_sb_write(true /* do_overflow */, 5 * Mi); HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { s.metablk.compress_ratio_limit = 0; // this will disallow every compression attempt; @@ -423,7 +423,7 @@ class VMetaBlkMgrTest : public ::testing::Test { LOGINFO("compression ratio limit changed to: {}", HS_DYNAMIC_CONFIG(metablk.compress_ratio_limit)); // since we only wrote 1 metablk, it will always pick up the same one; - do_sb_update(true /* aligned */, 12 * Mi); + do_sb_update(true /* aligned */, 5 * Mi); } void do_rand_load() { diff --git a/src/tests/test_scripts/CMakeLists.txt b/src/tests/test_scripts/CMakeLists.txt index c774ea025..aafa6842d 100644 --- a/src/tests/test_scripts/CMakeLists.txt +++ b/src/tests/test_scripts/CMakeLists.txt @@ -2,6 +2,7 @@ file(COPY vol_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) file(COPY home_blk_flip.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) file(COPY home_blk_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) file(COPY btree_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) +file(COPY log_meta_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts) #add_test(NAME TestVolRecovery COMMAND ${CMAKE_BINARY_DIR}/bin/scripts/vol_test.py --test_suits=recovery --dirpath=${CMAKE_BINARY_DIR}/bin/) #SET_TESTS_PROPERTIES(TestVolRecovery PROPERTIES DEPENDS TestVol) diff --git a/src/tests/test_scripts/log_meta_test.py b/src/tests/test_scripts/log_meta_test.py new file mode 100755 index 000000000..765680e31 --- /dev/null +++ b/src/tests/test_scripts/log_meta_test.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +## @file vol_test.py + +import subprocess +import os +import sys +import getopt +import sys +from multiprocessing import Process +sys.stdout.flush() +from time import sleep +import requests +from threading import Thread + +opts,args = getopt.getopt(sys.argv[1:], 'tdlme:', ['test_suits=', 'dirpath=', 'dev_list=', 'log_mods=', 'emulate_hdd=', 'http_port='] ) +test_suits = "" +dirpath = "./" +dev_list = "" +log_mods = "" +http_port = "" + +for opt,arg in opts: + if opt in ('-t', '--test_suits'): + test_suits = arg + print(("testing suits (%s)") % (arg)) + if opt in ('-d', '--dirpath'): + dirpath = arg + print(("dir path (%s)") % (arg)) + if opt in ('-l', '--dev_list'): + dev_list = arg + print(("device list (%s)") % (arg)) + if opt in ('-m', '--log_mods'): + log_mods = arg + print(("log_mods (%s)") % (arg)) + if opt in ('-p', '--http_port'): + http_port = " --http_port " + arg + print(("http_port (%s)") % (arg)) + +addln_opts = ' ' +if bool(dev_list and dev_list.strip()): + addln_opts += ' --device_list ' + addln_opts += dev_list + +if bool(log_mods and log_mods.strip()): + addln_opts += ' --log_mods ' + addln_opts += log_mods + +addln_opts += ' '.join(map(str, args)) + +print("addln_opts: " + addln_opts) + + +def meta_svc_nightly(): + print("meta blk store test started") + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.CompressionBackoff" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.RecoveryFromBadData" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.min_drive_size_test" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.single_read_test" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--run_time=7200 --num_io=1000000" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--min_write_size=65536 --max_write_size=2097152 --run_time=14400 --num_io=1000000" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--min_write_size=10485760 --max_write_size=104857600 --bitmap=1" + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + + cmd_opts = "--gtest_filter=VMetaBlkMgrTest.write_to_full_test" # write to file instead of real disk to save time; + subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True) + print("meta blk store test completed") + +def logstore_nightly(): + print("log store test started") + + cmd_opts = "--iterations=10" + subprocess.check_call(dirpath + "test_log_store " + cmd_opts + http_port, stderr=subprocess.STDOUT, shell=True) + + print("log store test completed") + +def nightly(): + logstore_nightly() + sleep(5) + + meta_svce_nightly() + sleep(5) + +# The name of the method to be called is the var test_suits +eval(f"{test_suits}()") +