Skip to content

Commit

Permalink
issue: 188 long duration test for meta svc and logstore svc (#256)
Browse files Browse the repository at this point in the history
* issue: 188 long duration test for meta svc and logstore svc
  • Loading branch information
yamingk authored Jan 17, 2024
1 parent 92b9dfe commit 01ccbb9
Show file tree
Hide file tree
Showing 6 changed files with 124 additions and 7 deletions.
5 changes: 4 additions & 1 deletion .jenkins/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ RUN set -eux; \
rm -rf /var/lib/apt/lists/*;

COPY test_index_btree /usr/local/bin/test_index_btree
COPY test_meta_blk_mgr /usr/local/bin/test_meta_blk_mgr
COPY test_log_store /usr/local/bin/test_log_store
COPY btree_test.py /usr/local/bin/scripts/btree_test.py
COPY log_meta_test.py /usr/local/bin/scripts/log_meta_test.py

EXPOSE 5000
# ########## ####### ############
# ########## ####### ############
3 changes: 3 additions & 0 deletions .jenkins/jenkinsfile_nightly
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ pipeline {
steps {
sh "conan create --build missing -o sisl:prerelease=True -o homestore:sanitize=True -o homestore:skip_testing=True -pr debug . ${PROJECT}/${VER}@"
sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;"
sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr\\;"
sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_log_store' -exec cp {} .jenkins/test_log_store\\;"
sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/btree_test.py' -exec install -Dm755 {} .jenkins/btree_test.py \\; "
sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/log_meta_test.py' -exec install -Dm755 {} .jenkins/log_meta_test.py \\; "
}
post {
failure {
Expand Down
21 changes: 17 additions & 4 deletions src/lib/device/virtual_dev.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,10 @@ BlkAllocStatus VirtualDev::commit_blk(BlkId const& blkid) {
HS_LOG(DEBUG, device, "commit_blk: bid {}", blkid.to_string());
auto const recovering = homestore::hs()->is_initializing();
if (!recovering) {
HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode", blkid.to_string());
HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode",
blkid.to_string());
} else {
chunk->blk_allocator_mutable()->mark_blk_allocated(blkid);
chunk->blk_allocator_mutable()->mark_blk_allocated(blkid);
}
return chunk->blk_allocator_mutable()->alloc_on_disk(blkid);
}
Expand All @@ -169,8 +170,13 @@ BlkAllocStatus VirtualDev::alloc_contiguous_blks(blk_count_t nblks, blk_alloc_hi
} else {
ret = alloc_blks(nblks, hints, mbid);
}
HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!");
out_blkid = mbid.to_single_blkid();

if (ret == BlkAllocStatus::SUCCESS || (ret == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) {
HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!");
out_blkid = mbid.to_single_blkid();
}

// for failure case, fall through and return the status to caller;
} catch (const std::exception& e) {
ret = BlkAllocStatus::FAILED;
HS_DBG_ASSERT(0, "{}", e.what());
Expand Down Expand Up @@ -235,6 +241,13 @@ BlkAllocStatus VirtualDev::alloc_blks(blk_count_t nblks, blk_alloc_hints const&

auto nblks_this_iter = out_bid.blk_count();
nblks_remain = (nblks_remain < nblks_this_iter) ? 0 : (nblks_remain - nblks_this_iter);

if (status != BlkAllocStatus::SUCCESS && status != BlkAllocStatus::PARTIAL) {
out_blkids.pop_back();
// all chunks has been tried, but still failed to allocate;
// break out and return status to caller;
break;
}
} while (nblks_remain);

return status;
Expand Down
4 changes: 2 additions & 2 deletions src/tests/test_meta_blk_mgr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ class VMetaBlkMgrTest : public ::testing::Test {

LOGINFO("compression ratio limit changed to: {}", HS_DYNAMIC_CONFIG(metablk.compress_ratio_limit));

do_sb_write(true /* do_overflow */, 15 * Mi);
do_sb_write(true /* do_overflow */, 5 * Mi);

HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) {
s.metablk.compress_ratio_limit = 0; // this will disallow every compression attempt;
Expand All @@ -423,7 +423,7 @@ class VMetaBlkMgrTest : public ::testing::Test {
LOGINFO("compression ratio limit changed to: {}", HS_DYNAMIC_CONFIG(metablk.compress_ratio_limit));

// since we only wrote 1 metablk, it will always pick up the same one;
do_sb_update(true /* aligned */, 12 * Mi);
do_sb_update(true /* aligned */, 5 * Mi);
}

void do_rand_load() {
Expand Down
1 change: 1 addition & 0 deletions src/tests/test_scripts/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ file(COPY vol_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
file(COPY home_blk_flip.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
file(COPY home_blk_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
file(COPY btree_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
file(COPY log_meta_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
#add_test(NAME TestVolRecovery COMMAND ${CMAKE_BINARY_DIR}/bin/scripts/vol_test.py --test_suits=recovery --dirpath=${CMAKE_BINARY_DIR}/bin/)
#SET_TESTS_PROPERTIES(TestVolRecovery PROPERTIES DEPENDS TestVol)

Expand Down
97 changes: 97 additions & 0 deletions src/tests/test_scripts/log_meta_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
## @file vol_test.py

import subprocess
import os
import sys
import getopt
import sys
from multiprocessing import Process
sys.stdout.flush()
from time import sleep
import requests
from threading import Thread

opts,args = getopt.getopt(sys.argv[1:], 'tdlme:', ['test_suits=', 'dirpath=', 'dev_list=', 'log_mods=', 'emulate_hdd=', 'http_port='] )
test_suits = ""
dirpath = "./"
dev_list = ""
log_mods = ""
http_port = ""

for opt,arg in opts:
if opt in ('-t', '--test_suits'):
test_suits = arg
print(("testing suits (%s)") % (arg))
if opt in ('-d', '--dirpath'):
dirpath = arg
print(("dir path (%s)") % (arg))
if opt in ('-l', '--dev_list'):
dev_list = arg
print(("device list (%s)") % (arg))
if opt in ('-m', '--log_mods'):
log_mods = arg
print(("log_mods (%s)") % (arg))
if opt in ('-p', '--http_port'):
http_port = " --http_port " + arg
print(("http_port (%s)") % (arg))

addln_opts = ' '
if bool(dev_list and dev_list.strip()):
addln_opts += ' --device_list '
addln_opts += dev_list

if bool(log_mods and log_mods.strip()):
addln_opts += ' --log_mods '
addln_opts += log_mods

addln_opts += ' '.join(map(str, args))

print("addln_opts: " + addln_opts)


def meta_svc_nightly():
print("meta blk store test started")
cmd_opts = "--gtest_filter=VMetaBlkMgrTest.CompressionBackoff"
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)

cmd_opts = "--gtest_filter=VMetaBlkMgrTest.RecoveryFromBadData"
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)

cmd_opts = "--gtest_filter=VMetaBlkMgrTest.min_drive_size_test"
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)

cmd_opts = "--gtest_filter=VMetaBlkMgrTest.single_read_test"
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)

cmd_opts = "--run_time=7200 --num_io=1000000"
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)

cmd_opts = "--min_write_size=65536 --max_write_size=2097152 --run_time=14400 --num_io=1000000"
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)

cmd_opts = "--min_write_size=10485760 --max_write_size=104857600 --bitmap=1"
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)

cmd_opts = "--gtest_filter=VMetaBlkMgrTest.write_to_full_test" # write to file instead of real disk to save time;
subprocess.check_call(dirpath + "test_meta_blk_mgr " + cmd_opts + http_port + addln_opts, stderr=subprocess.STDOUT, shell=True)
print("meta blk store test completed")

def logstore_nightly():
print("log store test started")

cmd_opts = "--iterations=10"
subprocess.check_call(dirpath + "test_log_store " + cmd_opts + http_port, stderr=subprocess.STDOUT, shell=True)

print("log store test completed")

def nightly():
logstore_nightly()
sleep(5)

meta_svce_nightly()
sleep(5)

# The name of the method to be called is the var test_suits
eval(f"{test_suits}()")

0 comments on commit 01ccbb9

Please sign in to comment.