From d91bcd7b996eb3123680eaeb629e7742aaee8956 Mon Sep 17 00:00:00 2001 From: Maxim Deb Natkh Date: Thu, 11 Jul 2024 20:14:45 +0200 Subject: [PATCH] [Filestore] support reuse-fs flag in build-arcadia-test + add multishard coreutils test for makeshift internal ci (#1584) * support reuse-fs flag in build-arcadia-test + add multishard coreutils test for makeshift internal ci --- .../tests/build_arcadia_test/common.py | 15 +++++++ .../tests/build_arcadia_test/coreutils.py | 39 ++++++++++++------- .../tests/build_arcadia_test/entrypoint.py | 6 +++ .../build_arcadia_test/scripts/coreutils.sh | 16 +++++--- .../core/tools/ci/runner/generate_index.py | 2 +- .../ci/runner/generate_report_coreutils.py | 7 ++-- cloud/storage/core/tools/ci/runner/index.xsl | 17 +++++++- .../core/tools/ci/runner/run_nfs_coreutils.sh | 19 ++++++--- 8 files changed, 90 insertions(+), 31 deletions(-) diff --git a/cloud/filestore/tests/build_arcadia_test/common.py b/cloud/filestore/tests/build_arcadia_test/common.py index 68072c270e..032efc5c92 100644 --- a/cloud/filestore/tests/build_arcadia_test/common.py +++ b/cloud/filestore/tests/build_arcadia_test/common.py @@ -199,3 +199,18 @@ def create_fs( else: logger.info(f'Error: {e}') raise ResourceExhaustedError('Cannot create file system') + + +def find_fs( + ycp: YcpWrapper, + fs_id: str, + logger +) -> Ycp.Filesystem: + try: + fss = ycp.list_filesystems() + for fs in fss: + if fs.id == fs_id: + return fs + except Exception as e: + logger.info(f'Error occurs while finding fs ({fs_id}) {e}') + raise Error('Cannot find fs') diff --git a/cloud/filestore/tests/build_arcadia_test/coreutils.py b/cloud/filestore/tests/build_arcadia_test/coreutils.py index 91b2a7b440..e558e5d47b 100644 --- a/cloud/filestore/tests/build_arcadia_test/coreutils.py +++ b/cloud/filestore/tests/build_arcadia_test/coreutils.py @@ -8,11 +8,11 @@ from cloud.blockstore.pylibs.ycp import YcpWrapper from cloud.filestore.tests.build_arcadia_test.common import ( DEVICE_NAME, LOG_LOCAL_PATH, LOG_PATH, MOUNT_PATH, TEST_FS_SIZE, Error, - create_fs, fetch_file_from_vm, mount_fs, run) + create_fs, fetch_file_from_vm, find_fs, mount_fs, run) _COREUTILS_SCRIPT_NAME = 'coreutils.sh' _COREUTILS_SCRIPT_PATH = f'/{MOUNT_PATH}/{_COREUTILS_SCRIPT_NAME}' -_COREUTILS_OUTPUT_VM_PATH = f'/{MOUNT_PATH}/coreutils_log.txt' +_COREUTILS_OUTPUT_VM_PATH = '~/coreutils_log.txt' _COREUTILS_OUTPUT_LOCAL_PATH = 'coreutils_log.txt' _COREUTILS_JUNIT_OUTPUT_LOCAL_PATH = 'junit_report.xml' @@ -202,15 +202,26 @@ def run_coreutils_test( def execute_coreutils_test(ycp, parser, instance, args, logger, module_factories): logger.info('Starting test with filestore') - with create_fs(ycp, TEST_FS_SIZE, 'network-ssd', logger) as fs: - with ycp.attach_fs(instance, fs, DEVICE_NAME): - mount_fs(instance.ip, args.dry_run, logger, - module_factories, ssh_key_path=args.ssh_key_path) - run_coreutils_test( - ycp, - instance.ip, - args.dry_run, - args.debug, - module_factories, - args.ssh_key_path, - logger) + if args.reuse_fs_id is None: + with create_fs(ycp, TEST_FS_SIZE, 'network-ssd', logger) as fs: + _execute_coreutils_test( + ycp, parser, instance, fs, args, logger, module_factories + ) + else: + fs = find_fs(ycp, args.reuse_fs_id, logger) + _execute_coreutils_test( + ycp, parser, instance, fs, args, logger, module_factories + ) + + +def _execute_coreutils_test(ycp, parser, instance, fs, args, logger, module_factories): + with ycp.attach_fs(instance, fs, DEVICE_NAME): + mount_fs(instance.ip, args.dry_run, logger, module_factories, ssh_key_path=args.ssh_key_path) + run_coreutils_test( + ycp, + instance.ip, + args.dry_run, + args.debug, + module_factories, + args.ssh_key_path, + logger) diff --git a/cloud/filestore/tests/build_arcadia_test/entrypoint.py b/cloud/filestore/tests/build_arcadia_test/entrypoint.py index 50d3e51910..4a40f7b90f 100644 --- a/cloud/filestore/tests/build_arcadia_test/entrypoint.py +++ b/cloud/filestore/tests/build_arcadia_test/entrypoint.py @@ -57,6 +57,12 @@ def _parse_args(): '--debug', action='store_true', help='dont delete instance if test fail') + test_arguments_group.add_argument( + '--reuse-fs-id', + type=str, + required=False, + default=None, + help='An existing FS id to reuse. If not set, a new one is created') common.add_common_parser_arguments(test_arguments_group) return parser diff --git a/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh b/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh index 089e17304b..1576d80d15 100644 --- a/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh +++ b/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh @@ -4,11 +4,15 @@ sudo apt update sudo apt -y install python3 python-is-python3 git build-essential autoconf automake autopoint bison gperf libtool texinfo wget cd $mountPath -git clone --depth=1 --branch v9.3 --single-branch https://git.savannah.gnu.org/git/coreutils.git -cd coreutils -./bootstrap +if [ ! -d "coreutils" ]; then + git clone --depth=1 --branch v9.3 --single-branch https://git.savannah.gnu.org/git/coreutils.git + cd coreutils + ./bootstrap + # Allow root user too run ./configure + export FORCE_UNSAFE_CONFIGURE=1 + ./configure +else + cd coreutils +fi -# Allow root user too run ./configure -export FORCE_UNSAFE_CONFIGURE=1 -./configure make check -j$$(nproc) | tee $coreutilsOutputPath diff --git a/cloud/storage/core/tools/ci/runner/generate_index.py b/cloud/storage/core/tools/ci/runner/generate_index.py index 99005f24f6..3979830fe8 100755 --- a/cloud/storage/core/tools/ci/runner/generate_index.py +++ b/cloud/storage/core/tools/ci/runner/generate_index.py @@ -108,7 +108,7 @@ def generate_nbs_section(index, results_xml_path): def generate_nfs_section(index, results_xml_path): - for tests_type in ["nfs_fio", "nfs_corruption", "xfs", "coreutils"]: + for tests_type in ["nfs_fio", "nfs_corruption", "xfs", "coreutils", "coreutils-multishard"]: generate_generic_tests_section(index, tests_type, results_xml_path) diff --git a/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py b/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py index c0ee4adf75..5df0515568 100755 --- a/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py +++ b/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py @@ -4,14 +4,14 @@ import sys -def transform_xml(results_filename, suite_name, suite_date, muted): +def transform_xml(results_filename, suite_name, suite_date, test_name, muted): root = ET.fromstring(open(results_filename, "r").read()) suite_attributes = { 'name': f'{suite_name}/{suite_date}', 'type': suite_name, 'date': suite_date, - 'link': f'/ci/results/coreutils/{suite_name}/{suite_date}/junit_report.xml', + 'link': f'/ci/results/{test_name}/{suite_name}/{suite_date}/junit_report.xml', } suite_element = ET.Element('suite', attrib=suite_attributes) @@ -48,7 +48,8 @@ def append_to_report(file_path, my_xml): suite_date = sys.argv[3] index_file = sys.argv[4] + test_name = sys.argv[5] - new_xml = transform_xml(original_xml, suite_name, suite_date, sys.argv[5:]) + new_xml = transform_xml(original_xml, suite_name, suite_date, test_name, sys.argv[6:]) append_to_report(index_file, new_xml) diff --git a/cloud/storage/core/tools/ci/runner/index.xsl b/cloud/storage/core/tools/ci/runner/index.xsl index 69a2f4b71c..39a3e6ad8a 100644 --- a/cloud/storage/core/tools/ci/runner/index.xsl +++ b/cloud/storage/core/tools/ci/runner/index.xsl @@ -204,6 +204,13 @@ + + + coreutils-multishard + + + + degradation_tests @@ -266,6 +273,12 @@ + + + coreutils-multishard + + + degradation_tests @@ -303,7 +316,7 @@ alt="Nightly build"> -

+

+

DM

@@ -365,6 +379,7 @@ +

DM

diff --git a/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh b/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh index 410255875e..0877de36f6 100755 --- a/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh +++ b/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh @@ -1,21 +1,27 @@ #!/usr/bin/env bash +set -ex + d="/root" scripts="${d}/runner" suffix="$(date +%y-%m-%d)" -testname="coreutils" +cluster="nemax" + +fs_id=$(jq -r '.fs_id' "$scripts/coreutils_config.json") function run_test () { - local cluster="$1" + local testname="$1"; shift + local args=("$@") results_dir="/var/www/build/results/${testname}/${cluster}/nfs/${suffix}" mkdir -p "$results_dir" - $d/yc-nfs-ci-build-arcadia-test --cluster "${cluster}" --zone-id eu-north1-b --test-case nfs-coreutils --platform-ids standard-v2 \ + $d/yc-nfs-ci-build-arcadia-test --cluster "${cluster}" --zone-id eu-north1-a --test-case nfs-coreutils --platform-ids standard-v2 \ --profile-name "${cluster}-tests" --debug --verbose --cluster-config-path $d/fio_dep/cluster-configs/ --ssh-key-path /root/.ssh/test-ssh-key \ - --no-generate-ycp-config --ycp-requests-template-path $d/fio_dep/ycp-request-templates 2>> "${results_dir}/${testname}".err >> "${results_dir}/${testname}".out + --no-generate-ycp-config --ycp-requests-template-path $d/fio_dep/ycp-request-templates "${args[@]}" \ + 2>> "${results_dir}/${testname}".err >> "${results_dir}/${testname}".out - $scripts/generate_report_coreutils.py junit_report.xml "${cluster}/nfs" "${suffix}" "/var/www/build/results/${testname}.xml" \ + $scripts/generate_report_coreutils.py junit_report.xml "${cluster}/nfs" "${suffix}" "/var/www/build/results/${testname}.xml" "${testname}" \ tests/cp/link-symlink.sh \ tests/ls/stat-free-color.sh \ tests/mv/mv-special-1.sh \ @@ -27,4 +33,5 @@ function run_test () { mv junit_report.xml "$results_dir/" } -run_test nemax +run_test coreutils-multishard --reuse-fs-id "$fs_id" +run_test coreutils