From d91bcd7b996eb3123680eaeb629e7742aaee8956 Mon Sep 17 00:00:00 2001
From: Maxim Deb Natkh
Date: Thu, 11 Jul 2024 20:14:45 +0200
Subject: [PATCH] [Filestore] support reuse-fs flag in build-arcadia-test + add
multishard coreutils test for makeshift internal ci (#1584)
* support reuse-fs flag in build-arcadia-test + add multishard coreutils test for makeshift internal ci
---
.../tests/build_arcadia_test/common.py | 15 +++++++
.../tests/build_arcadia_test/coreutils.py | 39 ++++++++++++-------
.../tests/build_arcadia_test/entrypoint.py | 6 +++
.../build_arcadia_test/scripts/coreutils.sh | 16 +++++---
.../core/tools/ci/runner/generate_index.py | 2 +-
.../ci/runner/generate_report_coreutils.py | 7 ++--
cloud/storage/core/tools/ci/runner/index.xsl | 17 +++++++-
.../core/tools/ci/runner/run_nfs_coreutils.sh | 19 ++++++---
8 files changed, 90 insertions(+), 31 deletions(-)
diff --git a/cloud/filestore/tests/build_arcadia_test/common.py b/cloud/filestore/tests/build_arcadia_test/common.py
index 68072c270e..032efc5c92 100644
--- a/cloud/filestore/tests/build_arcadia_test/common.py
+++ b/cloud/filestore/tests/build_arcadia_test/common.py
@@ -199,3 +199,18 @@ def create_fs(
else:
logger.info(f'Error: {e}')
raise ResourceExhaustedError('Cannot create file system')
+
+
+def find_fs(
+ ycp: YcpWrapper,
+ fs_id: str,
+ logger
+) -> Ycp.Filesystem:
+ try:
+ fss = ycp.list_filesystems()
+ for fs in fss:
+ if fs.id == fs_id:
+ return fs
+ except Exception as e:
+ logger.info(f'Error occurs while finding fs ({fs_id}) {e}')
+ raise Error('Cannot find fs')
diff --git a/cloud/filestore/tests/build_arcadia_test/coreutils.py b/cloud/filestore/tests/build_arcadia_test/coreutils.py
index 91b2a7b440..e558e5d47b 100644
--- a/cloud/filestore/tests/build_arcadia_test/coreutils.py
+++ b/cloud/filestore/tests/build_arcadia_test/coreutils.py
@@ -8,11 +8,11 @@
from cloud.blockstore.pylibs.ycp import YcpWrapper
from cloud.filestore.tests.build_arcadia_test.common import (
DEVICE_NAME, LOG_LOCAL_PATH, LOG_PATH, MOUNT_PATH, TEST_FS_SIZE, Error,
- create_fs, fetch_file_from_vm, mount_fs, run)
+ create_fs, fetch_file_from_vm, find_fs, mount_fs, run)
_COREUTILS_SCRIPT_NAME = 'coreutils.sh'
_COREUTILS_SCRIPT_PATH = f'/{MOUNT_PATH}/{_COREUTILS_SCRIPT_NAME}'
-_COREUTILS_OUTPUT_VM_PATH = f'/{MOUNT_PATH}/coreutils_log.txt'
+_COREUTILS_OUTPUT_VM_PATH = '~/coreutils_log.txt'
_COREUTILS_OUTPUT_LOCAL_PATH = 'coreutils_log.txt'
_COREUTILS_JUNIT_OUTPUT_LOCAL_PATH = 'junit_report.xml'
@@ -202,15 +202,26 @@ def run_coreutils_test(
def execute_coreutils_test(ycp, parser, instance, args, logger, module_factories):
logger.info('Starting test with filestore')
- with create_fs(ycp, TEST_FS_SIZE, 'network-ssd', logger) as fs:
- with ycp.attach_fs(instance, fs, DEVICE_NAME):
- mount_fs(instance.ip, args.dry_run, logger,
- module_factories, ssh_key_path=args.ssh_key_path)
- run_coreutils_test(
- ycp,
- instance.ip,
- args.dry_run,
- args.debug,
- module_factories,
- args.ssh_key_path,
- logger)
+ if args.reuse_fs_id is None:
+ with create_fs(ycp, TEST_FS_SIZE, 'network-ssd', logger) as fs:
+ _execute_coreutils_test(
+ ycp, parser, instance, fs, args, logger, module_factories
+ )
+ else:
+ fs = find_fs(ycp, args.reuse_fs_id, logger)
+ _execute_coreutils_test(
+ ycp, parser, instance, fs, args, logger, module_factories
+ )
+
+
+def _execute_coreutils_test(ycp, parser, instance, fs, args, logger, module_factories):
+ with ycp.attach_fs(instance, fs, DEVICE_NAME):
+ mount_fs(instance.ip, args.dry_run, logger, module_factories, ssh_key_path=args.ssh_key_path)
+ run_coreutils_test(
+ ycp,
+ instance.ip,
+ args.dry_run,
+ args.debug,
+ module_factories,
+ args.ssh_key_path,
+ logger)
diff --git a/cloud/filestore/tests/build_arcadia_test/entrypoint.py b/cloud/filestore/tests/build_arcadia_test/entrypoint.py
index 50d3e51910..4a40f7b90f 100644
--- a/cloud/filestore/tests/build_arcadia_test/entrypoint.py
+++ b/cloud/filestore/tests/build_arcadia_test/entrypoint.py
@@ -57,6 +57,12 @@ def _parse_args():
'--debug',
action='store_true',
help='dont delete instance if test fail')
+ test_arguments_group.add_argument(
+ '--reuse-fs-id',
+ type=str,
+ required=False,
+ default=None,
+ help='An existing FS id to reuse. If not set, a new one is created')
common.add_common_parser_arguments(test_arguments_group)
return parser
diff --git a/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh b/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh
index 089e17304b..1576d80d15 100644
--- a/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh
+++ b/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh
@@ -4,11 +4,15 @@ sudo apt update
sudo apt -y install python3 python-is-python3 git build-essential autoconf automake autopoint bison gperf libtool texinfo wget
cd $mountPath
-git clone --depth=1 --branch v9.3 --single-branch https://git.savannah.gnu.org/git/coreutils.git
-cd coreutils
-./bootstrap
+if [ ! -d "coreutils" ]; then
+ git clone --depth=1 --branch v9.3 --single-branch https://git.savannah.gnu.org/git/coreutils.git
+ cd coreutils
+ ./bootstrap
+ # Allow root user too run ./configure
+ export FORCE_UNSAFE_CONFIGURE=1
+ ./configure
+else
+ cd coreutils
+fi
-# Allow root user too run ./configure
-export FORCE_UNSAFE_CONFIGURE=1
-./configure
make check -j$$(nproc) | tee $coreutilsOutputPath
diff --git a/cloud/storage/core/tools/ci/runner/generate_index.py b/cloud/storage/core/tools/ci/runner/generate_index.py
index 99005f24f6..3979830fe8 100755
--- a/cloud/storage/core/tools/ci/runner/generate_index.py
+++ b/cloud/storage/core/tools/ci/runner/generate_index.py
@@ -108,7 +108,7 @@ def generate_nbs_section(index, results_xml_path):
def generate_nfs_section(index, results_xml_path):
- for tests_type in ["nfs_fio", "nfs_corruption", "xfs", "coreutils"]:
+ for tests_type in ["nfs_fio", "nfs_corruption", "xfs", "coreutils", "coreutils-multishard"]:
generate_generic_tests_section(index, tests_type, results_xml_path)
diff --git a/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py b/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py
index c0ee4adf75..5df0515568 100755
--- a/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py
+++ b/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py
@@ -4,14 +4,14 @@
import sys
-def transform_xml(results_filename, suite_name, suite_date, muted):
+def transform_xml(results_filename, suite_name, suite_date, test_name, muted):
root = ET.fromstring(open(results_filename, "r").read())
suite_attributes = {
'name': f'{suite_name}/{suite_date}',
'type': suite_name,
'date': suite_date,
- 'link': f'/ci/results/coreutils/{suite_name}/{suite_date}/junit_report.xml',
+ 'link': f'/ci/results/{test_name}/{suite_name}/{suite_date}/junit_report.xml',
}
suite_element = ET.Element('suite', attrib=suite_attributes)
@@ -48,7 +48,8 @@ def append_to_report(file_path, my_xml):
suite_date = sys.argv[3]
index_file = sys.argv[4]
+ test_name = sys.argv[5]
- new_xml = transform_xml(original_xml, suite_name, suite_date, sys.argv[5:])
+ new_xml = transform_xml(original_xml, suite_name, suite_date, test_name, sys.argv[6:])
append_to_report(index_file, new_xml)
diff --git a/cloud/storage/core/tools/ci/runner/index.xsl b/cloud/storage/core/tools/ci/runner/index.xsl
index 69a2f4b71c..39a3e6ad8a 100644
--- a/cloud/storage/core/tools/ci/runner/index.xsl
+++ b/cloud/storage/core/tools/ci/runner/index.xsl
@@ -204,6 +204,13 @@
+
+
+ coreutils-multishard
+
+
+
+
degradation_tests
@@ -266,6 +273,12 @@
+
+
+ coreutils-multishard
+
+
+
degradation_tests
@@ -303,7 +316,7 @@
alt="Nightly build">
-
+
+
DM
@@ -365,6 +379,7 @@
+
DM
diff --git a/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh b/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh
index 410255875e..0877de36f6 100755
--- a/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh
+++ b/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh
@@ -1,21 +1,27 @@
#!/usr/bin/env bash
+set -ex
+
d="/root"
scripts="${d}/runner"
suffix="$(date +%y-%m-%d)"
-testname="coreutils"
+cluster="nemax"
+
+fs_id=$(jq -r '.fs_id' "$scripts/coreutils_config.json")
function run_test () {
- local cluster="$1"
+ local testname="$1"; shift
+ local args=("$@")
results_dir="/var/www/build/results/${testname}/${cluster}/nfs/${suffix}"
mkdir -p "$results_dir"
- $d/yc-nfs-ci-build-arcadia-test --cluster "${cluster}" --zone-id eu-north1-b --test-case nfs-coreutils --platform-ids standard-v2 \
+ $d/yc-nfs-ci-build-arcadia-test --cluster "${cluster}" --zone-id eu-north1-a --test-case nfs-coreutils --platform-ids standard-v2 \
--profile-name "${cluster}-tests" --debug --verbose --cluster-config-path $d/fio_dep/cluster-configs/ --ssh-key-path /root/.ssh/test-ssh-key \
- --no-generate-ycp-config --ycp-requests-template-path $d/fio_dep/ycp-request-templates 2>> "${results_dir}/${testname}".err >> "${results_dir}/${testname}".out
+ --no-generate-ycp-config --ycp-requests-template-path $d/fio_dep/ycp-request-templates "${args[@]}" \
+ 2>> "${results_dir}/${testname}".err >> "${results_dir}/${testname}".out
- $scripts/generate_report_coreutils.py junit_report.xml "${cluster}/nfs" "${suffix}" "/var/www/build/results/${testname}.xml" \
+ $scripts/generate_report_coreutils.py junit_report.xml "${cluster}/nfs" "${suffix}" "/var/www/build/results/${testname}.xml" "${testname}" \
tests/cp/link-symlink.sh \
tests/ls/stat-free-color.sh \
tests/mv/mv-special-1.sh \
@@ -27,4 +33,5 @@ function run_test () {
mv junit_report.xml "$results_dir/"
}
-run_test nemax
+run_test coreutils-multishard --reuse-fs-id "$fs_id"
+run_test coreutils