[Filestore] support reuse-fs flag in build-arcadia-test + add multish…

…ard coreutils test for makeshift internal ci (#1584) * support reuse-fs flag in build-arcadia-test + add multishard coreutils test for makeshift internal ci
ydb-platform · Jul 11, 2024 · d91bcd7 · d91bcd7
1 parent 363e6ef
commit d91bcd7
Show file tree

Hide file tree

Showing 8 changed files with 90 additions and 31 deletions.
diff --git a/cloud/filestore/tests/build_arcadia_test/common.py b/cloud/filestore/tests/build_arcadia_test/common.py
@@ -199,3 +199,18 @@ def create_fs(
         else:
             logger.info(f'Error: {e}')
             raise ResourceExhaustedError('Cannot create file system')
+
+
+def find_fs(
+    ycp: YcpWrapper,
+    fs_id: str,
+    logger
+) -> Ycp.Filesystem:
+    try:
+        fss = ycp.list_filesystems()
+        for fs in fss:
+            if fs.id == fs_id:
+                return fs
+    except Exception as e:
+        logger.info(f'Error occurs while finding fs ({fs_id}) {e}')
+        raise Error('Cannot find fs')
diff --git a/cloud/filestore/tests/build_arcadia_test/coreutils.py b/cloud/filestore/tests/build_arcadia_test/coreutils.py
@@ -8,11 +8,11 @@
 from cloud.blockstore.pylibs.ycp import YcpWrapper
 from cloud.filestore.tests.build_arcadia_test.common import (
     DEVICE_NAME, LOG_LOCAL_PATH, LOG_PATH, MOUNT_PATH, TEST_FS_SIZE, Error,
-    create_fs, fetch_file_from_vm, mount_fs, run)
+    create_fs, fetch_file_from_vm, find_fs, mount_fs, run)
 
 _COREUTILS_SCRIPT_NAME = 'coreutils.sh'
 _COREUTILS_SCRIPT_PATH = f'/{MOUNT_PATH}/{_COREUTILS_SCRIPT_NAME}'
-_COREUTILS_OUTPUT_VM_PATH = f'/{MOUNT_PATH}/coreutils_log.txt'
+_COREUTILS_OUTPUT_VM_PATH = '~/coreutils_log.txt'
 _COREUTILS_OUTPUT_LOCAL_PATH = 'coreutils_log.txt'
 _COREUTILS_JUNIT_OUTPUT_LOCAL_PATH = 'junit_report.xml'
 
@@ -202,15 +202,26 @@ def run_coreutils_test(
 
 def execute_coreutils_test(ycp, parser, instance, args, logger, module_factories):
     logger.info('Starting test with filestore')
-    with create_fs(ycp, TEST_FS_SIZE, 'network-ssd', logger) as fs:
-        with ycp.attach_fs(instance, fs, DEVICE_NAME):
-            mount_fs(instance.ip, args.dry_run, logger,
-                     module_factories, ssh_key_path=args.ssh_key_path)
-            run_coreutils_test(
-                ycp,
-                instance.ip,
-                args.dry_run,
-                args.debug,
-                module_factories,
-                args.ssh_key_path,
-                logger)
+    if args.reuse_fs_id is None:
+        with create_fs(ycp, TEST_FS_SIZE, 'network-ssd', logger) as fs:
+            _execute_coreutils_test(
+                ycp, parser, instance, fs, args, logger, module_factories
+            )
+    else:
+        fs = find_fs(ycp, args.reuse_fs_id, logger)
+        _execute_coreutils_test(
+            ycp, parser, instance, fs, args, logger, module_factories
+        )
+
+
+def _execute_coreutils_test(ycp, parser, instance, fs, args, logger, module_factories):
+    with ycp.attach_fs(instance, fs, DEVICE_NAME):
+        mount_fs(instance.ip, args.dry_run, logger, module_factories, ssh_key_path=args.ssh_key_path)
+        run_coreutils_test(
+            ycp,
+            instance.ip,
+            args.dry_run,
+            args.debug,
+            module_factories,
+            args.ssh_key_path,
+            logger)
diff --git a/cloud/filestore/tests/build_arcadia_test/entrypoint.py b/cloud/filestore/tests/build_arcadia_test/entrypoint.py
@@ -57,6 +57,12 @@ def _parse_args():
         '--debug',
         action='store_true',
         help='dont delete instance if test fail')
+    test_arguments_group.add_argument(
+        '--reuse-fs-id',
+        type=str,
+        required=False,
+        default=None,
+        help='An existing FS id to reuse. If not set, a new one is created')
     common.add_common_parser_arguments(test_arguments_group)
 
     return parser

diff --git a/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh b/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh
@@ -4,11 +4,15 @@ sudo apt update
 sudo apt -y install python3 python-is-python3 git build-essential autoconf automake autopoint bison gperf libtool texinfo wget
 cd $mountPath
 
-git clone --depth=1 --branch v9.3 --single-branch https://git.savannah.gnu.org/git/coreutils.git
-cd coreutils
-./bootstrap
+if [ ! -d "coreutils" ]; then
+    git clone --depth=1 --branch v9.3 --single-branch https://git.savannah.gnu.org/git/coreutils.git
+    cd coreutils
+    ./bootstrap
+    # Allow root user too run ./configure
+    export FORCE_UNSAFE_CONFIGURE=1
+    ./configure
+else
+    cd coreutils
+fi
 
-# Allow root user too run ./configure
-export FORCE_UNSAFE_CONFIGURE=1
-./configure
 make check -j$$(nproc) | tee $coreutilsOutputPath
diff --git a/cloud/storage/core/tools/ci/runner/generate_index.py b/cloud/storage/core/tools/ci/runner/generate_index.py
@@ -108,7 +108,7 @@ def generate_nbs_section(index, results_xml_path):
 
 
 def generate_nfs_section(index, results_xml_path):
-    for tests_type in ["nfs_fio", "nfs_corruption", "xfs", "coreutils"]:
+    for tests_type in ["nfs_fio", "nfs_corruption", "xfs", "coreutils", "coreutils-multishard"]:
         generate_generic_tests_section(index, tests_type, results_xml_path)
 
 

diff --git a/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py b/cloud/storage/core/tools/ci/runner/generate_report_coreutils.py
@@ -4,14 +4,14 @@
 import sys
 
 
-def transform_xml(results_filename, suite_name, suite_date, muted):
+def transform_xml(results_filename, suite_name, suite_date, test_name, muted):
     root = ET.fromstring(open(results_filename, "r").read())
 
     suite_attributes = {
         'name': f'{suite_name}/{suite_date}',
         'type': suite_name,
         'date': suite_date,
-        'link': f'/ci/results/coreutils/{suite_name}/{suite_date}/junit_report.xml',
+        'link': f'/ci/results/{test_name}/{suite_name}/{suite_date}/junit_report.xml',
     }
     suite_element = ET.Element('suite', attrib=suite_attributes)
 
@@ -48,7 +48,8 @@ def append_to_report(file_path, my_xml):
     suite_date = sys.argv[3]
 
     index_file = sys.argv[4]
+    test_name = sys.argv[5]
 
-    new_xml = transform_xml(original_xml, suite_name, suite_date, sys.argv[5:])
+    new_xml = transform_xml(original_xml, suite_name, suite_date, test_name, sys.argv[6:])
 
     append_to_report(index_file, new_xml)
diff --git a/cloud/storage/core/tools/ci/runner/index.xsl b/cloud/storage/core/tools/ci/runner/index.xsl
@@ -204,6 +204,13 @@
     </xsl:call-template>
 </xsl:template>
 
+<xsl:template match="coreutils-multishard" mode="detailed">
+    <xsl:call-template name="detailed">
+        <xsl:with-param name="suite-kind">coreutils-multishard</xsl:with-param>
+        <xsl:with-param name="show-all-testsuites" select="false()" />
+    </xsl:call-template>
+</xsl:template>
+
 <xsl:template match="degradation_tests" mode="detailed">
     <xsl:call-template name="detailed">
         <xsl:with-param name="suite-kind">degradation_tests</xsl:with-param>
@@ -266,6 +273,12 @@
     </xsl:call-template>
 </xsl:template>
 
+<xsl:template match="coreutils-multishard" mode="brief">
+    <xsl:call-template name="brief">
+        <xsl:with-param name="suite-kind">coreutils-multishard</xsl:with-param>
+    </xsl:call-template>
+</xsl:template>
+
 <xsl:template match="degradation_tests" mode="brief">
     <xsl:call-template name="brief">
         <xsl:with-param name="suite-kind">degradation_tests</xsl:with-param>
@@ -303,7 +316,7 @@
                              alt="Nightly build">
                         </img>
                     </a>
-                </p>
+	        </p>
                 <p>
                     <a href="https://github.com/ydb-platform/nbs/actions/workflows/nightly-asan.yaml">
                         <img src="https://github.com/ydb-platform/nbs/actions/workflows/nightly-asan.yaml/badge.svg?branch=main"
@@ -334,6 +347,7 @@
                     <xsl:apply-templates select="nfs_corruption" mode="brief"/>
                     <xsl:apply-templates select="xfs" mode="brief"/>
                     <xsl:apply-templates select="coreutils" mode="brief"/>
+                    <xsl:apply-templates select="coreutils-multishard" mode="brief"/>
                 </div>
                 <div style="border: 1px solid black; margin-bottom: 20px; padding: 10px">
                     <h3>DM</h3>
@@ -365,6 +379,7 @@
                     <xsl:apply-templates select="nfs_corruption" mode="detailed"/>
                     <xsl:apply-templates select="xfs" mode="detailed"/>
                     <xsl:apply-templates select="coreutils" mode="detailed"/>
+                    <xsl:apply-templates select="coreutils-multishard" mode="detailed"/>
                 </div>
                 <div style="border: 1px solid black; margin-bottom: 20px; padding: 10px">
                     <h3>DM</h3>

diff --git a/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh b/cloud/storage/core/tools/ci/runner/run_nfs_coreutils.sh
@@ -1,21 +1,27 @@
 #!/usr/bin/env bash
 
+set -ex
+
 d="/root"
 scripts="${d}/runner"
 suffix="$(date +%y-%m-%d)"
-testname="coreutils"
+cluster="nemax"
+
+fs_id=$(jq -r '.fs_id' "$scripts/coreutils_config.json")
 
 function run_test () {
-    local cluster="$1"
+    local testname="$1"; shift
+    local args=("$@")
 
     results_dir="/var/www/build/results/${testname}/${cluster}/nfs/${suffix}"
     mkdir -p "$results_dir"
 
-    $d/yc-nfs-ci-build-arcadia-test --cluster "${cluster}" --zone-id eu-north1-b --test-case nfs-coreutils --platform-ids standard-v2 \
+    $d/yc-nfs-ci-build-arcadia-test --cluster "${cluster}" --zone-id eu-north1-a --test-case nfs-coreutils --platform-ids standard-v2 \
         --profile-name "${cluster}-tests" --debug --verbose --cluster-config-path $d/fio_dep/cluster-configs/ --ssh-key-path /root/.ssh/test-ssh-key \
-        --no-generate-ycp-config --ycp-requests-template-path $d/fio_dep/ycp-request-templates 2>> "${results_dir}/${testname}".err >> "${results_dir}/${testname}".out
+        --no-generate-ycp-config --ycp-requests-template-path $d/fio_dep/ycp-request-templates "${args[@]}"  \
+        2>> "${results_dir}/${testname}".err >> "${results_dir}/${testname}".out
 
-    $scripts/generate_report_coreutils.py junit_report.xml "${cluster}/nfs" "${suffix}" "/var/www/build/results/${testname}.xml" \
+    $scripts/generate_report_coreutils.py junit_report.xml "${cluster}/nfs" "${suffix}" "/var/www/build/results/${testname}.xml" "${testname}" \
         tests/cp/link-symlink.sh \
         tests/ls/stat-free-color.sh \
         tests/mv/mv-special-1.sh \
@@ -27,4 +33,5 @@ function run_test () {
     mv junit_report.xml "$results_dir/"
 }
 
-run_test nemax
+run_test coreutils-multishard --reuse-fs-id "$fs_id"
+run_test coreutils