From 890c7c1e66ecc6eea218e8b9b1c3d8733428aea4 Mon Sep 17 00:00:00 2001 From: Anton Myagkov Date: Thu, 17 Oct 2024 09:46:09 +0200 Subject: [PATCH] NBSDUTY-123: log blockdev output & fix cleanup for csi tests (#2297) 1. log output of "blockdev --getsize64" 2. move unpublish/unstage/delete volume commands to cleanup function 3. add 1 second pause before cleanup to distinguish dmesg log before and after test failure --- cloud/blockstore/tests/csi_driver/test.py | 76 ++++++------------- .../csi_driver/internal/mounter/mounter.go | 3 +- 2 files changed, 27 insertions(+), 52 deletions(-) diff --git a/cloud/blockstore/tests/csi_driver/test.py b/cloud/blockstore/tests/csi_driver/test.py index 9581d80f0c..85a51f807a 100644 --- a/cloud/blockstore/tests/csi_driver/test.py +++ b/cloud/blockstore/tests/csi_driver/test.py @@ -256,12 +256,6 @@ def expand_volume(self, pod_id: str, volume_id: str, size: int): ) -def cleanup_after_test(env: CsiLoadTest): - if env is None: - return - env.tear_down() - - def log_called_process_error(exc): logging.error( "Failed %s, stdout: %s, stderr: %s", @@ -272,6 +266,25 @@ def log_called_process_error(exc): ) +def cleanup_after_test(env: CsiLoadTest, volume_name: str = "", pods: list[str] = []): + if env is None: + return + + # sleep 1 second to distingish dmesg logs before and after test failure + time.sleep(1) + + for pod_id in pods: + with called_process_error_logged(): + env.csi.unpublish_volume(pod_id, volume_name) + + with called_process_error_logged(): + env.csi.unstage_volume(volume_name) + with called_process_error_logged(): + env.csi.delete_volume(volume_name) + + env.tear_down() + + @contextlib.contextmanager def called_process_error_logged(): try: @@ -302,17 +315,11 @@ def test_nbs_csi_driver_mounted_disk_protected_from_deletion(): if result.returncode != 1: raise AssertionError("Destroyvolume must return exit code 1") assert "E_REJECTED" in result.stdout - with called_process_error_logged(): - env.csi.unpublish_volume(pod_id, volume_name) - with called_process_error_logged(): - env.csi.unstage_volume(volume_name) - with called_process_error_logged(): - env.csi.delete_volume(volume_name) except subprocess.CalledProcessError as e: log_called_process_error(e) raise finally: - cleanup_after_test(env) + cleanup_after_test(env, volume_name, [pod_id]) def test_nbs_csi_driver_volume_stat(): @@ -366,18 +373,11 @@ def test_nbs_csi_driver_volume_stat(): nodesUsage2 = usage_array2[1] assert 2 == nodesUsage1["available"] - nodesUsage2["available"] assert 2 == nodesUsage2["used"] - nodesUsage1["used"] - - with called_process_error_logged(): - env.csi.unpublish_volume(pod_id, volume_name) - with called_process_error_logged(): - env.csi.unstage_volume(volume_name) - with called_process_error_logged(): - env.csi.delete_volume(volume_name) except subprocess.CalledProcessError as e: log_called_process_error(e) raise finally: - cleanup_after_test(env) + cleanup_after_test(env, volume_name, [pod_id]) @pytest.mark.parametrize('mount_path,volume_access_type,vm_mode', @@ -456,13 +456,7 @@ def test_node_volume_expand(fs_type): log_called_process_error(e) raise finally: - with called_process_error_logged(): - env.csi.unpublish_volume(pod_id, volume_name) - with called_process_error_logged(): - env.csi.unstage_volume(volume_name) - with called_process_error_logged(): - env.csi.delete_volume(volume_name) - cleanup_after_test(env) + cleanup_after_test(env, volume_name, [pod_id]) @pytest.mark.parametrize('vm_mode', [True, False]) @@ -483,15 +477,7 @@ def test_publish_volume_twice_on_the_same_node(vm_mode): log_called_process_error(e) raise finally: - with called_process_error_logged(): - env.csi.unpublish_volume(pod_id1, volume_name) - with called_process_error_logged(): - env.csi.unpublish_volume(pod_id2, volume_name) - with called_process_error_logged(): - env.csi.unstage_volume(volume_name) - with called_process_error_logged(): - env.csi.delete_volume(volume_name) - cleanup_after_test(env) + cleanup_after_test(env, volume_name, [pod_id1, pod_id2]) def test_restart_kubelet_with_old_format_endpoint(): @@ -511,13 +497,7 @@ def test_restart_kubelet_with_old_format_endpoint(): log_called_process_error(e) raise finally: - with called_process_error_logged(): - env.csi.unpublish_volume(pod_id1, volume_name) - with called_process_error_logged(): - env.csi.unstage_volume(volume_name) - with called_process_error_logged(): - env.csi.delete_volume(volume_name) - cleanup_after_test(env) + cleanup_after_test(env, volume_name, [pod_id1]) def test_restart_kubelet_with_new_format_endpoint(): @@ -537,10 +517,4 @@ def test_restart_kubelet_with_new_format_endpoint(): log_called_process_error(e) raise finally: - with called_process_error_logged(): - env.csi.unpublish_volume(pod_id1, volume_name) - with called_process_error_logged(): - env.csi.unstage_volume(volume_name) - with called_process_error_logged(): - env.csi.delete_volume(volume_name) - cleanup_after_test(env) + cleanup_after_test(env, volume_name, [pod_id1]) diff --git a/cloud/blockstore/tools/csi_driver/internal/mounter/mounter.go b/cloud/blockstore/tools/csi_driver/internal/mounter/mounter.go index af54da1060..4bd96ee883 100644 --- a/cloud/blockstore/tools/csi_driver/internal/mounter/mounter.go +++ b/cloud/blockstore/tools/csi_driver/internal/mounter/mounter.go @@ -61,7 +61,8 @@ func (m *mounter) IsFilesystemExisted(device string) (bool, error) { } if deviceSize == 0 { - return false, fmt.Errorf("size of device %q is empty", device) + return false, fmt.Errorf( + "size of device %q is empty. blockdev output: %q", device, out) } out, err = exec.Command("blkid", device).CombinedOutput()