From 99adff3b856bdb42b2b5fa1b575a812da624e7c0 Mon Sep 17 00:00:00 2001 From: xin liang Date: Tue, 16 Jul 2024 20:12:27 +0800 Subject: [PATCH 01/17] Dev: behave: Add sbd_ui.feature to test the crm sbd UI --- data-manifest | 1 + test/features/sbd_ui.feature | 90 ++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 test/features/sbd_ui.feature diff --git a/data-manifest b/data-manifest index 88aa8fee0..cd3a92d3e 100644 --- a/data-manifest +++ b/data-manifest @@ -89,6 +89,7 @@ test/features/qdevice_usercase.feature test/features/qdevice_validate.feature test/features/resource_failcount.feature test/features/resource_set.feature +test/features/sbd_ui.feature test/features/ssh_agent.feature test/features/steps/behave_agent.py test/features/steps/const.py diff --git a/test/features/sbd_ui.feature b/test/features/sbd_ui.feature new file mode 100644 index 000000000..898be019c --- /dev/null +++ b/test/features/sbd_ui.feature @@ -0,0 +1,90 @@ +@sbd +Feature: crm sbd ui test cases + + Tag @clean means need to stop cluster service if the service is available + + @clean + Scenario: Syntax check for crm sbd + Given Cluster service is "stopped" on "hanode1" + Given Cluster service is "stopped" on "hanode2" + Given Has disk "/dev/sda5" on "hanode1" + Given Has disk "/dev/sda6" on "hanode1" + Given Has disk "/dev/sda7" on "hanode1" + Given Has disk "/dev/sda8" on "hanode1" + Given Has disk "/dev/sda5" on "hanode2" + Given Has disk "/dev/sda6" on "hanode2" + Given Has disk "/dev/sda7" on "hanode2" + Given Has disk "/dev/sda8" on "hanode2" + When Try "crm sbd configure /dev/sda5" + Then Except "ERROR: pacemaker.service is not active" + When Run "crm cluster init -y" on "hanode1" + And Run "crm cluster join -c hanode1 -y" on "hanode2" + And Run ": > /etc/sysconfig/sbd" on "hanode1" + And Try "crm sbd configure watchdog-timeout=30" + Then Except "ERROR: No device specified" + When Run "crm cluster init sbd -s /dev/sda5 -y" on "hanode1" + Then Cluster service is "started" on "hanode1" + Then Cluster service is "started" on "hanode2" + And Service "sbd" is "started" on "hanode1" + And Resource "stonith-sbd" type "fence_sbd" is "Started" + + When Try "crm sbd configure show sysconfig xxx" + Then Except "ERROR: Invalid argument" + When Try "crm sbd configure show testing" + Then Except "ERROR: Unknown argument: testing" + When Try "crm sbd configure" + Then Except "ERROR: No argument" + When Try "crm sbd configure testing" + Then Except "ERROR: Invalid argument: testing" + When Try "crm sbd configure watchdog-timeout=f" + Then Except "ERROR: Invalid timeout value: f" + When Try "crm sbd configure name=testing" + Then Except "ERROR: Unknown argument: name=testing" + When Try "crm sbd configure device=/dev/sda5 device=/dev/sda5" + Then Except "ERROR: Duplicate device" + When Try "crm sbd configure device=/dev/sda6 device=/dev/sda7 device=/dev/sda8" + Then Except "ERROR: sbd.configure: Maximum number of SBD device is 3" + + Scenario: sbd configure for diskbased sbd + # Update disk metadata + When Run "crm sbd configure watchdog-timeout=30 msgwait-timeout=60" on "hanode1" + Then Run "crm sbd configure show disk_metadata|grep -E "watchdog.*30"" OK + Then Run "crm sbd configure show disk_metadata|grep -E "msgwait.*60"" OK + # Add a sbd disk with the existing sbd metadata + Given Run "crm sbd configure show sysconfig|grep "SBD_DEVICE=/dev/sda5"" OK + When Run "crm -F sbd configure device=/dev/sda6" on "hanode1" + Then Run "crm sbd configure show sysconfig|grep -E "SBD_DEVICE=\"/dev/sda5;/dev/sda6\""" OK + Then Run "crm sbd configure show sysconfig|grep -E "SBD_DEVICE=\"/dev/sda5;/dev/sda6\""" OK on "hanode2" + And Run "crm sbd configure show disk_metadata |grep -A 8 '/dev/sda6'|grep -E "watchdog.*30"" OK + And Run "crm sbd configure show disk_metadata |grep -A 8 '/dev/sda6'|grep -E "msgwait.*60"" OK + # Remove a sbd disk + When Run "crm sbd remove device=/dev/sda5" on "hanode1" + Then Run "crm sbd configure show sysconfig|grep "SBD_DEVICE=/dev/sda6"" OK + Then Run "crm sbd configure show sysconfig|grep "SBD_DEVICE=/dev/sda6"" OK on "hanode2" + # Replace a sbd disk + When Run "crm -F sbd configure device=/dev/sda7" on "hanode1" + Then Run "crm sbd configure show sysconfig|grep -E "SBD_DEVICE=\"/dev/sda6;/dev/sda7\""" OK + Then Run "crm sbd configure show sysconfig|grep -E "SBD_DEVICE=\"/dev/sda6;/dev/sda7\""" OK on "hanode2" + And Run "crm sbd configure show disk_metadata |grep -A 8 '/dev/sda7'|grep -E "watchdog.*30"" OK + And Run "crm sbd configure show disk_metadata |grep -A 8 '/dev/sda7'|grep -E "msgwait.*60"" OK + # Remove sbd from cluster + When Run "crm sbd remove" on "hanode1" + And Run "crm cluster restart --all" on "hanode1" + Then Service "sbd.service" is "stopped" on "hanode1" + Then Service "sbd.service" is "stopped" on "hanode2" + + Scenario: sbd configure for diskless sbd + # Newly setup + When Run "crm sbd configure device=""" on "hanode1" + Then Expected "Diskless SBD requires cluster with three or more nodes." in stderr + And Service "sbd" is "started" on "hanode1" + And Service "sbd" is "started" on "hanode2" + And Resource "stonith:fence_sbd" not configured + # Shoud not has any sbd device configured + When Try "crm sbd configure show sysconfig|grep -E "SBD_DEVICE=.+"" + Then Expected return code is "1" + # Remove sbd from cluster + When Run "crm sbd remove" on "hanode1" + And Run "crm cluster restart --all" on "hanode1" + Then Service "sbd.service" is "stopped" on "hanode1" + Then Service "sbd.service" is "stopped" on "hanode2" From 87a2a29b23f053e9e3c158ba0eea8d4332231c88 Mon Sep 17 00:00:00 2001 From: xin liang Date: Fri, 14 Jun 2024 14:40:31 +0800 Subject: [PATCH 02/17] Dev: ui_sbd: Add new 'crm sbd' sublevel (jsc#PED-8256) ** Motivation The main configurations for sbd use cases are scattered among sysconfig, on-disk meta data, CIB, and even could be related to other OS components eg. coredump, SCSI, multipath. It's desirable to reduce the management complexity among them and to streamline the workflow for the main use case scenarios. ** Changed include **** Disk-based SBD scenarios 1. Show usage when syntax error 2. Completion 3. Display SBD related configuration (UC4 in PED-8256) 4. Change the on-disk meta data of the existing sbd disks (UC2.1 in PED-8256) 5. Add a sbd disk with the existing sbd configuration (UC2.2 in PED-8256) 6. Remove a sbd disk (UC2.3 in PED-8256) 7. Remove sbd from cluster 8. Replace the storage for a sbd disk (UC2.4 in PED-8256)] 9. display status (focusing on the runtime information only) (UC5 in PED-8256) **** Disk-less SBD scenarios 1. Show usage when syntax error (diskless) 2. completion (diskless) 3. Display SBD related configuration (UC4 in PED-8256, diskless) 4. Manipulate the basic diskless sbd configuration (UC3.1 in PED-8256) --- crmsh/bootstrap.py | 32 +- crmsh/constants.py | 3 + crmsh/ocfs2.py | 2 +- crmsh/qdevice.py | 8 +- crmsh/sbd.py | 794 ++++++++++++++++++++++++--------------------- crmsh/ui_root.py | 5 + crmsh/ui_sbd.py | 482 +++++++++++++++++++++++++++ crmsh/utils.py | 25 +- crmsh/watchdog.py | 22 +- 9 files changed, 980 insertions(+), 393 deletions(-) create mode 100644 crmsh/ui_sbd.py diff --git a/crmsh/bootstrap.py b/crmsh/bootstrap.py index b240916a7..1b70a3111 100644 --- a/crmsh/bootstrap.py +++ b/crmsh/bootstrap.py @@ -215,12 +215,15 @@ def _validate_sbd_option(self): """ Validate sbd options """ + from .sbd import SBDUtils if self.sbd_devices and self.diskless_sbd: utils.fatal("Can't use -s and -S options together") + if self.sbd_devices: + SBDUtils.verify_sbd_device(self.sbd_devices) if self.stage == "sbd": if not self.sbd_devices and not self.diskless_sbd and self.yes_to_all: utils.fatal("Stage sbd should specify sbd device by -s or diskless sbd by -S option") - if ServiceManager().service_is_active("sbd.service") and not config.core.force: + if ServiceManager().service_is_active(constants.SBD_SERVICE) and not config.core.force: utils.fatal("Can't configure stage sbd: sbd.service already running! Please use crm option '-F' if need to redeploy") if self.cluster_is_running: utils.check_all_nodes_reachable() @@ -298,7 +301,7 @@ def validate_option(self): def init_sbd_manager(self): from .sbd import SBDManager - self.sbd_manager = SBDManager(self) + self.sbd_manager = SBDManager(bootstrap_context=self) def detect_platform(self): """ @@ -400,7 +403,7 @@ def prompt_for_string(msg, match=None, default='', valid_func=None, prev_value=[ def confirm(msg): - if _context.yes_to_all: + if config.core.force or (_context and _context.yes_to_all): return True disable_completion() rc = logger_utils.confirm(msg) @@ -410,12 +413,12 @@ def confirm(msg): def disable_completion(): - if _context.ui_context: + if _context and _context.ui_context: _context.ui_context.disable_completion() def enable_completion(): - if _context.ui_context: + if _context and _context.ui_context: _context.ui_context.setup_readline() @@ -500,7 +503,7 @@ def is_online(): return False # if peer_node is None, this is in the init process - if _context.cluster_node is None: + if not _context or _context.cluster_node is None: return True # In join process # If the joining node is already online but can't find the init node @@ -773,7 +776,7 @@ def start_pacemaker(node_list=[], enable_flag=False): # not _context means not in init or join process if not _context and \ utils.package_is_installed("sbd") and \ - ServiceManager().service_is_enabled("sbd.service") and \ + ServiceManager().service_is_enabled(constants.SBD_SERVICE) and \ SBDTimeout.is_sbd_delay_start(): target_dir = "/run/systemd/system/sbd.service.d/" cmd1 = "mkdir -p {}".format(target_dir) @@ -1400,7 +1403,7 @@ def init_sbd(): import crmsh.sbd if _context.stage == "sbd": crmsh.sbd.clean_up_existing_sbd_resource() - _context.sbd_manager.sbd_init() + _context.sbd_manager.init_and_deploy_sbd() def init_upgradeutil(): @@ -1437,7 +1440,9 @@ def init_cluster(): rsc_defaults rsc-options: resource-stickiness=1 migration-threshold=3 """) - _context.sbd_manager.configure_sbd_resource_and_properties() + if ServiceManager().service_is_enabled(constants.SBD_SERVICE): + _context.sbd_manager.configure_sbd() + def init_admin(): @@ -2737,7 +2742,7 @@ def adjust_stonith_timeout(): """ Adjust stonith-timeout for sbd and other scenarios """ - if ServiceManager().service_is_active("sbd.service"): + if ServiceManager().service_is_active(constants.SBD_SERVICE): from .sbd import SBDTimeout SBDTimeout.adjust_sbd_timeout_related_cluster_configuration() else: @@ -2801,7 +2806,12 @@ def sync_file(path): """ Sync files between cluster nodes """ - if _context.skip_csync2: + if _context: + skip_csync2 = _context.skip_csync2 + else: + skip_csync2 = not ServiceManager().service_is_active(CSYNC2_SERVICE) + + if skip_csync2: utils.cluster_copy_file(path, nodes=_context.node_list_in_cluster, output=False) else: csync2_update(path) diff --git a/crmsh/constants.py b/crmsh/constants.py index 4bdb6704f..89686aa83 100644 --- a/crmsh/constants.py +++ b/crmsh/constants.py @@ -450,4 +450,7 @@ HIDDEN_COMMANDS = {'ms'} NO_SSH_ERROR_MSG = "ssh-related operations are disabled. crmsh works in local mode." + +PCMK_SERVICE = "pacemaker.service" +SBD_SERVICE = "sbd.service" # vim:ts=4:sw=4:et: diff --git a/crmsh/ocfs2.py b/crmsh/ocfs2.py index 346cc5c20..6b5414a42 100644 --- a/crmsh/ocfs2.py +++ b/crmsh/ocfs2.py @@ -119,7 +119,7 @@ def _check_sbd_and_ocfs2_dev(self): """ from . import sbd if ServiceManager().service_is_enabled("sbd.service"): - sbd_device_list = sbd.SBDManager.get_sbd_device_from_config() + sbd_device_list = sbd.SBDUtils.get_sbd_device_from_config() for dev in self.ocfs2_devices: if dev in sbd_device_list: self._dynamic_raise_error("{} cannot be the same with SBD device".format(dev)) diff --git a/crmsh/qdevice.py b/crmsh/qdevice.py index 982d7a688..e81fae41a 100644 --- a/crmsh/qdevice.py +++ b/crmsh/qdevice.py @@ -614,15 +614,15 @@ def adjust_sbd_watchdog_timeout_with_qdevice(self): """ Adjust SBD_WATCHDOG_TIMEOUT when configuring qdevice and diskless SBD """ - from .sbd import SBDManager, SBDTimeout + from .sbd import SBDManager, SBDTimeout, SBDUtils utils.check_all_nodes_reachable() - self.using_diskless_sbd = SBDManager.is_using_diskless_sbd() + self.using_diskless_sbd = SBDUtils.is_using_diskless_sbd() # add qdevice after diskless sbd started if self.using_diskless_sbd: - res = SBDManager.get_sbd_value_from_config("SBD_WATCHDOG_TIMEOUT") + res = SBDUtils.get_sbd_value_from_config("SBD_WATCHDOG_TIMEOUT") if not res or int(res) < SBDTimeout.SBD_WATCHDOG_TIMEOUT_DEFAULT_WITH_QDEVICE: sbd_watchdog_timeout_qdevice = SBDTimeout.SBD_WATCHDOG_TIMEOUT_DEFAULT_WITH_QDEVICE - SBDManager.update_configuration({"SBD_WATCHDOG_TIMEOUT": str(sbd_watchdog_timeout_qdevice)}) + SBDManager.update_sbd_configuration({"SBD_WATCHDOG_TIMEOUT": str(sbd_watchdog_timeout_qdevice)}) utils.set_property("stonith-timeout", SBDTimeout.get_stonith_timeout()) @qnetd_lock_for_same_cluster_name diff --git a/crmsh/sbd.py b/crmsh/sbd.py index d7f569e68..4b732b1ad 100644 --- a/crmsh/sbd.py +++ b/crmsh/sbd.py @@ -1,5 +1,6 @@ import os import re +import typing from . import utils, sh from . import bootstrap from .bootstrap import SYSCONFIG_SBD, SBD_SYSTEMD_DELAY_START_DIR @@ -7,17 +8,130 @@ from . import constants from . import corosync from . import xmlutil +from . import watchdog from .service_manager import ServiceManager from .sh import ShellUtils logger = log.setup_logger(__name__) -logger_utils = log.LoggerUtils(logger) + + +class SBDUtils: + ''' + Consolidate sbd related utility methods + ''' + @staticmethod + def get_sbd_device_metadata(dev, timeout_only=False, remote=None) -> dict: + ''' + Extract metadata from sbd device header + ''' + sbd_info = {} + try: + out = sh.cluster_shell().get_stdout_or_raise_error(f"sbd -d {dev} dump", remote) + except: + return sbd_info + + pattern = r"UUID\s+:\s+(\S+)|Timeout\s+\((\w+)\)\s+:\s+(\d+)" + matches = re.findall(pattern, out) + for uuid, timeout_type, timeout_value in matches: + if uuid and not timeout_only: + sbd_info["uuid"] = uuid + elif timeout_type and timeout_value: + sbd_info[timeout_type] = int(timeout_value) + return sbd_info + + @staticmethod + def get_device_uuid(dev, node=None): + ''' + Get UUID for specific device and node + ''' + res = SBDUtils.get_sbd_device_metadata(dev, remote=node).get("uuid") + if not res: + raise ValueError(f"Cannot find sbd device UUID for {dev}") + return res + + @staticmethod + def compare_device_uuid(dev, node_list): + ''' + Compare local sbd device UUID with other node's sbd device UUID + ''' + if not node_list: + return + local_uuid = SBDUtils.get_device_uuid(dev) + for node in node_list: + remote_uuid = SBDUtils.get_device_uuid(dev, node) + if local_uuid != remote_uuid: + raise ValueError(f"Device {dev} doesn't have the same UUID with {node}") + + @staticmethod + def verify_sbd_device(dev_list, compare_node_list=[]): + if len(dev_list) > SBDManager.SBD_DEVICE_MAX: + raise ValueError(f"Maximum number of SBD device is {SBDManager.SBD_DEVICE_MAX}") + for dev in dev_list: + if not utils.is_block_device(dev): + raise ValueError(f"{dev} doesn't look like a block device") + SBDUtils.compare_device_uuid(dev, compare_node_list) + + @staticmethod + def get_sbd_value_from_config(key): + ''' + Get value from /etc/sysconfig/sbd + ''' + return utils.parse_sysconfig(SYSCONFIG_SBD).get(key) + + @staticmethod + def get_sbd_device_from_config(): + ''' + Get sbd device list from config + ''' + res = SBDUtils.get_sbd_value_from_config("SBD_DEVICE") + return res.split(';') if res else [] + + @staticmethod + def is_using_diskless_sbd(): + ''' + Check if using diskless SBD + ''' + dev_list = SBDUtils.get_sbd_device_from_config() + return not dev_list and ServiceManager().service_is_active(constants.SBD_SERVICE) + + @staticmethod + def has_sbd_device_already_initialized(dev) -> bool: + ''' + Check if sbd device already initialized + ''' + cmd = "sbd -d {} dump".format(dev) + rc, _, _ = ShellUtils().get_stdout_stderr(cmd) + return rc == 0 + + @staticmethod + def no_overwrite_device_check(dev) -> bool: + ''' + Check if device already initialized and ask if need to overwrite + ''' + initialized = SBDUtils.has_sbd_device_already_initialized(dev) + return initialized and \ + not bootstrap.confirm(f"{dev} has already been initialized by SBD, do you want to overwrite it?") + + @staticmethod + def check_devices_metadata_consistent(dev_list) -> bool: + ''' + Check if all devices have the same metadata + ''' + consistent = True + if len(dev_list) < 2: + return consistent + first_dev_metadata = SBDUtils.get_sbd_device_metadata(dev_list[0], timeout_only=True) + for dev in dev_list[1:]: + if SBDUtils.get_sbd_device_metadata(dev, timeout_only=True) != first_dev_metadata: + logger.warning(f"Device {dev} doesn't have the same metadata as {dev_list[0]}") + consistent = False + return consistent class SBDTimeout(object): - """ + ''' Consolidate sbd related timeout methods and constants - """ + ''' STONITH_WATCHDOG_TIMEOUT_DEFAULT = -1 SBD_WATCHDOG_TIMEOUT_DEFAULT = 5 SBD_WATCHDOG_TIMEOUT_DEFAULT_S390 = 15 @@ -25,15 +139,14 @@ class SBDTimeout(object): QDEVICE_SYNC_TIMEOUT_MARGIN = 5 def __init__(self, context=None): - """ + ''' Init function - """ + ''' self.context = context self.sbd_msgwait = None self.stonith_timeout = None self.sbd_watchdog_timeout = self.SBD_WATCHDOG_TIMEOUT_DEFAULT self.stonith_watchdog_timeout = self.STONITH_WATCHDOG_TIMEOUT_DEFAULT - self.sbd_delay_start = None self.two_node_without_qdevice = False def initialize_timeout(self): @@ -44,10 +157,10 @@ def initialize_timeout(self): self._set_sbd_msgwait() def _set_sbd_watchdog_timeout(self): - """ + ''' Set sbd_watchdog_timeout from profiles.yml if exists Then adjust it if in s390 environment - """ + ''' if "sbd.watchdog_timeout" in self.context.profiles_dict: self.sbd_watchdog_timeout = int(self.context.profiles_dict["sbd.watchdog_timeout"]) if self.context.is_s390 and self.sbd_watchdog_timeout < self.SBD_WATCHDOG_TIMEOUT_DEFAULT_S390: @@ -55,10 +168,10 @@ def _set_sbd_watchdog_timeout(self): self.sbd_watchdog_timeout = self.SBD_WATCHDOG_TIMEOUT_DEFAULT_S390 def _set_sbd_msgwait(self): - """ + ''' Set sbd msgwait from profiles.yml if exists Default is 2 * sbd_watchdog_timeout - """ + ''' sbd_msgwait_default = 2 * self.sbd_watchdog_timeout sbd_msgwait = sbd_msgwait_default if "sbd.msgwait" in self.context.profiles_dict: @@ -68,10 +181,25 @@ def _set_sbd_msgwait(self): sbd_msgwait = sbd_msgwait_default self.sbd_msgwait = sbd_msgwait + @classmethod + def get_advised_sbd_timeout(cls, diskless=False) -> typing.Tuple[int, int]: + ''' + Get suitable sbd_watchdog_timeout and sbd_msgwait + ''' + ctx = bootstrap.Context() + ctx.diskless_sbd = diskless + ctx.load_profiles() + time_inst = cls(ctx) + time_inst.initialize_timeout() + + sbd_watchdog_timeout = time_inst.sbd_watchdog_timeout + sbd_msgwait = None if diskless else time_inst.sbd_msgwait + return sbd_watchdog_timeout, sbd_msgwait + def _adjust_sbd_watchdog_timeout_with_diskless_and_qdevice(self): - """ + ''' When using diskless SBD with Qdevice, adjust value of sbd_watchdog_timeout - """ + ''' # add sbd after qdevice started if corosync.is_qdevice_configured() and ServiceManager().service_is_active("corosync-qdevice.service"): qdevice_sync_timeout = utils.get_qdevice_sync_timeout() @@ -87,44 +215,42 @@ def _adjust_sbd_watchdog_timeout_with_diskless_and_qdevice(self): @staticmethod def get_sbd_msgwait(dev): - """ + ''' Get msgwait for sbd device - """ - out = sh.cluster_shell().get_stdout_or_raise_error("sbd -d {} dump".format(dev)) - # Format like "Timeout (msgwait) : 30" - res = re.search("\(msgwait\)\s+:\s+(\d+)", out) + ''' + res = SBDUtils.get_sbd_device_metadata(dev).get("msgwait") if not res: - raise ValueError("Cannot get sbd msgwait for {}".format(dev)) - return int(res.group(1)) + raise ValueError(f"Cannot get sbd msgwait for {dev}") + return res @staticmethod def get_sbd_watchdog_timeout(): - """ + ''' Get SBD_WATCHDOG_TIMEOUT from /etc/sysconfig/sbd - """ - res = SBDManager.get_sbd_value_from_config("SBD_WATCHDOG_TIMEOUT") + ''' + res = SBDUtils.get_sbd_value_from_config("SBD_WATCHDOG_TIMEOUT") if not res: raise ValueError("Cannot get the value of SBD_WATCHDOG_TIMEOUT") return int(res) @staticmethod def get_stonith_watchdog_timeout(): - """ + ''' For non-bootstrap case, get stonith-watchdog-timeout value from cluster property - """ + ''' default = SBDTimeout.STONITH_WATCHDOG_TIMEOUT_DEFAULT - if not ServiceManager().service_is_active("pacemaker.service"): + if not ServiceManager().service_is_active(constants.PCMK_SERVICE): return default value = utils.get_property("stonith-watchdog-timeout") return int(value.strip('s')) if value else default def _load_configurations(self): - """ + ''' Load necessary configurations for both disk-based/disk-less sbd - """ + ''' self.two_node_without_qdevice = utils.is_2node_cluster_without_qdevice() - dev_list = SBDManager.get_sbd_device_from_config() + dev_list = SBDUtils.get_sbd_device_from_config() if dev_list: # disk-based self.disk_based = True self.msgwait = SBDTimeout.get_sbd_msgwait(dev_list[0]) @@ -134,19 +260,19 @@ def _load_configurations(self): self.sbd_watchdog_timeout = SBDTimeout.get_sbd_watchdog_timeout() self.stonith_watchdog_timeout = SBDTimeout.get_stonith_watchdog_timeout() self.sbd_delay_start_value_expected = self.get_sbd_delay_start_expected() if utils.detect_virt() else "no" - self.sbd_delay_start_value_from_config = SBDManager.get_sbd_value_from_config("SBD_DELAY_START") + self.sbd_delay_start_value_from_config = SBDUtils.get_sbd_value_from_config("SBD_DELAY_START") logger.debug("Inspect SBDTimeout: %s", vars(self)) def get_stonith_timeout_expected(self): - """ + ''' Get stonith-timeout value for sbd cases, formulas are: value_from_sbd = 1.2 * (pcmk_delay_max + msgwait) # for disk-based sbd value_from_sbd = 1.2 * max (stonith_watchdog_timeout, 2*SBD_WATCHDOG_TIMEOUT) # for disk-less sbd stonith_timeout = max(value_from_sbd, constants.STONITH_TIMEOUT_DEFAULT) + token + consensus - """ + ''' if self.disk_based: value_from_sbd = int(1.2*(self.pcmk_delay_max + self.msgwait)) else: @@ -163,12 +289,12 @@ def get_stonith_timeout(cls): return cls_inst.get_stonith_timeout_expected() def get_sbd_delay_start_expected(self): - """ + ''' Get the value for SBD_DELAY_START, formulas are: SBD_DELAY_START = (token + consensus + pcmk_delay_max + msgwait) # for disk-based sbd SBD_DELAY_START = (token + consensus + 2*SBD_WATCHDOG_TIMEOUT) # for disk-less sbd - """ + ''' token_and_consensus_timeout = corosync.token_and_consensus_timeout() if self.disk_based: value = token_and_consensus_timeout + self.pcmk_delay_max + self.msgwait @@ -178,34 +304,38 @@ def get_sbd_delay_start_expected(self): @staticmethod def get_sbd_delay_start_sec_from_sysconfig(): - """ + ''' Get suitable systemd start timeout for sbd.service - """ + ''' # TODO 5ms, 5us, 5s, 5m, 5h are also valid for sbd sysconfig - value = SBDManager.get_sbd_value_from_config("SBD_DELAY_START") + value = SBDUtils.get_sbd_value_from_config("SBD_DELAY_START") if utils.is_boolean_true(value): return 2*SBDTimeout.get_sbd_watchdog_timeout() return int(value) @staticmethod def is_sbd_delay_start(): - """ + ''' Check if SBD_DELAY_START is not no or not set - """ - res = SBDManager.get_sbd_value_from_config("SBD_DELAY_START") + ''' + res = SBDUtils.get_sbd_value_from_config("SBD_DELAY_START") return res and res != "no" + @staticmethod + def get_sbd_systemd_start_timeout() -> int: + cmd = "systemctl show -p TimeoutStartUSec sbd --value" + out = sh.cluster_shell().get_stdout_or_raise_error(cmd) + return utils.get_systemd_timeout_start_in_sec(out) + def adjust_systemd_start_timeout(self): - """ + ''' Adjust start timeout for sbd when set SBD_DELAY_START - """ - sbd_delay_start_value = SBDManager.get_sbd_value_from_config("SBD_DELAY_START") + ''' + sbd_delay_start_value = SBDUtils.get_sbd_value_from_config("SBD_DELAY_START") if sbd_delay_start_value == "no": return - cmd = "systemctl show -p TimeoutStartUSec sbd --value" - out = sh.cluster_shell().get_stdout_or_raise_error(cmd) - start_timeout = utils.get_systemd_timeout_start_in_sec(out) + start_timeout = SBDTimeout.get_sbd_systemd_start_timeout() if start_timeout > int(sbd_delay_start_value): return @@ -216,15 +346,15 @@ def adjust_systemd_start_timeout(self): utils.cluster_run_cmd("systemctl daemon-reload") def adjust_stonith_timeout(self): - """ + ''' Adjust stonith-timeout property - """ + ''' utils.set_property("stonith-timeout", self.get_stonith_timeout_expected(), conditional=True) def adjust_sbd_delay_start(self): - """ + ''' Adjust SBD_DELAY_START in /etc/sysconfig/sbd - """ + ''' expected_value = str(self.sbd_delay_start_value_expected) config_value = self.sbd_delay_start_value_from_config if expected_value == config_value: @@ -232,29 +362,23 @@ def adjust_sbd_delay_start(self): if expected_value == "no" \ or (not re.search(r'\d+', config_value)) \ or (int(expected_value) > int(config_value)): - SBDManager.update_configuration({"SBD_DELAY_START": expected_value}) + SBDManager.update_sbd_configuration({"SBD_DELAY_START": expected_value}) @classmethod def adjust_sbd_timeout_related_cluster_configuration(cls): - """ + ''' Adjust sbd timeout related configurations - """ + ''' cls_inst = cls() cls_inst._load_configurations() + cls_inst.adjust_sbd_delay_start() + cls_inst.adjust_stonith_timeout() + cls_inst.adjust_systemd_start_timeout() - message = "Adjusting sbd related timeout values" - with logger_utils.status_long(message): - cls_inst.adjust_sbd_delay_start() - cls_inst.adjust_stonith_timeout() - cls_inst.adjust_systemd_start_timeout() - -class SBDManager(object): - """ - Class to manage sbd configuration and services - """ +class SBDManager: SYSCONFIG_SBD_TEMPLATE = "/usr/share/fillup-templates/sysconfig.sbd" - SBD_STATUS_DESCRIPTION = """Configure SBD: + SBD_STATUS_DESCRIPTION = '''Configure SBD: If you have shared storage, for example a SAN or iSCSI target, you can use it avoid split-brain scenarios by configuring SBD. This requires a 1 MB partition, accessible to all nodes in the @@ -262,91 +386,174 @@ class SBDManager(object): across all nodes in the cluster, so /dev/disk/by-id/* devices are a good choice. Note that all data on the partition you specify here will be destroyed. -""" - SBD_WARNING = "Not configuring SBD - STONITH will be disabled." +''' + NO_SBD_WARNING = "Not configuring SBD - STONITH will be disabled." + DISKLESS_SBD_MIN_EXPECTED_VOTE = 3 DISKLESS_SBD_WARNING = "Diskless SBD requires cluster with three or more nodes. If you want to use diskless SBD for 2-node cluster, should be combined with QDevice." - PARSE_RE = "[; ]" - DISKLESS_CRM_CMD = "crm configure property stonith-enabled=true stonith-watchdog-timeout={} stonith-timeout={}" SBD_RA = "stonith:fence_sbd" SBD_RA_ID = "stonith-sbd" + SBD_DEVICE_MAX = 3 + + def __init__( + self, + device_list_to_init: typing.List[str] | None = None, + timeout_dict: typing.Dict[str, int] | None = None, + update_dict: typing.Dict[str, str] | None = None, + no_overwrite_dev_map: typing.Dict[str, bool] | None = None, + new_config: bool = False, + diskless_sbd: bool = False, + bootstrap_context: bootstrap.Context | None = None + ): + ''' + Init function which can be called from crm sbd subcommand or bootstrap + ''' + self.package_installed = utils.package_is_installed("sbd") + if not self.package_installed: + return - def __init__(self, context): - """ - Init function + self.device_list_to_init = device_list_to_init or [] + self.timeout_dict = timeout_dict or {} + self.update_dict = update_dict or {} + self.diskless_sbd = diskless_sbd + self.cluster_is_running = ServiceManager().service_is_active(constants.PCMK_SERVICE) + self.bootstrap_context = bootstrap_context + self.no_overwrite_dev_map = no_overwrite_dev_map or {} + self.new_config = new_config + + # From bootstrap init or join process, override the values + if self.bootstrap_context: + self.device_list_to_init = self.bootstrap_context.sbd_devices + self.diskless_sbd = self.bootstrap_context.diskless_sbd + self.cluster_is_running = self.bootstrap_context.cluster_is_running + + def _load_attributes_from_bootstrap(self): + if not self.bootstrap_context: + return + timeout_inst = SBDTimeout(self.bootstrap_context) + timeout_inst.initialize_timeout() + self.timeout_dict["watchdog"] = timeout_inst.sbd_watchdog_timeout + if not self.diskless_sbd: + self.timeout_dict["msgwait"] = timeout_inst.sbd_msgwait + self.update_dict["SBD_WATCHDOG_TIMEOUT"] = str(timeout_inst.sbd_watchdog_timeout) + self.update_dict["SBD_WATCHDOG_DEV"] = watchdog.Watchdog.get_watchdog_device(self.bootstrap_context.watchdog) - sbd_devices is provided by '-s' option on init process - diskless_sbd is provided by '-S' option on init process - """ - self.sbd_devices_input = context.sbd_devices - self.diskless_sbd = context.diskless_sbd - self._sbd_devices = None - self._watchdog_inst = None - self._context = context - self._delay_start = False - self.timeout_inst = None - self.no_overwrite_map = {} - self.no_update_config = False + @staticmethod + def convert_timeout_dict_to_opt_str(timeout_dict: typing.Dict[str, int]) -> str: + timeout_option_map = { + "watchdog": "-1", + "allocate": "-2", + "loop": "-3", + "msgwait": "-4" + } + return ' '.join([f"{timeout_option_map[k]} {v}" for k, v in timeout_dict.items() + if k in timeout_option_map]) + + def update_configuration(self) -> None: + ''' + Update and sync sbd configuration + ''' + if not self.update_dict: + return + if (self.bootstrap_context and self.bootstrap_context.type == "init") or self.new_config: + utils.copy_local_file(self.SYSCONFIG_SBD_TEMPLATE, SYSCONFIG_SBD) + + for key, value in self.update_dict.items(): + logger.info("Update %s in %s: %s", key, SYSCONFIG_SBD, value) + utils.sysconfig_set(SYSCONFIG_SBD, **self.update_dict) + bootstrap.sync_file(SYSCONFIG_SBD) + logger.info("Already synced %s to all nodes", SYSCONFIG_SBD) + + @classmethod + def update_sbd_configuration(cls, update_dict: typing.Dict[str, str]) -> None: + inst = cls(update_dict=update_dict) + inst.update_configuration() + + def initialize_sbd(self): + if self.diskless_sbd: + logger.info("Configuring diskless SBD") + self._warn_diskless_sbd() + return + elif not all(self.no_overwrite_dev_map.values()): + logger.info("Configuring disk-based SBD") + + opt_str = SBDManager.convert_timeout_dict_to_opt_str(self.timeout_dict) + shell = sh.cluster_shell() + for dev in self.device_list_to_init: + # skip if device already initialized and not overwrite + if dev in self.no_overwrite_dev_map and self.no_overwrite_dev_map[dev]: + continue + logger.info("Initializing SBD device %s", dev) + cmd = f"sbd {opt_str} -d {dev} create" + logger.debug("Running command: %s", cmd) + shell.get_stdout_or_raise_error(cmd) + + SBDUtils.check_devices_metadata_consistent(self.device_list_to_init) @staticmethod - def _get_device_uuid(dev, node=None): - """ - Get UUID for specific device and node - """ - out = sh.cluster_shell().get_stdout_or_raise_error("sbd -d {} dump".format(dev), node) - res = re.search("UUID\s*:\s*(.*)\n", out) - if not res: - raise ValueError("Cannot find sbd device UUID for {}".format(dev)) - return res.group(1) + def enable_sbd_service(): + cluster_nodes = utils.list_cluster_nodes() or [utils.this_node()] + service_manager = ServiceManager() - def _compare_device_uuid(self, dev, node_list): - """ - Compare local sbd device UUID with other node's sbd device UUID - """ - if not node_list: + for node in cluster_nodes: + if not service_manager.service_is_enabled(constants.SBD_SERVICE, node): + logger.info("Enable %s on node %s", constants.SBD_SERVICE, node) + service_manager.enable_service(constants.SBD_SERVICE, node) + + @staticmethod + def restart_cluster_if_possible(): + if not ServiceManager().service_is_active(constants.PCMK_SERVICE): return - local_uuid = self._get_device_uuid(dev) - for node in node_list: - remote_uuid = self._get_device_uuid(dev, node) - if local_uuid != remote_uuid: - raise ValueError("Device {} doesn't have the same UUID with {}".format(dev, node)) - - def _verify_sbd_device(self, dev_list, compare_node_list=[]): - """ - Verify sbd device - """ - if len(dev_list) > 3: - raise ValueError("Maximum number of SBD device is 3") - for dev in dev_list: - if not utils.is_block_device(dev): - raise ValueError("{} doesn't look like a block device".format(dev)) - self._compare_device_uuid(dev, compare_node_list) + if xmlutil.CrmMonXmlParser().is_any_resource_running(): + logger.warning("Resource is running, need to restart cluster service manually on each node") + else: + logger.info("Restarting cluster service") + utils.cluster_run_cmd("crm cluster restart") + bootstrap.wait_for_cluster() - def _no_overwrite_check(self, dev): - """ - Check if device already initialized and if need to overwrite - """ - return SBDManager.has_sbd_device_already_initialized(dev) and not bootstrap.confirm("SBD is already configured to use {} - overwrite?".format(dev)) + def configure_sbd(self): + ''' + Configure fence_sbd resource and related properties + ''' + if self.diskless_sbd: + utils.set_property("stonith-watchdog-timeout", SBDTimeout.STONITH_WATCHDOG_TIMEOUT_DEFAULT) + elif not xmlutil.CrmMonXmlParser().is_resource_configured(self.SBD_RA): + all_device_list = SBDUtils.get_sbd_device_from_config() + devices_param_str = f"params devices=\"{','.join(all_device_list)}\"" + cmd = f"crm configure primitive {self.SBD_RA_ID} {self.SBD_RA} {devices_param_str}" + sh.cluster_shell().get_stdout_or_raise_error(cmd) + utils.set_property("stonith-enabled", "true") - def _get_sbd_device_interactive(self): - """ + def _warn_diskless_sbd(self, peer=None): + ''' + Give warning when configuring diskless sbd + ''' + # When in sbd stage or join process + if (self.diskless_sbd and self.cluster_is_running) or peer: + vote_dict = utils.get_quorum_votes_dict(peer) + expected_vote = int(vote_dict.get('Expected', 0)) + if expected_vote < self.DISKLESS_SBD_MIN_EXPECTED_VOTE: + logger.warning(self.DISKLESS_SBD_WARNING) + # When in init process + elif self.diskless_sbd: + logger.warning(self.DISKLESS_SBD_WARNING) + + def get_sbd_device_interactive(self): + ''' Get sbd device on interactive mode - """ - if self._context.yes_to_all: - logger.warning(self.SBD_WARNING) + ''' + if self.bootstrap_context.yes_to_all: + logger.warning(self.NO_SBD_WARNING) return - logger.info(self.SBD_STATUS_DESCRIPTION) - if not bootstrap.confirm("Do you wish to use SBD?"): - logger.warning(self.SBD_WARNING) + logger.warning(self.NO_SBD_WARNING) return - configured_dev_list = self._get_sbd_device_from_config() - for dev in configured_dev_list: - self.no_overwrite_map[dev] = self._no_overwrite_check(dev) - if self.no_overwrite_map and all(self.no_overwrite_map.values()): - self.no_update_config = True - return configured_dev_list + configured_devices = SBDUtils.get_sbd_device_from_config() + for dev in configured_devices: + self.no_overwrite_dev_map[dev] = SBDUtils.no_overwrite_device_check(dev) + if self.no_overwrite_dev_map and all(self.no_overwrite_dev_map.values()): + return configured_devices dev_list = [] dev_looks_sane = False @@ -356,21 +563,20 @@ def _get_sbd_device_interactive(self): self.diskless_sbd = True return - dev_list = utils.re_split_string(self.PARSE_RE, dev) + dev_list = utils.re_split_string("[; ]", dev) try: - self._verify_sbd_device(dev_list) - except ValueError as err_msg: - logger.error(str(err_msg)) + SBDUtils.verify_sbd_device(dev_list) + except ValueError as e: + logger.error(e) continue - for dev in dev_list: - if dev not in self.no_overwrite_map: - self.no_overwrite_map[dev] = self._no_overwrite_check(dev) - if self.no_overwrite_map[dev]: + if dev not in self.no_overwrite_dev_map: + self.no_overwrite_dev_map[dev] = SBDUtils.no_overwrite_device_check(dev) + if self.no_overwrite_dev_map[dev]: if dev == dev_list[-1]: return dev_list continue - logger.warning("All data on {} will be destroyed!".format(dev)) + logger.warning("All data on %s will be destroyed", dev) if bootstrap.confirm('Are you sure you wish to use this device?'): dev_looks_sane = True else: @@ -379,250 +585,73 @@ def _get_sbd_device_interactive(self): return dev_list - def _get_sbd_device(self): - """ - Get sbd device from options or interactive mode - """ - dev_list = [] - if self.sbd_devices_input: - dev_list = self.sbd_devices_input - self._verify_sbd_device(dev_list) - for dev in dev_list: - self.no_overwrite_map[dev] = self._no_overwrite_check(dev) - if all(self.no_overwrite_map.values()) and dev_list == self._get_sbd_device_from_config(): - self.no_update_config = True + def get_sbd_device_from_bootstrap(self): + ''' + Handle sbd device input from 'crm cluster init' with -s or -S option + -s is for disk-based sbd + -S is for diskless sbd + ''' + # specified sbd device with -s option + if self.device_list_to_init: + self.update_dict["SBD_DEVICE"] = ';'.join(self.device_list_to_init) + # no -s and no -S option elif not self.diskless_sbd: - dev_list = self._get_sbd_device_interactive() - self._sbd_devices = dev_list - - def _initialize_sbd(self): - """ - Initialize SBD parameters according to profiles.yml, or the crmsh defined defaulst as the last resort. - This covers both disk-based-sbd, and diskless-sbd scenarios. - For diskless-sbd, set sbd_watchdog_timeout then return; - For disk-based-sbd, also calculate the msgwait value, then initialize the SBD device. - """ - msg = "" - if self.diskless_sbd: - msg = "Configuring diskless SBD" - elif not all(self.no_overwrite_map.values()): - msg = "Initializing SBD" - if msg: - logger.info(msg) - self.timeout_inst = SBDTimeout(self._context) - self.timeout_inst.initialize_timeout() - if self.diskless_sbd: + self.device_list_to_init = self.get_sbd_device_interactive() + + def init_and_deploy_sbd(self): + ''' + The process of deploying sbd includes: + 1. Initialize sbd device + 2. Write config file /etc/sysconfig/sbd + 3. Enable sbd.service + 4. Restart cluster service if possible + 5. Configure stonith-sbd resource and related properties + ''' + if not self.package_installed: return - opt = "-4 {} -1 {}".format(self.timeout_inst.sbd_msgwait, self.timeout_inst.sbd_watchdog_timeout) - - for dev in self._sbd_devices: - if dev in self.no_overwrite_map and self.no_overwrite_map[dev]: - continue - rc, _, err = bootstrap.invoke("sbd {} -d {} create".format(opt, dev)) - if not rc: - utils.fatal("Failed to initialize SBD device {}: {}".format(dev, err)) - - def _update_sbd_configuration(self): - """ - Update /etc/sysconfig/sbd - """ - if self.no_update_config: - bootstrap.sync_file(SYSCONFIG_SBD) - return - - utils.copy_local_file(self.SYSCONFIG_SBD_TEMPLATE, SYSCONFIG_SBD) - sbd_config_dict = { - "SBD_WATCHDOG_DEV": self._watchdog_inst.watchdog_device_name, - "SBD_WATCHDOG_TIMEOUT": str(self.timeout_inst.sbd_watchdog_timeout) - } - if self._sbd_devices: - sbd_config_dict["SBD_DEVICE"] = ';'.join(self._sbd_devices) - utils.sysconfig_set(SYSCONFIG_SBD, **sbd_config_dict) - bootstrap.sync_file(SYSCONFIG_SBD) - - def _get_sbd_device_from_config(self): - """ - Gets currently configured SBD device, i.e. what's in /etc/sysconfig/sbd - """ - res = SBDManager.get_sbd_value_from_config("SBD_DEVICE") - if res: - return utils.re_split_string(self.PARSE_RE, res) - else: - return [] - - def _restart_cluster_and_configure_sbd_ra(self): - """ - Try to configure sbd resource, restart cluster on needed - """ - if not xmlutil.CrmMonXmlParser().is_any_resource_running(): - logger.info("Restarting cluster service") - utils.cluster_run_cmd("crm cluster restart") - bootstrap.wait_for_cluster() - self.configure_sbd_resource_and_properties() - else: - logger.warning("To start sbd.service, need to restart cluster service manually on each node") - if self.diskless_sbd: - cmd = self.DISKLESS_CRM_CMD.format(self.timeout_inst.stonith_watchdog_timeout, SBDTimeout.get_stonith_timeout()) - logger.warning("Then run \"{}\" on any node".format(cmd)) - else: - self.configure_sbd_resource_and_properties() - - def _enable_sbd_service(self): - """ - Try to enable sbd service - """ - if self._context.cluster_is_running: - # in sbd stage, enable sbd.service on cluster wide - utils.cluster_run_cmd("systemctl enable sbd.service") - self._restart_cluster_and_configure_sbd_ra() - else: - # in init process - bootstrap.invoke("systemctl enable sbd.service") - - def _warn_diskless_sbd(self, peer=None): - """ - Give warning when configuring diskless sbd - """ - # When in sbd stage or join process - if (self.diskless_sbd and self._context.cluster_is_running) or peer: - vote_dict = utils.get_quorum_votes_dict(peer) - expected_vote = int(vote_dict['Expected']) - if (expected_vote < 2 and peer) or (expected_vote < 3 and not peer): - logger.warning(self.DISKLESS_SBD_WARNING) - # When in init process - elif self.diskless_sbd: - logger.warning(self.DISKLESS_SBD_WARNING) - - def sbd_init(self): - """ - Function sbd_init includes these steps: - 1. Get sbd device from options or interactive mode - 2. Initialize sbd device - 3. Write config file /etc/sysconfig/sbd - """ - from .watchdog import Watchdog - - if not utils.package_is_installed("sbd"): - return - self._watchdog_inst = Watchdog(_input=self._context.watchdog) - self._watchdog_inst.init_watchdog() - self._get_sbd_device() - if not self._sbd_devices and not self.diskless_sbd: - bootstrap.invoke("systemctl disable sbd.service") - return - self._warn_diskless_sbd() - self._initialize_sbd() - self._update_sbd_configuration() - self._enable_sbd_service() - - def configure_sbd_resource_and_properties(self): - """ - Configure stonith-sbd resource and related properties - """ - if not utils.package_is_installed("sbd") or \ - not ServiceManager().service_is_enabled("sbd.service") or \ - xmlutil.CrmMonXmlParser().is_resource_configured(self.SBD_RA): - return - shell = sh.cluster_shell() + if self.bootstrap_context: + self.get_sbd_device_from_bootstrap() + if not self.device_list_to_init and not self.diskless_sbd: + ServiceManager().disable_service(constants.SBD_SERVICE) + return + self._load_attributes_from_bootstrap() - # disk-based sbd - if self._get_sbd_device_from_config(): - devices_param_str = f"params devices=\"{','.join(self._sbd_devices)}\"" - cmd = f"crm configure primitive {self.SBD_RA_ID} {self.SBD_RA} {devices_param_str}" - shell.get_stdout_or_raise_error(cmd) - utils.set_property("stonith-enabled", "true") - # disk-less sbd - else: - if self.timeout_inst is None: - self.timeout_inst = SBDTimeout(self._context) - self.timeout_inst.initialize_timeout() - cmd = self.DISKLESS_CRM_CMD.format(self.timeout_inst.stonith_watchdog_timeout, constants.STONITH_TIMEOUT_DEFAULT) - shell.get_stdout_or_raise_error(cmd) + self.initialize_sbd() + self.update_configuration() + SBDManager.enable_sbd_service() - # in sbd stage - if self._context.cluster_is_running: + if self.cluster_is_running: + SBDManager.restart_cluster_if_possible() + self.configure_sbd() bootstrap.adjust_properties() def join_sbd(self, remote_user, peer_host): - """ + ''' Function join_sbd running on join process only On joining process, check whether peer node has enabled sbd.service If so, check prerequisites of SBD and verify sbd device on join node - """ - from .watchdog import Watchdog - - if not utils.package_is_installed("sbd"): + ''' + if not self.package_installed: return - if not os.path.exists(SYSCONFIG_SBD) or not ServiceManager().service_is_enabled("sbd.service", peer_host): - bootstrap.invoke("systemctl disable sbd.service") + + service_manager = ServiceManager() + if not os.path.exists(SYSCONFIG_SBD) or not service_manager.service_is_enabled(constants.SBD_SERVICE, peer_host): + service_manager.disable_service(constants.SBD_SERVICE) return + + from .watchdog import Watchdog self._watchdog_inst = Watchdog(remote_user=remote_user, peer_host=peer_host) self._watchdog_inst.join_watchdog() - dev_list = self._get_sbd_device_from_config() + + dev_list = SBDUtils.get_sbd_device_from_config() if dev_list: - self._verify_sbd_device(dev_list, [peer_host]) + SBDUtils.verify_sbd_device(dev_list, [peer_host]) else: self._warn_diskless_sbd(peer_host) - logger.info("Got {}SBD configuration".format("" if dev_list else "diskless ")) - bootstrap.invoke("systemctl enable sbd.service") - - @classmethod - def verify_sbd_device(cls): - """ - This classmethod is for verifying sbd device on a running cluster - Raise ValueError for exceptions - """ - inst = cls(bootstrap.Context()) - dev_list = inst._get_sbd_device_from_config() - if not dev_list: - raise ValueError("No sbd device configured") - inst._verify_sbd_device(dev_list, utils.list_cluster_nodes_except_me()) - - @classmethod - def get_sbd_device_from_config(cls): - """ - Get sbd device list from config - """ - inst = cls(bootstrap.Context()) - return inst._get_sbd_device_from_config() - @classmethod - def is_using_diskless_sbd(cls): - """ - Check if using diskless SBD - """ - inst = cls(bootstrap.Context()) - dev_list = inst._get_sbd_device_from_config() - if not dev_list and ServiceManager().service_is_active("sbd.service"): - return True - return False - - @staticmethod - def update_configuration(sbd_config_dict): - """ - Update and sync sbd configuration - """ - utils.sysconfig_set(SYSCONFIG_SBD, **sbd_config_dict) - bootstrap.sync_file(SYSCONFIG_SBD) - - @staticmethod - def get_sbd_value_from_config(key): - """ - Get value from /etc/sysconfig/sbd - """ - conf = utils.parse_sysconfig(SYSCONFIG_SBD) - res = conf.get(key) - return res - - @staticmethod - def has_sbd_device_already_initialized(dev): - """ - Check if sbd device already initialized - """ - cmd = "sbd -d {} dump".format(dev) - rc, _, _ = ShellUtils().get_stdout_stderr(cmd) - return rc == 0 + logger.info("Got {}SBD configuration".format("" if dev_list else "diskless ")) + service_manager.enable_service(constants.SBD_SERVICE) def clean_up_existing_sbd_resource(): @@ -630,5 +659,40 @@ def clean_up_existing_sbd_resource(): sbd_id_list = xmlutil.CrmMonXmlParser().get_resource_id_list_via_type(SBDManager.SBD_RA) if xmlutil.CrmMonXmlParser().is_resource_started(SBDManager.SBD_RA): for sbd_id in sbd_id_list: + logger.info("Stop sbd resource '%s'(%s)", sbd_id, SBDManager.SBD_RA) utils.ext_cmd("crm resource stop {}".format(sbd_id)) + logger.info("Remove sbd resource '%s'", ';' .join(sbd_id_list)) utils.ext_cmd("crm configure delete {}".format(' '.join(sbd_id_list))) + + +def enable_sbd_on_cluster(): + cluster_nodes = utils.list_cluster_nodes() + service_manager = ServiceManager() + for node in cluster_nodes: + if not service_manager.service_is_enabled(constants.SBD_SERVICE, node): + logger.info("Enable %s on node %s", constants.SBD_SERVICE, node) + service_manager.enable_service(constants.SBD_SERVICE, node) + + +def disable_sbd_from_cluster(): + ''' + Disable SBD from cluster, the process includes: + - stop and remove sbd agent + - disable sbd.service + - adjust cluster attributes + - adjust related timeout values + ''' + clean_up_existing_sbd_resource() + + cluster_nodes = utils.list_cluster_nodes() + service_manager = ServiceManager() + for node in cluster_nodes: + if service_manager.service_is_enabled(constants.SBD_SERVICE, node): + logger.info("Disable %s on node %s", constants.SBD_SERVICE, node) + service_manager.disable_service(constants.SBD_SERVICE, node) + + out = sh.cluster_shell().get_stdout_or_raise_error("stonith_admin -L") + res = re.search("([0-9]+) fence device[s]* found", out) + # after disable sbd.service, check if sbd is the last stonith device + if res and int(res.group(1)) <= 1: + utils.cleanup_stonith_related_properties() diff --git a/crmsh/ui_root.py b/crmsh/ui_root.py index 61a8be882..7e35586b8 100644 --- a/crmsh/ui_root.py +++ b/crmsh/ui_root.py @@ -33,6 +33,7 @@ from . import ui_resource from . import ui_script from . import ui_site +from . import ui_sbd class Root(command.UI): @@ -150,6 +151,10 @@ def do_report(self, context, *args): def do_resource(self): pass + @command.level(ui_sbd.SBD) + def do_sbd(self): + pass + @command.level(ui_script.Script) @command.help('''Cluster scripts Cluster scripts can perform cluster-wide configuration, diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py new file mode 100644 index 000000000..a99ab18cf --- /dev/null +++ b/crmsh/ui_sbd.py @@ -0,0 +1,482 @@ +import logging +import typing +import re +import os + +from crmsh import sbd +from crmsh import watchdog +from crmsh import command +from crmsh import utils +from crmsh import bootstrap +from crmsh import completers +from crmsh import sh +from crmsh import xmlutil +from crmsh import constants +from crmsh.service_manager import ServiceManager +from crmsh.bootstrap import SYSCONFIG_SBD + + +logger = logging.getLogger(__name__) + + +def sbd_devices_completer(completed_list: typing.List[str]) -> typing.List[str]: + ''' + completion for sbd devices + ''' + if not ServiceManager().service_is_active(constants.SBD_SERVICE): + return [] + dev_list = sbd.SBDUtils.get_sbd_device_from_config() + if dev_list: + return [dev for dev in dev_list if dev not in completed_list] + return [] + + +def sbd_configure_completer(completed_list: typing.List[str]) -> typing.List[str]: + ''' + completion for sbd configure command + ''' + service_manager = ServiceManager() + if not service_manager.service_is_active(constants.PCMK_SERVICE): + return [] + sbd_service_is_enabled = service_manager.service_is_enabled(constants.SBD_SERVICE) + dev_list = sbd.SBDUtils.get_sbd_device_from_config() + # Show disk-based sbd configure options + # if there are devices in config or sbd.service is not enabled + is_diskbased = bool(dev_list) or not sbd_service_is_enabled + + parameters_pool = [] + if completed_list[1] == '': + parameters_pool = ["show"] + elif completed_list[1] == "show": + if len(completed_list) == 3: + show_types = SBD.SHOW_TYPES if is_diskbased else SBD.DISKLESS_SHOW_TYPES + return [t for t in show_types if t not in completed_list] + else: + return [] + if completed_list[-1] == "device=": + return [] + + timeout_types = SBD.TIMEOUT_TYPES if is_diskbased else SBD.DISKLESS_TIMEOUT_TYPES + parameters_pool.extend([f"{t}-timeout=" for t in timeout_types]) + parameters_pool.append("watchdog-device=") + parameters_pool = [ + p + for p in parameters_pool + if not any(c.startswith(p) for c in completed_list) + ] + + if is_diskbased: + dev_count = sum(1 for c in completed_list if c.startswith("device=")) + if dev_count < sbd.SBDManager.SBD_DEVICE_MAX: + parameters_pool.append("device=") + + return parameters_pool + + +class SBD(command.UI): + ''' + Class for sbd sub-level + + Includes commands: + - sbd configure + - sbd remove + - sbd status + ''' + name = "sbd" + TIMEOUT_TYPES = ("watchdog", "allocate", "loop", "msgwait") + DISKLESS_TIMEOUT_TYPES = ("watchdog",) + SHOW_TYPES = ("disk_metadata", "sysconfig", "property") + DISKLESS_SHOW_TYPES = ("sysconfig", "property") + RESTART_INFO = "Requires to restart cluster service to take effect" + PCMK_ATTRS = ( + "have-watchdog", + "stonith-timeout", + "stonith-watchdog-timeout", + "stonith-enabled", + "priority-fencing-delay", + "pcmk_delay_max" + ) + PARSE_RE = re.compile( + # Match "device" key with any value, including empty + r'(device)=("[^"]*"|[\w/\d;]*)' + # Match other keys with non-empty values, capturing possible suffix + r'|(\w+)(?:-(\w+))?=("[^"]+"|[\w/\d;]+)' + # Match standalone device path + r'|(/dev/[\w\d]+)' + ) + + class SyntaxError(Exception): + pass + + def __init__(self): + self.device_list_from_config: list[str] = None + self.device_meta_dict_runtime: dict[str, int] = None + self.watchdog_timeout_from_config: int = None + self.watchdog_device_from_config: str = None + self.service_manager: ServiceManager = None + self.cluster_shell: sh.cluster_shell = None + self.cluster_nodes: list[str] = None + self.crm_mon_xml_parser: xmlutil.CrmMonXmlParser = None + + command.UI.__init__(self) + + def _load_attributes(self): + self.device_list_from_config = sbd.SBDUtils.get_sbd_device_from_config() + self.device_meta_dict_runtime = {} + if self.device_list_from_config: + self.device_meta_dict_runtime = sbd.SBDUtils.get_sbd_device_metadata(self.device_list_from_config[0], timeout_only=True) + try: + self.watchdog_timeout_from_config = sbd.SBDTimeout.get_sbd_watchdog_timeout() + except: + self.watchdog_timeout_from_config = None + self.watchdog_device_from_config = watchdog.Watchdog.get_watchdog_device_from_sbd_config() + + self.service_manager = ServiceManager() + self.cluster_shell = sh.cluster_shell() + self.cluster_nodes = utils.list_cluster_nodes() or [utils.this_node()] + self.crm_mon_xml_parser = xmlutil.CrmMonXmlParser() + + def requires(self) -> bool: + ''' + Requirements check when entering sbd sub-level + ''' + if not utils.package_is_installed("sbd"): + logger.error("sbd is not installed") + return False + return True + + @property + def configure_usage(self) -> str: + ''' + Build usage string for sbd configure command, + including disk-based and diskless sbd cases + ''' + def build_timeout_usage_str(timeout_types: tuple[str]) -> str: + return " ".join([f"[{t}-timeout=]" for t in timeout_types]) + timeout_usage_str = build_timeout_usage_str(self.TIMEOUT_TYPES) + timeout_usage_str_diskless = build_timeout_usage_str(self.DISKLESS_TIMEOUT_TYPES) + show_usage_str = f"[{'|'.join(self.SHOW_TYPES)}]" + show_usage_str_diskless = f"[{'|'.join(self.DISKLESS_SHOW_TYPES)}]" + return ("Usage for disk-based SBD:\n" + f"crm sbd configure show {show_usage_str}\n" + f"crm sbd configure [device=]... {timeout_usage_str} [watchdog-device=]\n\n" + "Usage for diskless SBD:\n" + f"crm sbd configure show {show_usage_str_diskless}\n" + f"crm sbd configure device=\"\" {timeout_usage_str_diskless} [watchdog-device=]\n") + + @staticmethod + def _show_sysconfig() -> None: + ''' + Show pure content of /etc/sysconfig/sbd + ''' + with open(SYSCONFIG_SBD) as f: + content_list = [line.strip() for line in f.readlines() + if not line.startswith("#") + and line.strip()] + for line in content_list: + print(line) + + def _show_disk_metadata(self) -> None: + ''' + Show sbd disk metadata for each configured device + ''' + for dev in self.device_list_from_config: + print(self.cluster_shell.get_stdout_or_raise_error(f"sbd -d {dev} dump")) + print() + + def _show_property(self) -> None: + ''' + Show sbd-related properties from cluster and systemd + ''' + if self.service_manager.service_is_active(constants.PCMK_SERVICE): + cmd = "crm configure show" + else: # static case + cib_path = os.getenv("CIB_file", constants.CIB_RAW_FILE) + if not os.path.exists(cib_path): + return + cmd = f"CIB_file={cib_path} crm configure show" + out = self.cluster_shell.get_stdout_or_raise_error(cmd) + + regex = f"({'|'.join(self.PCMK_ATTRS)})=([^\s]+)" + matches = re.findall(regex, out) + for match in matches: + print(f"{match[0]}={match[1]}") + + systemd_start_timeout = sbd.SBDTimeout.get_sbd_systemd_start_timeout() + print(f"TimeoutStartUSec={systemd_start_timeout}") + + def _configure_show(self, args) -> None: + if len(args) > 2: + raise self.SyntaxError("Invalid argument") + elif len(args) == 2: + match args[1]: + case "disk_metadata": + self._show_disk_metadata() + case "sysconfig": + SBD._show_sysconfig() + case "property": + self._show_property() + case _: + raise self.SyntaxError(f"Unknown argument: {args[1]}") + else: + self._show_disk_metadata() + if self.device_list_from_config: + print() + SBD._show_sysconfig() + print() + self._show_property() + + def _parse_args(self, args: typing.List[str]) -> dict[str, int|str|list[str]]: + ''' + Parse arguments and verify them + + Possible arguments format like: + device="/dev/sdb5;/dev/sda6" + device="" watchdog-timeout=10 + /dev/sda5 watchdog-timeout=10 watchdog-device=/dev/watchdog + device=/dev/sdb5 device=/dev/sda6 watchdog-timeout=10 msgwait-timeout=20 + ''' + parameter_dict = {"device-list": []} + + for arg in args: + match = self.PARSE_RE.match(arg) + if not match: + raise self.SyntaxError(f"Invalid argument: {arg}") + device_key, device_value, key, suffix, value, device_path = match.groups() + + # device= parameter + if device_key: + if device_value: + parameter_dict.setdefault("device-list", []).extend(device_value.split(";")) + # explicitly set empty value, stands for diskless sbd + elif not parameter_dict.get("device-list"): + parameter_dict.pop("device-list", None) + # standalone device parameter + elif device_path: + parameter_dict.setdefault("device-list", []).append(device_path) + # timeout related parameters + elif key in self.TIMEOUT_TYPES and suffix and suffix == "timeout": + if not value.isdigit(): + raise self.SyntaxError(f"Invalid timeout value: {value}") + parameter_dict[key] = int(value) + # watchdog device parameter + elif key == "watchdog" and suffix == "device": + parameter_dict["watchdog-device"] = value + else: + raise self.SyntaxError(f"Unknown argument: {arg}") + + watchdog_device = parameter_dict.get("watchdog-device") + parameter_dict["watchdog-device"] = watchdog.Watchdog.get_watchdog_device(watchdog_device) + + logger.debug("Parsed arguments: %s", parameter_dict) + return parameter_dict + + @staticmethod + def _adjust_timeout_dict(timeout_dict: dict, diskless: bool = False) -> dict: + watchdog_timeout = timeout_dict.get("watchdog") + if not watchdog_timeout: + watchdog_timeout, _ = sbd.SBDTimeout.get_advised_sbd_timeout(diskless) + logger.info("No watchdog timeout specified, use advised value: %s", watchdog_timeout) + timeout_dict["watchdog"] = watchdog_timeout + + if diskless: + return timeout_dict + + msgwait_timeout = timeout_dict.get("msgwait") + if not msgwait_timeout: + msgwait_timeout = 2*watchdog_timeout + logger.info("No msgwait timeout specified, use 2*watchdog timeout: %s", msgwait_timeout) + timeout_dict["msgwait"] = msgwait_timeout + + if msgwait_timeout < 2*watchdog_timeout: + logger.warning("It's recommended to set msgwait timeout >= 2*watchdog timeout") + + return timeout_dict + + def _configure_diskbase(self, parameter_dict: dict): + ''' + Configure disk-based SBD based on input parameters and runtime config + ''' + if not self.device_list_from_config: + self.watchdog_timeout_from_config = None + self.watchdog_device_from_config = None + + update_dict = {} + device_list = parameter_dict.get("device-list", []) + if not device_list and not self.device_list_from_config: + raise self.SyntaxError("No device specified") + if len(device_list) > len(set(device_list)): + raise self.SyntaxError("Duplicate device") + watchdog_device = parameter_dict.get("watchdog-device") + if watchdog_device != self.watchdog_device_from_config: + update_dict["SBD_WATCHDOG_DEV"] = watchdog_device + timeout_dict = {k: v for k, v in parameter_dict.items() if k in self.TIMEOUT_TYPES} + + all_device_list = list( + dict.fromkeys(self.device_list_from_config + device_list) + ) + sbd.SBDUtils.verify_sbd_device(all_device_list) + + new_device_list = list( + set(device_list) - set(self.device_list_from_config) + ) + no_overwrite_dev_map : dict[str, bool] = { + dev: sbd.SBDUtils.no_overwrite_device_check(dev) for dev in new_device_list + } + if new_device_list: + update_dict["SBD_DEVICE"] = ";".join(all_device_list) + + device_list_to_init = [] + # initialize new devices only if no timeout parameter specified or timeout parameter is already in runtime config + if not timeout_dict or utils.is_subdict(timeout_dict, self.device_meta_dict_runtime): + device_list_to_init = new_device_list + # initialize all devices + else: + device_list_to_init = all_device_list + + # merge runtime timeout dict with new timeout dict + timeout_dict = self.device_meta_dict_runtime | timeout_dict + # adjust watchdog and msgwait timeout + timeout_dict = self._adjust_timeout_dict(timeout_dict) + watchdog_timeout = timeout_dict.get("watchdog") + if watchdog_timeout != self.watchdog_timeout_from_config: + update_dict["SBD_WATCHDOG_TIMEOUT"] = str(watchdog_timeout) + + sbd_manager = sbd.SBDManager( + device_list_to_init=device_list_to_init, + timeout_dict=timeout_dict, + update_dict=update_dict, + no_overwrite_dev_map=no_overwrite_dev_map, + new_config=False if self.device_list_from_config else True + ) + sbd_manager.init_and_deploy_sbd() + + def _configure_diskless(self, parameter_dict: dict): + ''' + Configure diskless SBD based on input parameters and runtime config + ''' + if self.device_list_from_config: + self.watchdog_timeout_from_config = None + self.watchdog_device_from_config = None + + update_dict = {} + parameter_dict = self._adjust_timeout_dict(parameter_dict, diskless=True) + watchdog_timeout = parameter_dict.get("watchdog") + if watchdog_timeout and watchdog_timeout != self.watchdog_timeout_from_config: + update_dict["SBD_WATCHDOG_TIMEOUT"] = str(watchdog_timeout) + watchdog_device = parameter_dict.get("watchdog-device") + if watchdog_device != self.watchdog_device_from_config: + update_dict["SBD_WATCHDOG_DEV"] = watchdog_device + + sbd_manager = sbd.SBDManager( + update_dict=update_dict, + diskless_sbd=True, + new_config=True if self.device_list_from_config else False + ) + sbd_manager.init_and_deploy_sbd() + + @command.completers_repeating(sbd_configure_completer) + def do_configure(self, context, *args) -> bool: + ''' + Implement sbd configure command + ''' + self._load_attributes() + + try: + if not args: + raise self.SyntaxError("No argument") + + if args[0] == "show": + self._configure_show(args) + return True + + if not self.service_manager.service_is_active(constants.PCMK_SERVICE): + logger.error("%s is not active", constants.PCMK_SERVICE) + return False + + parameter_dict = self._parse_args(args) + # disk-based sbd case + if "device-list" in parameter_dict: + return self._configure_diskbase(parameter_dict) + # diskless sbd case + else: + return self._configure_diskless(parameter_dict) + + except self.SyntaxError as e: + logger.error(str(e)) + print(self.configure_usage) + return False + + @command.completers_repeating(sbd_devices_completer) + def do_remove(self, context, *args) -> bool: + ''' + Implement sbd remove command + ''' + self._load_attributes() + + if not self.service_manager.service_is_active(constants.SBD_SERVICE): + logger.error("%s is not active", constants.SBD_SERVICE) + return False + + parameter_dict = self._parse_args(args) + dev_list = parameter_dict.get("device-list", []) + if dev_list: + if not self.device_list_from_config: + logger.error("No sbd device found in config") + return False + for dev in dev_list: + if dev not in self.device_list_from_config: + logger.error("Device %s is not in config", dev) + return False + changed_dev_list = set(self.device_list_from_config) - set(dev_list) + # remove part of devices from config + if changed_dev_list: + logger.info("Remove '%s' from %s", ";".join(dev_list), SYSCONFIG_SBD) + sbd.SBDManager.update_sbd_configuration({"SBD_DEVICE": ";".join(changed_dev_list)}) + # remove all devices, equivalent to stop sbd.service + else: + sbd.disable_sbd_from_cluster() + else: + sbd.disable_sbd_from_cluster() + + logger.info(self.RESTART_INFO) + return True + + def do_status(self, context) -> bool: + ''' + Implement sbd status command + ''' + self._load_attributes() + + print(f"{constants.SBD_SERVICE} status: (active|enabled|since)") + for node in self.cluster_nodes: + is_active = self.service_manager.service_is_active(constants.SBD_SERVICE, node) + is_active_str = "YES" if is_active else "NO" + is_enabled = self.service_manager.service_is_enabled(constants.SBD_SERVICE, node) + is_enabled_str = "YES" if is_enabled else "NO" + systemd_property = "ActiveEnterTimestamp" if is_active else "ActiveExitTimestamp" + since_str_prefix = "active since" if is_active else "disactive since" + systemctl_show_cmd = f"systemctl show {constants.SBD_SERVICE} --property={systemd_property} --value" + since = self.cluster_shell.get_stdout_or_raise_error(systemctl_show_cmd, node) or "N/A" + print(f"{node}: {is_active_str:<4}|{is_enabled_str:<4}|{since_str_prefix}: {since}") + print() + + print("watchdog info: (device|driver|kernel timeout)") + watchdog_sbd_re = "\[[0-9]+\] (/dev/.*)\nIdentity: Busy: .*sbd.*\nDriver: (.*)" + for node in self.cluster_nodes: + out = self.cluster_shell.get_stdout_or_raise_error("sbd query-watchdog", node) + res = re.search(watchdog_sbd_re, out) + if res: + device, driver = res.groups() + kernel_timeout = self.cluster_shell.get_stdout_or_raise_error("cat /proc/sys/kernel/watchdog_thresh", node) + print(f"{node}: {device}|{driver}|{kernel_timeout}") + else: + logger.error("Failed to get watchdog info from %s", node) + print() + + if self.crm_mon_xml_parser.is_resource_configured(sbd.SBDManager.SBD_RA): + print("fence_sbd status: ") + sbd_id_list = self.crm_mon_xml_parser.get_resource_id_list_via_type(sbd.SBDManager.SBD_RA) + for sbd_id in sbd_id_list: + out = self.cluster_shell.get_stdout_or_raise_error(f"crm resource status {sbd_id}") + print(out) diff --git a/crmsh/utils.py b/crmsh/utils.py index 893e5d14f..e35235e34 100644 --- a/crmsh/utils.py +++ b/crmsh/utils.py @@ -2518,7 +2518,7 @@ def has_stonith_running(): from . import sbd out = sh.cluster_shell().get_stdout_or_raise_error("stonith_admin -L") has_stonith_device = re.search("[1-9]+ fence device[s]* found", out) is not None - using_diskless_sbd = sbd.SBDManager.is_using_diskless_sbd() + using_diskless_sbd = sbd.SBDUtils.is_using_diskless_sbd() return has_stonith_device or using_diskless_sbd @@ -2778,13 +2778,15 @@ def get_pcmk_delay_max(two_node_without_qdevice=False): return 0 -def get_property(name, property_type="crm_config", peer=None): +def get_property(name, property_type="crm_config", peer=None, get_default=True): """ Get cluster properties "property_type" can be crm_config|rsc_defaults|op_defaults + "get_default" is used to get the default value from cluster metadata, + when it is False, the property value will be got from cib """ - if property_type == "crm_config": + if property_type == "crm_config" and get_default: cib_path = os.getenv('CIB_file', constants.CIB_RAW_FILE) cmd = "CIB_file={} sudo --preserve-env=CIB_file crm configure get_property {}".format(cib_path, name) else: @@ -3166,7 +3168,7 @@ def ssh_command(): def load_cib_file_env(): - if options.regression_tests or ServiceManager().service_is_active("pacemaker.service"): + if options.regression_tests or ServiceManager().service_is_active(constants.PCMK_SERVICE): return cib_file = os.environ.setdefault('CIB_file', constants.CIB_RAW_FILE) logger.warning("Cluster is not running, loading the CIB file from %s", cib_file) @@ -3208,4 +3210,19 @@ def fuzzy_match(rx): if m: return m return None + + +def cleanup_stonith_related_properties(): + for p in ("stonith-watchdog-timeout", "stonith-timeout", "priority-fencing-delay"): + if get_property(p, get_default=False): + delete_property(p) + if get_property("stonith-enabled") == "true": + set_property("stonith-enabled", "false") + + +def is_subdict(sub_dict, main_dict): + """ + Check if sub_dict is a sub-dictionary of main_dict + """ + return all(item in main_dict.items() for item in sub_dict.items()) # vim:ts=4:sw=4:et: diff --git a/crmsh/watchdog.py b/crmsh/watchdog.py index 6d0d2cff4..00e0f60a5 100644 --- a/crmsh/watchdog.py +++ b/crmsh/watchdog.py @@ -27,7 +27,7 @@ def watchdog_device_name(self): return self._watchdog_device_name @staticmethod - def _verify_watchdog_device(dev, ignore_error=False): + def verify_watchdog_device(dev, ignore_error=False): """ Use wdctl to verify watchdog device """ @@ -48,7 +48,7 @@ def _load_watchdog_driver(driver): invoke("systemctl restart systemd-modules-load") @staticmethod - def _get_watchdog_device_from_sbd_config(): + def get_watchdog_device_from_sbd_config(): """ Try to get watchdog device name from sbd config file """ @@ -81,7 +81,7 @@ def _get_device_through_driver(self, driver_name): Get watchdog device name which has driver_name """ for device, driver in self._watchdog_info_dict.items(): - if driver == driver_name and self._verify_watchdog_device(device): + if driver == driver_name and self.verify_watchdog_device(device): return device return None @@ -108,7 +108,7 @@ def _get_first_unused_device(self): Get first unused watchdog device name """ for dev in self._watchdog_info_dict: - if self._verify_watchdog_device(dev, ignore_error=True): + if self.verify_watchdog_device(dev, ignore_error=True): return dev return None @@ -120,8 +120,8 @@ def _set_input(self): 3. Set the self._input as softdog """ if not self._input: - dev = self._get_watchdog_device_from_sbd_config() - if dev and self._verify_watchdog_device(dev, ignore_error=True): + dev = self.get_watchdog_device_from_sbd_config() + if dev and self.verify_watchdog_device(dev, ignore_error=True): self._input = dev return first_unused = self._get_first_unused_device() @@ -131,7 +131,7 @@ def _valid_device(self, dev): """ Is an unused watchdog device """ - if dev in self._watchdog_info_dict and self._verify_watchdog_device(dev): + if dev in self._watchdog_info_dict and self.verify_watchdog_device(dev): return True return False @@ -142,7 +142,7 @@ def join_watchdog(self): """ self._set_watchdog_info() - res = self._get_watchdog_device_from_sbd_config() + res = self.get_watchdog_device_from_sbd_config() if not res: utils.fatal("Failed to get watchdog device from {}".format(SYSCONFIG_SBD)) self._input = res @@ -177,3 +177,9 @@ def init_watchdog(self): if res: self._watchdog_device_name = res return + + @classmethod + def get_watchdog_device(cls, dev_or_driver=None): + w = cls(_input=dev_or_driver) + w.init_watchdog() + return w.watchdog_device_name From ef7087ccb8969c3b6516dfe96af1c766b9a5bbab Mon Sep 17 00:00:00 2001 From: xin liang Date: Tue, 13 Aug 2024 21:44:37 +0800 Subject: [PATCH 03/17] Dev: behave: Adjust functional test for previous changes --- test/features/bootstrap_sbd_delay.feature | 12 ++++++++---- test/features/bootstrap_sbd_normal.feature | 4 +++- test/features/steps/step_implementation.py | 10 +++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/test/features/bootstrap_sbd_delay.feature b/test/features/bootstrap_sbd_delay.feature index b8d1970c4..3c3250c75 100644 --- a/test/features/bootstrap_sbd_delay.feature +++ b/test/features/bootstrap_sbd_delay.feature @@ -18,8 +18,10 @@ Feature: configure sbd delay start correctly And SBD option "SBD_DELAY_START" value is "no" And SBD option "SBD_WATCHDOG_TIMEOUT" value is "15" And SBD option "msgwait" value for "/dev/sda1" is "30" - # calculated and set by sbd RA - And Cluster property "stonith-timeout" is "43" + # original value is 43, which is calculated by external/sbd RA + # now fence_sbd doesn't calculate it, so this value is the default one + # from pacemaker + And Cluster property "stonith-timeout" is "60" And Parameter "pcmk_delay_max" not configured in "stonith-sbd" Given Has disk "/dev/sda1" on "hanode2" @@ -112,8 +114,10 @@ Feature: configure sbd delay start correctly And SBD option "SBD_DELAY_START" value is "no" And SBD option "SBD_WATCHDOG_TIMEOUT" value is "60" And SBD option "msgwait" value for "/dev/sda1" is "120" - # calculated and set by sbd RA - And Cluster property "stonith-timeout" is "172" + # original value is 172, which is calculated by external/sbd RA + # now fence_sbd doesn't calculate it, so this value is the default one + # from pacemaker + And Cluster property "stonith-timeout" is "60" And Parameter "pcmk_delay_max" not configured in "stonith-sbd" Given Has disk "/dev/sda1" on "hanode2" diff --git a/test/features/bootstrap_sbd_normal.feature b/test/features/bootstrap_sbd_normal.feature index fe73377f5..d5b2e4561 100644 --- a/test/features/bootstrap_sbd_normal.feature +++ b/test/features/bootstrap_sbd_normal.feature @@ -139,7 +139,7 @@ Feature: crmsh bootstrap sbd management And Online nodes are "hanode1 hanode2" When Run "crm configure primitive d Dummy op monitor interval=3s" on "hanode1" When Run "crm cluster init sbd -s /dev/sda1 -y" on "hanode1" - Then Expected "WARNING: To start sbd.service, need to restart cluster service manually on each node" in stderr + Then Expected "WARNING: Resource is running, need to restart cluster service manually on each node" in stderr Then Service "sbd" is "stopped" on "hanode1" And Service "sbd" is "stopped" on "hanode2" When Run "crm cluster restart" on "hanode1" @@ -263,6 +263,8 @@ Feature: crmsh bootstrap sbd management And Resource "stonith-sbd" type "fence_sbd" is "Started" And Run "ps -ef|grep -v grep|grep 'watcher: /dev/sda1 '" OK + When Try "crm cluster init sbd -s /dev/sda2 -y" + Then Except "ERROR: cluster.init: Can't configure stage sbd: sbd.service already running! Please use crm option '-F' if need to redeploy" When Run "crm -F cluster init sbd -s /dev/sda2 -y" on "hanode1" Then Service "sbd" is "started" on "hanode1" And Service "sbd" is "started" on "hanode2" diff --git a/test/features/steps/step_implementation.py b/test/features/steps/step_implementation.py index b1c4c2a8d..47b138393 100644 --- a/test/features/steps/step_implementation.py +++ b/test/features/steps/step_implementation.py @@ -439,7 +439,7 @@ def step_impl(context, res_id, node): @then('SBD option "{key}" value is "{value}"') def step_impl(context, key, value): - res = sbd.SBDManager.get_sbd_value_from_config(key) + res = sbd.SBDUtils.get_sbd_value_from_config(key) assert_eq(value, res) @@ -453,27 +453,27 @@ def step_impl(context, key, dev, value): def step_impl(context, key, value): res = crmutils.get_property(key) assert res is not None - assert_eq(value, str(res)) + assert_eq(value.strip('s'), str(res).strip('s')) @then('Property "{key}" in "{type}" is "{value}"') def step_impl(context, key, type, value): res = crmutils.get_property(key, type) assert res is not None - assert_eq(value, str(res)) + assert_eq(value.strip('s'), str(res).strip('s')) @then('Parameter "{param_name}" not configured in "{res_id}"') def step_impl(context, param_name, res_id): _, out, _ = run_command(context, "crm configure show {}".format(res_id)) - result = re.search("params {}=".format(param_name), out) + result = re.search("params .*{}=".format(param_name), out) assert result is None @then('Parameter "{param_name}" configured in "{res_id}"') def step_impl(context, param_name, res_id): _, out, _ = run_command(context, "crm configure show {}".format(res_id)) - result = re.search("params {}=".format(param_name), out) + result = re.search("params .*{}=".format(param_name), out) assert result is not None From d7c929964ce1b07d520dea194ddfdc37926e75a9 Mon Sep 17 00:00:00 2001 From: xin liang Date: Tue, 20 Aug 2024 07:25:16 +0800 Subject: [PATCH 04/17] Dev: doc: Add help info for crm sbd sublevel --- doc/crm.8.adoc | 67 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/doc/crm.8.adoc b/doc/crm.8.adoc index fd62df771..e887259fa 100644 --- a/doc/crm.8.adoc +++ b/doc/crm.8.adoc @@ -1,5 +1,5 @@ :man source: crm -:man version: 4.6.0 +:man version: 5.0.0 :man manual: crmsh documentation crm(8) @@ -2104,6 +2104,71 @@ Example: utilization xen1 set memory 4096 ............... +[[cmdhelp.sbd,SBD management]] +=== `sbd` - SBD management + +This level is for managing the SBD (STONITH Block Device) daemon. + +[[cmdhelp.sbd.configure,configure SBD]] +==== `configure` + +Configure the SBD daemon for both disk-based and disk-less mode. + +Main functionailities include: +- Show configured disk metadata +- Show contents of /etc/sysconfig/sbd +- Show SBD related cluster properties +- Newly setup SBD configuration on a running cluster +- Update the existing parameters +- Add more devices to the existing disk-based SBD configuration + +For more details on SBD and related parameters, please see man sbd(8). + +Usage: +............... +# For disk-based SBD +crm sbd configure show [disk_metadata|sysconfig|property] +crm sbd configure [device=]... [watchdog-device=] [watchdog-timeout=] [allocate-timeout=] [loop-timeout=] [msgwait-timeout=] + +# For disk-less SBD +crm sbd configure show [sysconfig|property] +crm sbd configure device="" [watchdog-device=] [watchdog-timeout=] +............... + +example: +............... +configure show +configure show disk_metadata +configure show sysconfig +configure show property +configure device="/dev/sdb1;/dev/sdb2" +configure device=/dev/sdb1 device=/dev/sdb2 +configure device=/dev/sdb1 watchdog-timeout=30 msgwait-timeout=60 +configure device="" watchdog-timeout=30 +............... + +[[cmdhelp.sbd.status,show SBD status]] +==== `status` + +Show the status of the SBD daemon. + +Usage: +............... +status +............... + +[[cmdhelp.sbd.remove,remove SBD configuration]] +==== `remove` + +Remove part of devices from the SBD configuration, or remove SBD +service from the cluster. + +Usage: +............... +remove +remove [ ...] +............... + [[cmdhelp.node,Node management]] === `node` - Node management From 0926c60cc6d2ca09673ca50242199eeb7bf683d8 Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 11 Sep 2024 10:11:42 +0800 Subject: [PATCH 05/17] Dev: ui_sbd: Add property/sysconfig section header for sbd configure show --- crmsh/ui_sbd.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index a99ab18cf..8a45baaab 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -173,6 +173,8 @@ def _show_sysconfig() -> None: content_list = [line.strip() for line in f.readlines() if not line.startswith("#") and line.strip()] + if content_list: + logger.info("crm sbd configure show sysconfig") for line in content_list: print(line) @@ -180,6 +182,8 @@ def _show_disk_metadata(self) -> None: ''' Show sbd disk metadata for each configured device ''' + if self.device_list_from_config: + logger.info("crm sbd configure show disk_metadata") for dev in self.device_list_from_config: print(self.cluster_shell.get_stdout_or_raise_error(f"sbd -d {dev} dump")) print() @@ -197,11 +201,14 @@ def _show_property(self) -> None: cmd = f"CIB_file={cib_path} crm configure show" out = self.cluster_shell.get_stdout_or_raise_error(cmd) + logger.info("crm sbd configure show property") regex = f"({'|'.join(self.PCMK_ATTRS)})=([^\s]+)" matches = re.findall(regex, out) for match in matches: print(f"{match[0]}={match[1]}") + print() + logger.info("systemctl show -p TimeoutStartUSec sbd --value") systemd_start_timeout = sbd.SBDTimeout.get_sbd_systemd_start_timeout() print(f"TimeoutStartUSec={systemd_start_timeout}") From ef30a96a8f0a47b5ab35b30958dace1a50bc1d5f Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 11 Sep 2024 10:16:10 +0800 Subject: [PATCH 06/17] Dev: ui_sbd: No need to consider static case when calling crm configure show After PR#1540 got merged, crmsh will load CIB_file env before calling above readonly command. --- crmsh/ui_sbd.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index 8a45baaab..849287f3a 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -192,14 +192,7 @@ def _show_property(self) -> None: ''' Show sbd-related properties from cluster and systemd ''' - if self.service_manager.service_is_active(constants.PCMK_SERVICE): - cmd = "crm configure show" - else: # static case - cib_path = os.getenv("CIB_file", constants.CIB_RAW_FILE) - if not os.path.exists(cib_path): - return - cmd = f"CIB_file={cib_path} crm configure show" - out = self.cluster_shell.get_stdout_or_raise_error(cmd) + out = self.cluster_shell.get_stdout_or_raise_error("crm configure show") logger.info("crm sbd configure show property") regex = f"({'|'.join(self.PCMK_ATTRS)})=([^\s]+)" From 15f8384566a2f3678c23aa9fa63d8855a7e3dd70 Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 11 Sep 2024 10:46:19 +0800 Subject: [PATCH 07/17] Dev: doc: Upadate crm.8.adoc for SBD help text --- doc/crm.8.adoc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/crm.8.adoc b/doc/crm.8.adoc index e887259fa..6d56e9775 100644 --- a/doc/crm.8.adoc +++ b/doc/crm.8.adoc @@ -2107,7 +2107,10 @@ utilization xen1 set memory 4096 [[cmdhelp.sbd,SBD management]] === `sbd` - SBD management -This level is for managing the SBD (STONITH Block Device) daemon. +This level displays the real-time SBD status and the static SBD configuration. +Additionally, it manages the configuration file for both disk-based and diskless SBD scenarios, +as well as the on-disk metadata for the disk-based scenario. +Currently, SBD management requires a running cluster. [[cmdhelp.sbd.configure,configure SBD]] ==== `configure` @@ -2150,7 +2153,9 @@ configure device="" watchdog-timeout=30 [[cmdhelp.sbd.status,show SBD status]] ==== `status` -Show the status of the SBD daemon. +Show the runtime status of the SBD daemon and +the other information of those SBD related components, +ie. watchdog, fence agent. Usage: ............... @@ -2166,7 +2171,7 @@ service from the cluster. Usage: ............... remove -remove [ ...] +remove [ ...] ............... [[cmdhelp.node,Node management]] From 83dabf9d2aff88009110a4ead620aade2a33d902 Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 11 Sep 2024 14:31:08 +0800 Subject: [PATCH 08/17] Dev: ui_sbd: Catch both stderr and stdout for crm resource status since crm_resource command will direct message to stderr when resource is not running --- crmsh/ui_sbd.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index 849287f3a..9832aa4a0 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -478,5 +478,8 @@ def do_status(self, context) -> bool: print("fence_sbd status: ") sbd_id_list = self.crm_mon_xml_parser.get_resource_id_list_via_type(sbd.SBDManager.SBD_RA) for sbd_id in sbd_id_list: - out = self.cluster_shell.get_stdout_or_raise_error(f"crm resource status {sbd_id}") - print(out) + rc, out, err = self.cluster_shell.get_rc_stdout_stderr_without_input(None, f"crm resource status {sbd_id}") + if out: + print(out) + if err: + print(err) From 4c63398116e8ecef33bd52ec516a7fd8115a0942 Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 11 Sep 2024 15:14:23 +0800 Subject: [PATCH 09/17] Dev: ui_sbd: Update regex for parsing SBD device by partlabel --- crmsh/ui_sbd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index 9832aa4a0..06de2a645 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -96,13 +96,15 @@ class SBD(command.UI): "priority-fencing-delay", "pcmk_delay_max" ) + # a commom character class for matching device path + dev_char_class = r'[\w/\d;\-:.]' PARSE_RE = re.compile( # Match "device" key with any value, including empty - r'(device)=("[^"]*"|[\w/\d;]*)' + fr'(device)=("[^"]*"|{dev_char_class}*)' # Match other keys with non-empty values, capturing possible suffix r'|(\w+)(?:-(\w+))?=("[^"]+"|[\w/\d;]+)' # Match standalone device path - r'|(/dev/[\w\d]+)' + fr'|(/dev/{dev_char_class}+)' ) class SyntaxError(Exception): From 1b6586437f1c57f07ee772fa1d8f6c61707f1167 Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 11 Sep 2024 15:33:44 +0800 Subject: [PATCH 10/17] Dev: ui_sbd: Clean up existing fence_sbd resource before configure diskless SBD --- crmsh/ui_sbd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index 06de2a645..fe5648aed 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -360,6 +360,7 @@ def _configure_diskless(self, parameter_dict: dict): if self.device_list_from_config: self.watchdog_timeout_from_config = None self.watchdog_device_from_config = None + sbd.clean_up_existing_sbd_resource() update_dict = {} parameter_dict = self._adjust_timeout_dict(parameter_dict, diskless=True) From 52de8332641cff4fe193ceb8a48e88047ebd9063 Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 11 Sep 2024 22:51:13 +0800 Subject: [PATCH 11/17] Dev: ui_sbd: Minor changes to the code --- crmsh/constants.py | 2 ++ crmsh/sbd.py | 3 +-- crmsh/ui_sbd.py | 17 +++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/crmsh/constants.py b/crmsh/constants.py index 89686aa83..0106e29be 100644 --- a/crmsh/constants.py +++ b/crmsh/constants.py @@ -453,4 +453,6 @@ PCMK_SERVICE = "pacemaker.service" SBD_SERVICE = "sbd.service" + +SHOW_SBD_START_TIMEOUT_CMD = "systemctl show -p TimeoutStartUSec sbd.service --value" # vim:ts=4:sw=4:et: diff --git a/crmsh/sbd.py b/crmsh/sbd.py index 4b732b1ad..d68fa1317 100644 --- a/crmsh/sbd.py +++ b/crmsh/sbd.py @@ -323,8 +323,7 @@ def is_sbd_delay_start(): @staticmethod def get_sbd_systemd_start_timeout() -> int: - cmd = "systemctl show -p TimeoutStartUSec sbd --value" - out = sh.cluster_shell().get_stdout_or_raise_error(cmd) + out = sh.cluster_shell().get_stdout_or_raise_error(constants.SHOW_SBD_START_TIMEOUT_CMD) return utils.get_systemd_timeout_start_in_sec(out) def adjust_systemd_start_timeout(self): diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index fe5648aed..c7438d1ce 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -129,7 +129,7 @@ def _load_attributes(self): self.device_meta_dict_runtime = sbd.SBDUtils.get_sbd_device_metadata(self.device_list_from_config[0], timeout_only=True) try: self.watchdog_timeout_from_config = sbd.SBDTimeout.get_sbd_watchdog_timeout() - except: + except Exception: self.watchdog_timeout_from_config = None self.watchdog_device_from_config = watchdog.Watchdog.get_watchdog_device_from_sbd_config() @@ -153,7 +153,7 @@ def configure_usage(self) -> str: Build usage string for sbd configure command, including disk-based and diskless sbd cases ''' - def build_timeout_usage_str(timeout_types: tuple[str]) -> str: + def build_timeout_usage_str(timeout_types: tuple[str, ...]) -> str: return " ".join([f"[{t}-timeout=]" for t in timeout_types]) timeout_usage_str = build_timeout_usage_str(self.TIMEOUT_TYPES) timeout_usage_str_diskless = build_timeout_usage_str(self.DISKLESS_TIMEOUT_TYPES) @@ -197,13 +197,13 @@ def _show_property(self) -> None: out = self.cluster_shell.get_stdout_or_raise_error("crm configure show") logger.info("crm sbd configure show property") - regex = f"({'|'.join(self.PCMK_ATTRS)})=([^\s]+)" + regex = f"({'|'.join(self.PCMK_ATTRS)})=(\\S+)" matches = re.findall(regex, out) for match in matches: print(f"{match[0]}={match[1]}") print() - logger.info("systemctl show -p TimeoutStartUSec sbd --value") + logger.info('%s', constants.SHOW_SBD_START_TIMEOUT_CMD) systemd_start_timeout = sbd.SBDTimeout.get_sbd_systemd_start_timeout() print(f"TimeoutStartUSec={systemd_start_timeout}") @@ -228,7 +228,7 @@ def _configure_show(self, args) -> None: print() self._show_property() - def _parse_args(self, args: typing.List[str]) -> dict[str, int|str|list[str]]: + def _parse_args(self, args: tuple[str, ...]) -> dict[str, int|str|list[str]]: ''' Parse arguments and verify them @@ -400,13 +400,14 @@ def do_configure(self, context, *args) -> bool: parameter_dict = self._parse_args(args) # disk-based sbd case if "device-list" in parameter_dict: - return self._configure_diskbase(parameter_dict) + self._configure_diskbase(parameter_dict) # diskless sbd case else: - return self._configure_diskless(parameter_dict) + self._configure_diskless(parameter_dict) + return True except self.SyntaxError as e: - logger.error(str(e)) + logger.error('%s', e) print(self.configure_usage) return False From 28f418e1f2cc7322f3bea6d047db4473189f856f Mon Sep 17 00:00:00 2001 From: xin liang Date: Fri, 13 Sep 2024 09:41:45 +0800 Subject: [PATCH 12/17] Dev: bootstrap: Check if sbd package is installed in the right place Changes: - Check at the beginning of the bootstrap process - Check at configure sbd stage in the interactive mode - Put the sbd not installed message in the constants.py --- crmsh/bootstrap.py | 9 ++++++++- crmsh/sbd.py | 24 +++++++++--------------- crmsh/ui_sbd.py | 4 ++-- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/crmsh/bootstrap.py b/crmsh/bootstrap.py index 1b70a3111..594d5f7f3 100644 --- a/crmsh/bootstrap.py +++ b/crmsh/bootstrap.py @@ -216,12 +216,19 @@ def _validate_sbd_option(self): Validate sbd options """ from .sbd import SBDUtils + with_sbd_option = self.sbd_devices or self.diskless_sbd + sbd_installed = utils.package_is_installed("sbd") + + if with_sbd_option and not sbd_installed: + utils.fatal(SBDManager.SBD_NOT_INSTALLED_MSG) if self.sbd_devices and self.diskless_sbd: utils.fatal("Can't use -s and -S options together") if self.sbd_devices: SBDUtils.verify_sbd_device(self.sbd_devices) if self.stage == "sbd": - if not self.sbd_devices and not self.diskless_sbd and self.yes_to_all: + if not sbd_installed: + utils.fatal(SBDManager.SBD_NOT_INSTALLED_MSG) + if not with_sbd_option and self.yes_to_all: utils.fatal("Stage sbd should specify sbd device by -s or diskless sbd by -S option") if ServiceManager().service_is_active(constants.SBD_SERVICE) and not config.core.force: utils.fatal("Can't configure stage sbd: sbd.service already running! Please use crm option '-F' if need to redeploy") diff --git a/crmsh/sbd.py b/crmsh/sbd.py index d68fa1317..7cf8bb792 100644 --- a/crmsh/sbd.py +++ b/crmsh/sbd.py @@ -389,6 +389,7 @@ class SBDManager: NO_SBD_WARNING = "Not configuring SBD - STONITH will be disabled." DISKLESS_SBD_MIN_EXPECTED_VOTE = 3 DISKLESS_SBD_WARNING = "Diskless SBD requires cluster with three or more nodes. If you want to use diskless SBD for 2-node cluster, should be combined with QDevice." + SBD_NOT_INSTALLED_MSG = "Package sbd is not installed." SBD_RA = "stonith:fence_sbd" SBD_RA_ID = "stonith-sbd" SBD_DEVICE_MAX = 3 @@ -406,10 +407,6 @@ def __init__( ''' Init function which can be called from crm sbd subcommand or bootstrap ''' - self.package_installed = utils.package_is_installed("sbd") - if not self.package_installed: - return - self.device_list_to_init = device_list_to_init or [] self.timeout_dict = timeout_dict or {} self.update_dict = update_dict or {} @@ -531,23 +528,26 @@ def _warn_diskless_sbd(self, peer=None): vote_dict = utils.get_quorum_votes_dict(peer) expected_vote = int(vote_dict.get('Expected', 0)) if expected_vote < self.DISKLESS_SBD_MIN_EXPECTED_VOTE: - logger.warning(self.DISKLESS_SBD_WARNING) + logger.warning('%s', self.DISKLESS_SBD_WARNING) # When in init process elif self.diskless_sbd: - logger.warning(self.DISKLESS_SBD_WARNING) + logger.warning('%s', self.DISKLESS_SBD_WARNING) def get_sbd_device_interactive(self): ''' Get sbd device on interactive mode ''' if self.bootstrap_context.yes_to_all: - logger.warning(self.NO_SBD_WARNING) + logger.warning('%s', self.NO_SBD_WARNING) return logger.info(self.SBD_STATUS_DESCRIPTION) if not bootstrap.confirm("Do you wish to use SBD?"): - logger.warning(self.NO_SBD_WARNING) + logger.warning('%s', self.NO_SBD_WARNING) return + if not utils.package_is_installed("sbd"): + utils.fatal(self.SBD_NOT_INSTALLED_MSG) + configured_devices = SBDUtils.get_sbd_device_from_config() for dev in configured_devices: self.no_overwrite_dev_map[dev] = SBDUtils.no_overwrite_device_check(dev) @@ -566,7 +566,7 @@ def get_sbd_device_interactive(self): try: SBDUtils.verify_sbd_device(dev_list) except ValueError as e: - logger.error(e) + logger.error('%s', e) continue for dev in dev_list: if dev not in self.no_overwrite_dev_map: @@ -606,9 +606,6 @@ def init_and_deploy_sbd(self): 4. Restart cluster service if possible 5. Configure stonith-sbd resource and related properties ''' - if not self.package_installed: - return - if self.bootstrap_context: self.get_sbd_device_from_bootstrap() if not self.device_list_to_init and not self.diskless_sbd: @@ -631,9 +628,6 @@ def join_sbd(self, remote_user, peer_host): On joining process, check whether peer node has enabled sbd.service If so, check prerequisites of SBD and verify sbd device on join node ''' - if not self.package_installed: - return - service_manager = ServiceManager() if not os.path.exists(SYSCONFIG_SBD) or not service_manager.service_is_enabled(constants.SBD_SERVICE, peer_host): service_manager.disable_service(constants.SBD_SERVICE) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index c7438d1ce..59ba02fe8 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -143,7 +143,7 @@ def requires(self) -> bool: Requirements check when entering sbd sub-level ''' if not utils.package_is_installed("sbd"): - logger.error("sbd is not installed") + logger.error('%s', sbd.SBDManager.SBD_NOT_INSTALLED_MSG) return False return True @@ -443,7 +443,7 @@ def do_remove(self, context, *args) -> bool: else: sbd.disable_sbd_from_cluster() - logger.info(self.RESTART_INFO) + logger.info('%s', self.RESTART_INFO) return True def do_status(self, context) -> bool: From 221eaa600bcd5f859e844bbe2bcc08c5ce710b96 Mon Sep 17 00:00:00 2001 From: xin liang Date: Fri, 13 Sep 2024 16:12:11 +0800 Subject: [PATCH 13/17] Dev: ui_sbd: Refactor do_status method - Split the method into smaller methods - Enhance the readability of output - Print the type of SBD --- crmsh/ui_sbd.py | 61 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index 59ba02fe8..43956e03a 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -446,13 +446,23 @@ def do_remove(self, context, *args) -> bool: logger.info('%s', self.RESTART_INFO) return True - def do_status(self, context) -> bool: - ''' - Implement sbd status command - ''' - self._load_attributes() + def _print_sbd_type(self): + if not self.service_manager.service_is_active(constants.SBD_SERVICE): + return + print(f"# Type of SBD:") + if self.device_list_from_config: + print("Disk-based SBD configured") + else: + print("Diskless SBD configured") + print() + + def _print_sbd_status(self): + padding = 2 + status_len = 8 + max_node_len = max(len(node) for node in self.cluster_nodes) + padding - print(f"{constants.SBD_SERVICE} status: (active|enabled|since)") + print(f"# Status of {constants.SBD_SERVICE}:") + print(f"{'Node':<{max_node_len}}|{'Active':<{status_len}}|{'Enabled':<{status_len}}|Since") for node in self.cluster_nodes: is_active = self.service_manager.service_is_active(constants.SBD_SERVICE, node) is_active_str = "YES" if is_active else "NO" @@ -462,24 +472,42 @@ def do_status(self, context) -> bool: since_str_prefix = "active since" if is_active else "disactive since" systemctl_show_cmd = f"systemctl show {constants.SBD_SERVICE} --property={systemd_property} --value" since = self.cluster_shell.get_stdout_or_raise_error(systemctl_show_cmd, node) or "N/A" - print(f"{node}: {is_active_str:<4}|{is_enabled_str:<4}|{since_str_prefix}: {since}") + print(f"{node:<{max_node_len}}|{is_active_str:<{status_len}}|{is_enabled_str:<{status_len}}|{since_str_prefix}: {since}") print() - print("watchdog info: (device|driver|kernel timeout)") + def _print_watchdog_info(self): + padding = 2 + max_node_len = max(len(node) for node in self.cluster_nodes) + padding + watchdog_sbd_re = "\[[0-9]+\] (/dev/.*)\nIdentity: Busy: .*sbd.*\nDriver: (.*)" - for node in self.cluster_nodes: + device_list, driver_list, kernel_timeout_list = [], [], [] + cluster_nodes = self.cluster_nodes[:] + for node in cluster_nodes[:]: out = self.cluster_shell.get_stdout_or_raise_error("sbd query-watchdog", node) res = re.search(watchdog_sbd_re, out) if res: device, driver = res.groups() kernel_timeout = self.cluster_shell.get_stdout_or_raise_error("cat /proc/sys/kernel/watchdog_thresh", node) - print(f"{node}: {device}|{driver}|{kernel_timeout}") + device_list.append(device) + driver_list.append(driver) + kernel_timeout_list.append(kernel_timeout) else: logger.error("Failed to get watchdog info from %s", node) + cluster_nodes.remove(node) + if not cluster_nodes: + return + + print("# Watchdog info:") + max_dev_len = max(len(dev) for dev in device_list) + padding + max_driver_len = max(len(driver) for driver in driver_list) + padding + print(f"{'Node':<{max_node_len}}|{'Device':<{max_dev_len}}|{'Driver':<{max_driver_len}}|Kernel Timeout") + for i, node in enumerate(cluster_nodes): + print(f"{node:<{max_node_len}}|{device_list[i]:<{max_dev_len}}|{driver_list[i]:<{max_driver_len}}|{kernel_timeout_list[i]}") print() + def _print_sbd_agent_status(self): if self.crm_mon_xml_parser.is_resource_configured(sbd.SBDManager.SBD_RA): - print("fence_sbd status: ") + print("# Status of fence_sbd:") sbd_id_list = self.crm_mon_xml_parser.get_resource_id_list_via_type(sbd.SBDManager.SBD_RA) for sbd_id in sbd_id_list: rc, out, err = self.cluster_shell.get_rc_stdout_stderr_without_input(None, f"crm resource status {sbd_id}") @@ -487,3 +515,14 @@ def do_status(self, context) -> bool: print(out) if err: print(err) + + def do_status(self, context) -> bool: + ''' + Implement sbd status command + ''' + self._load_attributes() + self._print_sbd_type() + self._print_sbd_status() + self._print_watchdog_info() + self._print_sbd_agent_status() + return True From b187d8f26a3dcc2c26308a2eb3bf8e099e7f41e7 Mon Sep 17 00:00:00 2001 From: xin liang Date: Thu, 19 Sep 2024 10:30:10 +0800 Subject: [PATCH 14/17] Dev: Refactor the code to avoid circular import --- crmsh/bootstrap.py | 18 ++++++---------- crmsh/qdevice.py | 2 +- crmsh/sbd.py | 25 +++++++++++----------- crmsh/ui_sbd.py | 5 ++--- crmsh/watchdog.py | 14 ++++++------ test/features/steps/step_implementation.py | 3 ++- 6 files changed, 32 insertions(+), 35 deletions(-) diff --git a/crmsh/bootstrap.py b/crmsh/bootstrap.py index 594d5f7f3..f34aeb3eb 100644 --- a/crmsh/bootstrap.py +++ b/crmsh/bootstrap.py @@ -44,6 +44,8 @@ from .sh import ShellUtils from .ui_node import NodeMgmt from .user_of_host import UserOfHost, UserNotFoundError +from .sbd import SBDUtils, SBDManager, SBDTimeout +from . import watchdog import crmsh.healthcheck @@ -56,21 +58,18 @@ COROSYNC_AUTH = "/etc/corosync/authkey" CRM_CFG = "/etc/crm/crm.conf" PROFILES_FILE = "/etc/crm/profiles.yml" -SYSCONFIG_SBD = "/etc/sysconfig/sbd" SYSCONFIG_PCMK = "/etc/sysconfig/pacemaker" SYSCONFIG_NFS = "/etc/sysconfig/nfs" PCMK_REMOTE_AUTH = "/etc/pacemaker/authkey" COROSYNC_CONF_ORIG = tmpfiles.create()[1] SERVICES_STOP_LIST = ["corosync-qdevice.service", "corosync.service", "hawk.service", CSYNC2_SERVICE] -WATCHDOG_CFG = "/etc/modules-load.d/watchdog.conf" BOOTH_DIR = "/etc/booth" BOOTH_CFG = "/etc/booth/booth.conf" BOOTH_AUTH = "/etc/booth/authkey" -SBD_SYSTEMD_DELAY_START_DIR = "/etc/systemd/system/sbd.service.d" FILES_TO_SYNC = (BOOTH_DIR, corosync.conf(), COROSYNC_AUTH, CSYNC2_CFG, CSYNC2_KEY, "/etc/ctdb/nodes", "/etc/drbd.conf", "/etc/drbd.d", "/etc/ha.d/ldirectord.cf", "/etc/lvm/lvm.conf", "/etc/multipath.conf", - "/etc/samba/smb.conf", SYSCONFIG_NFS, SYSCONFIG_PCMK, SYSCONFIG_SBD, PCMK_REMOTE_AUTH, WATCHDOG_CFG, - PROFILES_FILE, CRM_CFG, SBD_SYSTEMD_DELAY_START_DIR) + "/etc/samba/smb.conf", SYSCONFIG_NFS, SYSCONFIG_PCMK, SBDManager.SYSCONFIG_SBD, PCMK_REMOTE_AUTH, watchdog.Watchdog.WATCHDOG_CFG, + PROFILES_FILE, CRM_CFG, SBDManager.SBD_SYSTEMD_DELAY_START_DIR) INIT_STAGES_EXTERNAL = ("ssh", "csync2", "corosync", "sbd", "cluster", "ocfs2", "admin", "qdevice") INIT_STAGES_INTERNAL = ("csync2_remote", "qnetd_remote", "remote_auth") @@ -136,7 +135,7 @@ def __init__(self): self.profiles_dict = {} self.default_nic = None self.default_ip_list = [] - self.rm_list = [SYSCONFIG_SBD, CSYNC2_CFG, corosync.conf(), CSYNC2_KEY, + self.rm_list = [SBDManager.SYSCONFIG_SBD, CSYNC2_CFG, corosync.conf(), CSYNC2_KEY, COROSYNC_AUTH, "/var/lib/heartbeat/crm/*", "/var/lib/pacemaker/cib/*", "/var/lib/corosync/*", "/var/lib/pacemaker/pengine/*", PCMK_REMOTE_AUTH, "/var/lib/csync2/*", "~/.config/crm/*"] @@ -215,7 +214,6 @@ def _validate_sbd_option(self): """ Validate sbd options """ - from .sbd import SBDUtils with_sbd_option = self.sbd_devices or self.diskless_sbd sbd_installed = utils.package_is_installed("sbd") @@ -307,7 +305,6 @@ def validate_option(self): self._validate_sbd_option() def init_sbd_manager(self): - from .sbd import SBDManager self.sbd_manager = SBDManager(bootstrap_context=self) def detect_platform(self): @@ -779,7 +776,6 @@ def start_pacemaker(node_list=[], enable_flag=False): Return success node list """ - from .sbd import SBDTimeout # not _context means not in init or join process if not _context and \ utils.package_is_installed("sbd") and \ @@ -2101,8 +2097,7 @@ def rm_configuration_files(remote=None): shell.get_stdout_or_raise_error("rm -f {}".format(' '.join(_context.rm_list)), remote) # restore original sbd configuration file from /usr/share/fillup-templates/sysconfig.sbd if utils.package_is_installed("sbd", remote_addr=remote): - from .sbd import SBDManager - cmd = "cp {} {}".format(SBDManager.SYSCONFIG_SBD_TEMPLATE, SYSCONFIG_SBD) + cmd = "cp {} {}".format(SBDManager.SYSCONFIG_SBD_TEMPLATE, SBDManager.SYSCONFIG_SBD) shell.get_stdout_or_raise_error(cmd, remote) @@ -2750,7 +2745,6 @@ def adjust_stonith_timeout(): Adjust stonith-timeout for sbd and other scenarios """ if ServiceManager().service_is_active(constants.SBD_SERVICE): - from .sbd import SBDTimeout SBDTimeout.adjust_sbd_timeout_related_cluster_configuration() else: value = get_stonith_timeout_generally_expected() diff --git a/crmsh/qdevice.py b/crmsh/qdevice.py index e81fae41a..7c0ac993c 100644 --- a/crmsh/qdevice.py +++ b/crmsh/qdevice.py @@ -15,6 +15,7 @@ from . import lock from . import log from .service_manager import ServiceManager +from .sbd import SBDManager, SBDTimeout, SBDUtils logger = log.setup_logger(__name__) @@ -614,7 +615,6 @@ def adjust_sbd_watchdog_timeout_with_qdevice(self): """ Adjust SBD_WATCHDOG_TIMEOUT when configuring qdevice and diskless SBD """ - from .sbd import SBDManager, SBDTimeout, SBDUtils utils.check_all_nodes_reachable() self.using_diskless_sbd = SBDUtils.is_using_diskless_sbd() # add qdevice after diskless sbd started diff --git a/crmsh/sbd.py b/crmsh/sbd.py index 7cf8bb792..b40e9fc0d 100644 --- a/crmsh/sbd.py +++ b/crmsh/sbd.py @@ -3,7 +3,6 @@ import typing from . import utils, sh from . import bootstrap -from .bootstrap import SYSCONFIG_SBD, SBD_SYSTEMD_DELAY_START_DIR from . import log from . import constants from . import corosync @@ -76,7 +75,7 @@ def get_sbd_value_from_config(key): ''' Get value from /etc/sysconfig/sbd ''' - return utils.parse_sysconfig(SYSCONFIG_SBD).get(key) + return utils.parse_sysconfig(SBDManager.SYSCONFIG_SBD).get(key) @staticmethod def get_sbd_device_from_config(): @@ -338,10 +337,10 @@ def adjust_systemd_start_timeout(self): if start_timeout > int(sbd_delay_start_value): return - utils.mkdirp(SBD_SYSTEMD_DELAY_START_DIR) - sbd_delay_start_file = "{}/sbd_delay_start.conf".format(SBD_SYSTEMD_DELAY_START_DIR) + utils.mkdirp(SBDManager.SBD_SYSTEMD_DELAY_START_DIR) + sbd_delay_start_file = "{}/sbd_delay_start.conf".format(SBDManager.SBD_SYSTEMD_DELAY_START_DIR) utils.str2file("[Service]\nTimeoutSec={}".format(int(1.2*int(sbd_delay_start_value))), sbd_delay_start_file) - bootstrap.sync_file(SBD_SYSTEMD_DELAY_START_DIR) + bootstrap.sync_file(SBDManager.SBD_SYSTEMD_DELAY_START_DIR) utils.cluster_run_cmd("systemctl daemon-reload") def adjust_stonith_timeout(self): @@ -376,7 +375,9 @@ def adjust_sbd_timeout_related_cluster_configuration(cls): class SBDManager: + SYSCONFIG_SBD = "/etc/sysconfig/sbd" SYSCONFIG_SBD_TEMPLATE = "/usr/share/fillup-templates/sysconfig.sbd" + SBD_SYSTEMD_DELAY_START_DIR = "/etc/systemd/system/sbd.service.d" SBD_STATUS_DESCRIPTION = '''Configure SBD: If you have shared storage, for example a SAN or iSCSI target, you can use it avoid split-brain scenarios by configuring SBD. @@ -402,7 +403,7 @@ def __init__( no_overwrite_dev_map: typing.Dict[str, bool] | None = None, new_config: bool = False, diskless_sbd: bool = False, - bootstrap_context: bootstrap.Context | None = None + bootstrap_context: 'bootstrap.Context | None' = None ): ''' Init function which can be called from crm sbd subcommand or bootstrap @@ -451,13 +452,13 @@ def update_configuration(self) -> None: if not self.update_dict: return if (self.bootstrap_context and self.bootstrap_context.type == "init") or self.new_config: - utils.copy_local_file(self.SYSCONFIG_SBD_TEMPLATE, SYSCONFIG_SBD) + utils.copy_local_file(self.SYSCONFIG_SBD_TEMPLATE, self.SYSCONFIG_SBD) for key, value in self.update_dict.items(): - logger.info("Update %s in %s: %s", key, SYSCONFIG_SBD, value) - utils.sysconfig_set(SYSCONFIG_SBD, **self.update_dict) - bootstrap.sync_file(SYSCONFIG_SBD) - logger.info("Already synced %s to all nodes", SYSCONFIG_SBD) + logger.info("Update %s in %s: %s", key, self.SYSCONFIG_SBD, value) + utils.sysconfig_set(self.SYSCONFIG_SBD, **self.update_dict) + bootstrap.sync_file(self.SYSCONFIG_SBD) + logger.info("Already synced %s to all nodes", self.SYSCONFIG_SBD) @classmethod def update_sbd_configuration(cls, update_dict: typing.Dict[str, str]) -> None: @@ -629,7 +630,7 @@ def join_sbd(self, remote_user, peer_host): If so, check prerequisites of SBD and verify sbd device on join node ''' service_manager = ServiceManager() - if not os.path.exists(SYSCONFIG_SBD) or not service_manager.service_is_enabled(constants.SBD_SERVICE, peer_host): + if not os.path.exists(self.SYSCONFIG_SBD) or not service_manager.service_is_enabled(constants.SBD_SERVICE, peer_host): service_manager.disable_service(constants.SBD_SERVICE) return diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index 43956e03a..37290acfd 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -13,7 +13,6 @@ from crmsh import xmlutil from crmsh import constants from crmsh.service_manager import ServiceManager -from crmsh.bootstrap import SYSCONFIG_SBD logger = logging.getLogger(__name__) @@ -171,7 +170,7 @@ def _show_sysconfig() -> None: ''' Show pure content of /etc/sysconfig/sbd ''' - with open(SYSCONFIG_SBD) as f: + with open(sbd.SBDManager.SYSCONFIG_SBD) as f: content_list = [line.strip() for line in f.readlines() if not line.startswith("#") and line.strip()] @@ -435,7 +434,7 @@ def do_remove(self, context, *args) -> bool: changed_dev_list = set(self.device_list_from_config) - set(dev_list) # remove part of devices from config if changed_dev_list: - logger.info("Remove '%s' from %s", ";".join(dev_list), SYSCONFIG_SBD) + logger.info("Remove '%s' from %s", ";".join(dev_list), sbd.SBDManager.SYSCONFIG_SBD) sbd.SBDManager.update_sbd_configuration({"SBD_DEVICE": ";".join(changed_dev_list)}) # remove all devices, equivalent to stop sbd.service else: diff --git a/crmsh/watchdog.py b/crmsh/watchdog.py index 00e0f60a5..6b7c6cee9 100644 --- a/crmsh/watchdog.py +++ b/crmsh/watchdog.py @@ -1,14 +1,15 @@ import re from . import utils from .constants import SSH_OPTION -from .bootstrap import invoke, invokerc, WATCHDOG_CFG, SYSCONFIG_SBD from .sh import ShellUtils +from . import sbd class Watchdog(object): """ Class to find valid watchdog device name """ + WATCHDOG_CFG = "/etc/modules-load.d/watchdog.conf" QUERY_CMD = "sudo sbd query-watchdog" DEVICE_FIND_REGREX = "\[[0-9]+\] (/dev/.*)\n.*\nDriver: (.*)" @@ -44,15 +45,15 @@ def _load_watchdog_driver(driver): """ Load specific watchdog driver """ - invoke("echo {} > {}".format(driver, WATCHDOG_CFG)) - invoke("systemctl restart systemd-modules-load") + ShellUtils().get_stdout_stderr(f"echo {driver} > {Watchdog.WATCHDOG_CFG}") + ShellUtils().get_stdout_stderr("systemctl restart systemd-modules-load") @staticmethod def get_watchdog_device_from_sbd_config(): """ Try to get watchdog device name from sbd config file """ - conf = utils.parse_sysconfig(SYSCONFIG_SBD) + conf = utils.parse_sysconfig(sbd.SBDManager.SYSCONFIG_SBD) return conf.get("SBD_WATCHDOG_DEV") @staticmethod @@ -144,7 +145,7 @@ def join_watchdog(self): res = self.get_watchdog_device_from_sbd_config() if not res: - utils.fatal("Failed to get watchdog device from {}".format(SYSCONFIG_SBD)) + utils.fatal("Failed to get watchdog device from {}".format(sbd.SBDManager.SYSCONFIG_SBD)) self._input = res if not self._valid_device(self._input): @@ -164,7 +165,8 @@ def init_watchdog(self): return # self._input is invalid, exit - if not invokerc("modinfo {}".format(self._input)): + rc, _, _ = ShellUtils().get_stdout_stderr(f"modinfo {self._input}") + if rc != 0: utils.fatal("Should provide valid watchdog device or driver name by -w option") # self._input is a driver name, load it if it was unloaded diff --git a/test/features/steps/step_implementation.py b/test/features/steps/step_implementation.py index 47b138393..b87b52c31 100644 --- a/test/features/steps/step_implementation.py +++ b/test/features/steps/step_implementation.py @@ -7,8 +7,9 @@ import behave from behave import given, when, then import behave_agent -from crmsh import corosync, sbd, userdir, bootstrap +from crmsh import corosync, userdir, bootstrap from crmsh import utils as crmutils +from crmsh import sbd from crmsh.sh import ShellUtils from utils import check_cluster_state, check_service_state, online, run_command, me, \ run_command_local_or_remote, file_in_archive, \ From 4e8e361513c78f8ef8b3c086f349719df73a721b Mon Sep 17 00:00:00 2001 From: xin liang Date: Fri, 20 Sep 2024 09:49:29 +0800 Subject: [PATCH 15/17] Dev: report: Dump output of 'crm sbd configure show' and 'crm sbd status' to the report result --- crmsh/report/collect.py | 13 +++++++++---- crmsh/report/utils.py | 2 +- crmsh/utils.py | 8 ++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/crmsh/report/collect.py b/crmsh/report/collect.py index 24c6c7145..441487916 100644 --- a/crmsh/report/collect.py +++ b/crmsh/report/collect.py @@ -447,11 +447,16 @@ def collect_sbd_info(context: core.Context) -> None: return sbd_f = os.path.join(context.work_dir, constants.SBD_F) - cmd = ". {};export SBD_DEVICE;{};{}".format(constants.SBDCONF, "sbd dump", "sbd list") + cmd_list = [ + f". {constants.SBDCONF};export SBD_DEVICE;sbd dump;sbd list", + "crm sbd configure show", + "crm sbd status" + ] with open(sbd_f, "w") as f: - f.write("\n\n#=====[ Command ] ==========================#\n") - f.write(f"# {cmd}\n") - f.write(utils.get_cmd_output(cmd)) + for cmd in cmd_list: + f.write("\n\n#=====[ Command ] ==========================#\n") + f.write(f"# {cmd}\n") + f.write(utils.get_cmd_output(cmd)) logger.debug(f"Dump SBD config file into {utils.real_path(sbd_f)}") diff --git a/crmsh/report/utils.py b/crmsh/report/utils.py index 6c439126d..686bcfe88 100644 --- a/crmsh/report/utils.py +++ b/crmsh/report/utils.py @@ -739,7 +739,7 @@ def get_cmd_output(cmd: str, timeout: int = None) -> str: out_str += f"{out}\n" if err: out_str += f"{err}\n" - return out_str + return crmutils.strip_ansi_escape_sequences(out_str) def get_timespan_str(context: core.Context) -> str: diff --git a/crmsh/utils.py b/crmsh/utils.py index e35235e34..68ab61456 100644 --- a/crmsh/utils.py +++ b/crmsh/utils.py @@ -3225,4 +3225,12 @@ def is_subdict(sub_dict, main_dict): Check if sub_dict is a sub-dictionary of main_dict """ return all(item in main_dict.items() for item in sub_dict.items()) + + +def strip_ansi_escape_sequences(text): + """ + Remove ANSI escape sequences from text + """ + ansi_escape_pattern = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') + return ansi_escape_pattern.sub('', text) # vim:ts=4:sw=4:et: From c58b6b82e5fe74fdc55c30b55a8af11b8cb497df Mon Sep 17 00:00:00 2001 From: xin liang Date: Fri, 20 Sep 2024 10:26:09 +0800 Subject: [PATCH 16/17] Dev: ui_sbd: No need to specify device="" when trying to modify properties under diskless sbd --- crmsh/ui_sbd.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crmsh/ui_sbd.py b/crmsh/ui_sbd.py index 37290acfd..d2892de93 100644 --- a/crmsh/ui_sbd.py +++ b/crmsh/ui_sbd.py @@ -269,6 +269,12 @@ def _parse_args(self, args: tuple[str, ...]) -> dict[str, int|str|list[str]]: watchdog_device = parameter_dict.get("watchdog-device") parameter_dict["watchdog-device"] = watchdog.Watchdog.get_watchdog_device(watchdog_device) + # No need to specify device="" when trying to modify properties under diskless sbd + if sbd.SBDUtils.is_using_diskless_sbd() \ + and "device-list" in parameter_dict \ + and not parameter_dict["device-list"]: + parameter_dict.pop("device-list") + logger.debug("Parsed arguments: %s", parameter_dict) return parameter_dict From 8cd77052197c8b48d1a807d593ff827830bd6a51 Mon Sep 17 00:00:00 2001 From: xin liang Date: Fri, 20 Sep 2024 10:52:32 +0800 Subject: [PATCH 17/17] Dev: unittests: Adjust unit test for previous commits --- test/unittests/test_bootstrap.py | 23 +- test/unittests/test_ocfs2.py | 2 +- test/unittests/test_qdevice.py | 6 +- test/unittests/test_report_collect.py | 8 +- test/unittests/test_sbd.py | 1000 ++++--------------------- test/unittests/test_utils.py | 2 +- test/unittests/test_watchdog.py | 64 +- 7 files changed, 197 insertions(+), 908 deletions(-) diff --git a/test/unittests/test_bootstrap.py b/test/unittests/test_bootstrap.py index 12de9e253..e5b23065e 100644 --- a/test/unittests/test_bootstrap.py +++ b/test/unittests/test_bootstrap.py @@ -126,8 +126,10 @@ def test_initialize_qdevice_with_user(self, mock_qdevice): ctx.initialize_qdevice() mock_qdevice.assert_called_once_with(qnetd_addr='node3', port=123, ssh_user='alice', algo=None, tie_breaker=None, tls=None, cmds=None, mode=None, is_stage=False) + @mock.patch('crmsh.utils.package_is_installed') @mock.patch('crmsh.utils.fatal') - def test_validate_sbd_option_error_together(self, mock_error): + def test_validate_sbd_option_error_together(self, mock_error, mock_installed): + mock_installed.return_value = True mock_error.side_effect = SystemExit ctx = crmsh.bootstrap.Context() ctx.sbd_devices = ["/dev/sda1"] @@ -136,8 +138,10 @@ def test_validate_sbd_option_error_together(self, mock_error): ctx._validate_sbd_option() mock_error.assert_called_once_with("Can't use -s and -S options together") + @mock.patch('crmsh.utils.package_is_installed') @mock.patch('crmsh.utils.fatal') - def test_validate_sbd_option_error_sbd_stage_no_option(self, mock_error): + def test_validate_sbd_option_error_sbd_stage_no_option(self, mock_error, mock_installed): + mock_installed.return_value = True mock_error.side_effect = SystemExit ctx = crmsh.bootstrap.Context() ctx.stage = "sbd" @@ -146,9 +150,11 @@ def test_validate_sbd_option_error_sbd_stage_no_option(self, mock_error): ctx._validate_sbd_option() mock_error.assert_called_once_with("Stage sbd should specify sbd device by -s or diskless sbd by -S option") + @mock.patch('crmsh.utils.package_is_installed') @mock.patch('crmsh.utils.fatal') @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - def test_validate_sbd_option_error_sbd_stage_service(self, mock_active, mock_error): + def test_validate_sbd_option_error_sbd_stage_service(self, mock_active, mock_error, mock_installed): + mock_installed.return_value = True mock_error.side_effect = SystemExit ctx = crmsh.bootstrap.Context() ctx.stage = "sbd" @@ -159,10 +165,12 @@ def test_validate_sbd_option_error_sbd_stage_service(self, mock_active, mock_err mock_error.assert_called_once_with("Can't configure stage sbd: sbd.service already running! Please use crm option '-F' if need to redeploy") mock_active.assert_called_once_with("sbd.service") + @mock.patch('crmsh.utils.package_is_installed') @mock.patch('crmsh.utils.check_all_nodes_reachable') @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - def test_validate_sbd_option_error_sbd_stage(self, mock_active, mock_check_all): + def test_validate_sbd_option_error_sbd_stage(self, mock_active, mock_check_all, mock_installed): options = mock.Mock(stage="sbd", diskless_sbd=True, cluster_is_running=True) + mock_installed.return_value = True ctx = crmsh.bootstrap.Context() ctx.stage = "sbd" ctx.diskless_sbd = True @@ -1345,13 +1353,12 @@ def test_adjust_pcmk_delay(self, mock_cib_factory, mock_run, mock_debug): bootstrap.adjust_pcmk_delay_max(False) mock_run.assert_called_once_with("crm resource param res_1 delete pcmk_delay_max") - @mock.patch('crmsh.sbd.SBDTimeout') + @mock.patch('crmsh.sbd.SBDTimeout.adjust_sbd_timeout_related_cluster_configuration') @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - def test_adjust_stonith_timeout_sbd(self, mock_is_active, mock_sbd_timeout): + def test_adjust_stonith_timeout_sbd(self, mock_is_active, mock_sbd_adjust_timeout): mock_is_active.return_value = True - mock_sbd_timeout.adjust_sbd_timeout_related_cluster_configuration = mock.Mock() bootstrap.adjust_stonith_timeout() - mock_sbd_timeout.adjust_sbd_timeout_related_cluster_configuration.assert_called_once_with() + mock_sbd_adjust_timeout.assert_called_once_with() @mock.patch('crmsh.utils.set_property') @mock.patch('crmsh.bootstrap.get_stonith_timeout_generally_expected') diff --git a/test/unittests/test_ocfs2.py b/test/unittests/test_ocfs2.py index 603c68d6c..37821fbd2 100644 --- a/test/unittests/test_ocfs2.py +++ b/test/unittests/test_ocfs2.py @@ -157,7 +157,7 @@ def test_dynamic_raise_error(self): self.assertEqual("error messages", str(err.exception)) @mock.patch('crmsh.ocfs2.OCFS2Manager._dynamic_raise_error') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_device_from_config') + @mock.patch('crmsh.sbd.SBDUtils.get_sbd_device_from_config') @mock.patch('crmsh.service_manager.ServiceManager.service_is_enabled') def test_check_sbd_and_ocfs2_dev(self, mock_enabled, mock_get_device, mock_error): mock_enabled.return_value = True diff --git a/test/unittests/test_qdevice.py b/test/unittests/test_qdevice.py index c3ec38016..6e9df2e01 100644 --- a/test/unittests/test_qdevice.py +++ b/test/unittests/test_qdevice.py @@ -817,9 +817,9 @@ def test_config_and_start_qdevice(self, mock_rm_db, mock_status_long, mock_evalu @mock.patch('crmsh.utils.set_property') @mock.patch('crmsh.sbd.SBDTimeout.get_stonith_timeout') - @mock.patch('crmsh.sbd.SBDManager.update_configuration') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - @mock.patch('crmsh.sbd.SBDManager.is_using_diskless_sbd') + @mock.patch('crmsh.sbd.SBDManager.update_sbd_configuration') + @mock.patch('crmsh.sbd.SBDUtils.get_sbd_value_from_config') + @mock.patch('crmsh.sbd.SBDUtils.is_using_diskless_sbd') @mock.patch('crmsh.utils.check_all_nodes_reachable') def test_adjust_sbd_watchdog_timeout_with_qdevice(self, mock_check_reachable, mock_using_diskless_sbd, mock_get_sbd_value, mock_update_config, mock_get_timeout, mock_set_property): mock_using_diskless_sbd.return_value = True diff --git a/test/unittests/test_report_collect.py b/test/unittests/test_report_collect.py index e13443236..37d98b16b 100644 --- a/test/unittests/test_report_collect.py +++ b/test/unittests/test_report_collect.py @@ -187,7 +187,7 @@ def test_collect_sbd_info(self, mock_exists, mock_copy, mock_which, mock_run, mo mock_open_write = mock.mock_open() file_handle = mock_open_write.return_value.__enter__.return_value mock_open_file.return_value = mock_open_write.return_value - mock_run.return_value = "data" + mock_run.side_effect = ["data", "data", "data"] mock_ctx_inst = mock.Mock(work_dir="/opt") collect.collect_sbd_info(mock_ctx_inst) @@ -199,6 +199,12 @@ def test_collect_sbd_info(self, mock_exists, mock_copy, mock_which, mock_run, mo file_handle.write.assert_has_calls([ mock.call("\n\n#=====[ Command ] ==========================#\n"), mock.call("# . /etc/sysconfig/sbd;export SBD_DEVICE;sbd dump;sbd list\n"), + mock.call("data"), + mock.call("\n\n#=====[ Command ] ==========================#\n"), + mock.call("# crm sbd configure show\n"), + mock.call("data"), + mock.call("\n\n#=====[ Command ] ==========================#\n"), + mock.call("# crm sbd status\n"), mock.call("data") ]) mock_debug.assert_called_once_with(f"Dump SBD config file into {constants.SBD_F}") diff --git a/test/unittests/test_sbd.py b/test/unittests/test_sbd.py index e8f0f259b..75ed59a85 100644 --- a/test/unittests/test_sbd.py +++ b/test/unittests/test_sbd.py @@ -1,296 +1,149 @@ -import os -import unittest import logging +import unittest +from unittest.mock import patch, MagicMock +from crmsh.sbd import SBDUtils, SBDManager -try: - from unittest import mock -except ImportError: - import mock - -from crmsh import bootstrap -from crmsh import sbd - - -class TestSBDTimeout(unittest.TestCase): - """ - Unitary tests for crmsh.sbd.SBDTimeout - """ - - @classmethod - def setUpClass(cls): - """ - Global setUp. - """ - - def setUp(self): - """ - Test setUp. - """ - _dict = {"sbd.watchdog_timeout": 5, "sbd.msgwait": 10} - _inst_q = mock.Mock() - self.sbd_timeout_inst = sbd.SBDTimeout(mock.Mock(profiles_dict=_dict, is_s390=True, qdevice_inst=_inst_q)) - - def tearDown(self): - """ - Test tearDown. - """ - - @classmethod - def tearDownClass(cls): - """ - Global tearDown. - """ - - def test_initialize_timeout(self): - self.sbd_timeout_inst._set_sbd_watchdog_timeout = mock.Mock() - self.sbd_timeout_inst._set_sbd_msgwait = mock.Mock() - self.sbd_timeout_inst._adjust_sbd_watchdog_timeout_with_diskless_and_qdevice = mock.Mock() - self.sbd_timeout_inst.initialize_timeout() - self.sbd_timeout_inst._set_sbd_watchdog_timeout.assert_called_once() - self.sbd_timeout_inst._set_sbd_msgwait.assert_not_called() - self.sbd_timeout_inst._adjust_sbd_watchdog_timeout_with_diskless_and_qdevice.assert_called_once() - - @mock.patch('logging.Logger.warning') - def test_set_sbd_watchdog_timeout(self, mock_warn): - self.sbd_timeout_inst._set_sbd_watchdog_timeout() - mock_warn.assert_called_once_with("sbd_watchdog_timeout is set to %d for s390, it was %d", sbd.SBDTimeout.SBD_WATCHDOG_TIMEOUT_DEFAULT_S390, 5) - - @mock.patch('logging.Logger.warning') - def test_set_sbd_msgwait(self, mock_warn): - self.sbd_timeout_inst.sbd_watchdog_timeout = 15 - self.sbd_timeout_inst._set_sbd_msgwait() - mock_warn.assert_called_once_with("sbd msgwait is set to %d, it was %d", 30, 10) - - @mock.patch('logging.Logger.warning') - @mock.patch('crmsh.utils.get_qdevice_sync_timeout') - @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - @mock.patch('crmsh.corosync.is_qdevice_configured') - def test_adjust_sbd_watchdog_timeout_with_diskless_and_qdevice_sbd_stage(self, mock_is_configured, mock_is_active, mock_get_sync, mock_warn): - mock_is_configured.return_value = True - mock_is_active.return_value = True - mock_get_sync.return_value = 15 - self.sbd_timeout_inst.sbd_watchdog_timeout = 5 - self.sbd_timeout_inst._adjust_sbd_watchdog_timeout_with_diskless_and_qdevice() - mock_warn.assert_called_once_with("sbd_watchdog_timeout is set to 20 for qdevice, it was 5") - - @mock.patch('logging.Logger.warning') - @mock.patch('crmsh.corosync.is_qdevice_configured') - def test_adjust_sbd_watchdog_timeout_with_diskless_and_qdevice_all(self, mock_is_configured, mock_warn): - mock_is_configured.return_value = False - self.sbd_timeout_inst.sbd_watchdog_timeout = 5 - self.sbd_timeout_inst._adjust_sbd_watchdog_timeout_with_diskless_and_qdevice() - mock_warn.assert_called_once_with("sbd_watchdog_timeout is set to 35 for qdevice, it was 5") - - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - def test_get_sbd_msgwait_exception(self, mock_run): - mock_run.return_value = "data" - with self.assertRaises(ValueError) as err: - sbd.SBDTimeout.get_sbd_msgwait("/dev/sda1") - self.assertEqual("Cannot get sbd msgwait for /dev/sda1", str(err.exception)) - mock_run.assert_called_once_with("sbd -d /dev/sda1 dump") - - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - def test_get_sbd_msgwait(self, mock_run): - mock_run.return_value = """ - Timeout (loop) : 1 - Timeout (msgwait) : 10 - ==Header on disk /dev/sda1 is dumped - """ - res = sbd.SBDTimeout.get_sbd_msgwait("/dev/sda1") - assert res == 10 - mock_run.assert_called_once_with("sbd -d /dev/sda1 dump") - - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_get_sbd_watchdog_timeout_exception(self, mock_get): - mock_get.return_value = None - with self.assertRaises(ValueError) as err: - sbd.SBDTimeout.get_sbd_watchdog_timeout() - self.assertEqual("Cannot get the value of SBD_WATCHDOG_TIMEOUT", str(err.exception)) - mock_get.assert_called_once_with("SBD_WATCHDOG_TIMEOUT") - - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_get_sbd_watchdog_timeout(self, mock_get): - mock_get.return_value = 5 - res = sbd.SBDTimeout.get_sbd_watchdog_timeout() - assert res == 5 - mock_get.assert_called_once_with("SBD_WATCHDOG_TIMEOUT") - - @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - def test_get_stonith_watchdog_timeout_return(self, mock_active): - mock_active.return_value = False - res = sbd.SBDTimeout.get_stonith_watchdog_timeout() - assert res == sbd.SBDTimeout.STONITH_WATCHDOG_TIMEOUT_DEFAULT - mock_active.assert_called_once_with("pacemaker.service") - - @mock.patch('crmsh.utils.get_property') - @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - def test_get_stonith_watchdog_timeout(self, mock_active, mock_get_property): - mock_active.return_value = True - mock_get_property.return_value = "60s" - res = sbd.SBDTimeout.get_stonith_watchdog_timeout() - assert res == 60 - mock_active.assert_called_once_with("pacemaker.service") - - @mock.patch('logging.Logger.debug') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - @mock.patch('crmsh.utils.detect_virt') - @mock.patch('crmsh.sbd.SBDTimeout.get_sbd_delay_start_expected') - @mock.patch('crmsh.utils.get_pcmk_delay_max') - @mock.patch('crmsh.sbd.SBDTimeout.get_sbd_msgwait') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_device_from_config') - @mock.patch('crmsh.utils.is_2node_cluster_without_qdevice') - def test_load_configurations(self, mock_2node, mock_get_sbd_dev, mock_get_msgwait, mock_pcmk_delay, mock_delay_expected, mock_detect, mock_get_sbd_value, mock_debug): - mock_2node.return_value = True - mock_debug.return_value = False - mock_get_sbd_value.return_value = "no" - mock_get_sbd_dev.return_value = ["/dev/sda1"] - mock_get_msgwait.return_value = 30 - mock_pcmk_delay.return_value = 30 - - self.sbd_timeout_inst._load_configurations() - - mock_2node.assert_called_once_with() - mock_get_sbd_dev.assert_called_once_with() - mock_get_msgwait.assert_called_once_with("/dev/sda1") - mock_pcmk_delay.assert_called_once_with(True) - - @mock.patch('logging.Logger.debug') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - @mock.patch('crmsh.utils.detect_virt') - @mock.patch('crmsh.sbd.SBDTimeout.get_sbd_delay_start_expected') - @mock.patch('crmsh.sbd.SBDTimeout.get_stonith_watchdog_timeout') - @mock.patch('crmsh.sbd.SBDTimeout.get_sbd_watchdog_timeout') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_device_from_config') - @mock.patch('crmsh.utils.is_2node_cluster_without_qdevice') - def test_load_configurations_diskless(self, mock_2node, mock_get_sbd_dev, mock_get_watchdog_timeout, mock_get_stonith_watchdog_timeout, mock_delay_expected, mock_detect, mock_get_sbd_value, mock_debug): - mock_2node.return_value = True - mock_debug.return_value = False - mock_get_sbd_value.return_value = "no" - mock_get_sbd_dev.return_value = [] - mock_get_watchdog_timeout.return_value = 30 - mock_get_stonith_watchdog_timeout.return_value = 30 - - self.sbd_timeout_inst._load_configurations() - - mock_2node.assert_called_once_with() - mock_get_sbd_dev.assert_called_once_with() - mock_get_watchdog_timeout.assert_called_once_with() - mock_get_stonith_watchdog_timeout.assert_called_once_with() - - @mock.patch('crmsh.corosync.token_and_consensus_timeout') - @mock.patch('logging.Logger.debug') - def test_get_stonith_timeout_expected(self, mock_debug, mock_general): - self.sbd_timeout_inst.disk_based = True - self.sbd_timeout_inst.pcmk_delay_max = 30 - self.sbd_timeout_inst.msgwait = 30 - mock_general.return_value = 11 - res = self.sbd_timeout_inst.get_stonith_timeout_expected() - assert res == 83 - - @mock.patch('crmsh.corosync.token_and_consensus_timeout') - @mock.patch('logging.Logger.debug') - def test_get_stonith_timeout_expected_diskless(self, mock_debug, mock_general): - self.sbd_timeout_inst.disk_based = False - self.sbd_timeout_inst.stonith_watchdog_timeout = -1 - self.sbd_timeout_inst.sbd_watchdog_timeout = 15 - mock_general.return_value = 11 - res = self.sbd_timeout_inst.get_stonith_timeout_expected() - assert res == 71 - - @mock.patch('crmsh.corosync.token_and_consensus_timeout') - def test_get_sbd_delay_start_expected(self, mock_corosync): - mock_corosync.return_value = 30 - self.sbd_timeout_inst.disk_based = True - self.sbd_timeout_inst.pcmk_delay_max = 30 - self.sbd_timeout_inst.msgwait = 30 - res = self.sbd_timeout_inst.get_sbd_delay_start_expected() - assert res == 90 - - @mock.patch('crmsh.corosync.token_and_consensus_timeout') - def test_get_sbd_delay_start_expected_diskless(self, mock_corosync): - mock_corosync.return_value = 30 - self.sbd_timeout_inst.disk_based = False - self.sbd_timeout_inst.sbd_watchdog_timeout = 30 - res = self.sbd_timeout_inst.get_sbd_delay_start_expected() - assert res == 90 - - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_is_sbd_delay_start(self, mock_get_sbd_value): - mock_get_sbd_value.return_value = "100" - assert sbd.SBDTimeout.is_sbd_delay_start() is True - mock_get_sbd_value.assert_called_once_with("SBD_DELAY_START") - - @mock.patch('crmsh.sbd.SBDManager.update_configuration') - def test_adjust_sbd_delay_start_return(self, mock_update): - self.sbd_timeout_inst.sbd_delay_start_value_expected = 100 - self.sbd_timeout_inst.sbd_delay_start_value_from_config = "100" - self.sbd_timeout_inst.adjust_sbd_delay_start() - mock_update.assert_not_called() - - @mock.patch('crmsh.sbd.SBDManager.update_configuration') - def test_adjust_sbd_delay_start(self, mock_update): - self.sbd_timeout_inst.sbd_delay_start_value_expected = 100 - self.sbd_timeout_inst.sbd_delay_start_value_from_config = "no" - self.sbd_timeout_inst.adjust_sbd_delay_start() - mock_update.assert_called_once_with({"SBD_DELAY_START": "100"}) - - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_adjust_systemd_start_timeout_no_delay_start_no(self, mock_get_sbd_value, mock_run): - mock_get_sbd_value.return_value = "no" - self.sbd_timeout_inst.adjust_systemd_start_timeout() - mock_run.assert_not_called() - @mock.patch('crmsh.utils.mkdirp') - @mock.patch('crmsh.utils.get_systemd_timeout_start_in_sec') - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_adjust_systemd_start_timeout_no_delay_start_return(self, mock_get_sbd_value, mock_run, mock_get_systemd_sec, mock_mkdirp): - mock_get_sbd_value.return_value = "10" - mock_run.return_value = "1min 30s" - mock_get_systemd_sec.return_value = 90 - self.sbd_timeout_inst.adjust_systemd_start_timeout() - mock_run.assert_called_once_with("systemctl show -p TimeoutStartUSec sbd --value") - mock_get_systemd_sec.assert_called_once_with("1min 30s") - mock_mkdirp.assert_not_called() +class TestSBDUtils(unittest.TestCase): - @mock.patch('crmsh.utils.cluster_run_cmd') - @mock.patch('crmsh.bootstrap.sync_file') - @mock.patch('crmsh.utils.str2file') - @mock.patch('crmsh.utils.mkdirp') - @mock.patch('crmsh.utils.get_systemd_timeout_start_in_sec') - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_adjust_systemd_start_timeout_no_delay_start(self, mock_get_sbd_value, mock_run, mock_get_systemd_sec, mock_mkdirp, mock_str2file, mock_csync2, mock_cluster_run): - mock_get_sbd_value.return_value = "100" - mock_run.return_value = "1min 30s" - mock_get_systemd_sec.return_value = 90 - self.sbd_timeout_inst.adjust_systemd_start_timeout() - mock_run.assert_called_once_with("systemctl show -p TimeoutStartUSec sbd --value") - mock_get_systemd_sec.assert_called_once_with("1min 30s") - mock_mkdirp.assert_called_once_with(bootstrap.SBD_SYSTEMD_DELAY_START_DIR) - mock_str2file.assert_called_once_with('[Service]\nTimeoutSec=120', '/etc/systemd/system/sbd.service.d/sbd_delay_start.conf') - mock_csync2.assert_called_once_with(bootstrap.SBD_SYSTEMD_DELAY_START_DIR) - mock_cluster_run.assert_called_once_with("systemctl daemon-reload") + @patch('crmsh.sh.cluster_shell') + def test_get_sbd_device_metadata_success(self, mock_cluster_shell): + mock_cluster_shell.return_value.get_stdout_or_raise_error.return_value = """ + UUID : 1234-5678 + Timeout (watchdog) : 5 + Timeout (msgwait) : 10 + """ + result = SBDUtils.get_sbd_device_metadata("/dev/sbd_device") + expected = {'uuid': '1234-5678', 'watchdog': 5, 'msgwait': 10} + self.assertEqual(result, expected) + + @patch('crmsh.sh.cluster_shell') + def test_get_sbd_device_metadata_exception(self, mock_cluster_shell): + mock_cluster_shell.return_value.get_stdout_or_raise_error.side_effect = Exception + result = SBDUtils.get_sbd_device_metadata("/dev/sbd_device") + self.assertEqual(result, {}) + + @patch('crmsh.sh.cluster_shell') + def test_get_sbd_device_metadata_timeout_only(self, mock_cluster_shell): + mock_cluster_shell.return_value.get_stdout_or_raise_error.return_value = """ + UUID : 1234-5678 + Timeout (watchdog) : 5 + Timeout (msgwait) : 10 + """ + result = SBDUtils.get_sbd_device_metadata("/dev/sbd_device", timeout_only=True) + expected = {'watchdog': 5, 'msgwait': 10} + self.assertNotIn('uuid', result) + self.assertEqual(result, expected) + + @patch('crmsh.sbd.SBDUtils.get_sbd_device_metadata') + def test_get_device_uuid_success(self, mock_get_sbd_device_metadata): + mock_get_sbd_device_metadata.return_value = {'uuid': '1234-5678'} + result = SBDUtils.get_device_uuid("/dev/sbd_device") + self.assertEqual(result, '1234-5678') + + @patch('crmsh.sbd.SBDUtils.get_sbd_device_metadata') + def test_get_device_uuid_no_uuid_found(self, mock_get_sbd_device_metadata): + mock_get_sbd_device_metadata.return_value = {} + with self.assertRaises(ValueError) as context: + SBDUtils.get_device_uuid("/dev/sbd_device") + self.assertTrue("Cannot find sbd device UUID for /dev/sbd_device" in str(context.exception)) + + @patch('crmsh.sbd.SBDUtils.get_device_uuid') + def test_compare_device_uuid_empty_node_list(self, mock_get_device_uuid): + result = SBDUtils.compare_device_uuid("/dev/sbd_device", []) + self.assertIsNone(result) + + @patch('crmsh.sbd.SBDUtils.get_device_uuid') + def test_compare_device_uuid_same_uuid(self, mock_get_device_uuid): + mock_get_device_uuid.return_value = '1234-5678' + SBDUtils.compare_device_uuid("/dev/sbd_device", ["node1", "node2"]) + + @patch('crmsh.sbd.SBDUtils.get_device_uuid') + def test_compare_device_uuid_different_uuid(self, mock_get_device_uuid): + mock_get_device_uuid.side_effect = lambda dev, node=None: '1234-5678' if node is None else '8765-4321' + with self.assertRaises(ValueError): + SBDUtils.compare_device_uuid("/dev/sbd_device", ["node1"]) - @mock.patch('crmsh.sbd.SBDTimeout.get_sbd_watchdog_timeout') - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_get_sbd_delay_start_sec_from_sysconfig_yes(self, mock_get_sbd_value, mock_get_sbd_timeout): - mock_get_sbd_value.return_value = "yes" - mock_get_sbd_timeout.return_value = 30 - assert sbd.SBDTimeout.get_sbd_delay_start_sec_from_sysconfig() == 60 - mock_get_sbd_value.assert_called_once_with("SBD_DELAY_START") + @patch('crmsh.utils.is_block_device') + @patch('crmsh.sbd.SBDUtils.compare_device_uuid') + def test_verify_sbd_device_exceeds_max(self, mock_compare_device_uuid, mock_is_block_device): + dev_list = [f"/dev/sbd_device_{i}" for i in range(SBDManager.SBD_DEVICE_MAX + 1)] + with self.assertRaises(ValueError): + SBDUtils.verify_sbd_device(dev_list) - @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config') - def test_get_sbd_delay_start_sec_from_sysconfig(self, mock_get_sbd_value): - mock_get_sbd_value.return_value = "30" - assert sbd.SBDTimeout.get_sbd_delay_start_sec_from_sysconfig() == 30 - mock_get_sbd_value.assert_called_once_with("SBD_DELAY_START") + @patch('crmsh.utils.is_block_device') + @patch('crmsh.sbd.SBDUtils.compare_device_uuid') + def test_verify_sbd_device_non_block(self, mock_compare_device_uuid, mock_is_block_device): + mock_is_block_device.return_value = False + with self.assertRaises(ValueError): + SBDUtils.verify_sbd_device(["/dev/not_a_block_device"]) + + @patch('crmsh.utils.is_block_device') + @patch('crmsh.sbd.SBDUtils.compare_device_uuid') + def test_verify_sbd_device_valid(self, mock_compare_device_uuid, mock_is_block_device): + mock_is_block_device.return_value = True + SBDUtils.verify_sbd_device(["/dev/sbd_device"], ["node1", "node2"]) + + @patch('crmsh.utils.parse_sysconfig') + def test_get_sbd_value_from_config(self, mock_parse_sysconfig): + mock_parse_sysconfig.return_value = {'SBD_DEVICE': '/dev/sbd_device'} + result = SBDUtils.get_sbd_value_from_config("SBD_DEVICE") + self.assertEqual(result, '/dev/sbd_device') + + @patch('crmsh.sbd.SBDUtils.get_sbd_value_from_config') + def test_get_sbd_device_from_config(self, mock_get_sbd_value_from_config): + mock_get_sbd_value_from_config.return_value = '/dev/sbd_device;/dev/another_sbd_device' + result = SBDUtils.get_sbd_device_from_config() + self.assertEqual(result, ['/dev/sbd_device', '/dev/another_sbd_device']) + + @patch('crmsh.sbd.SBDUtils.get_sbd_device_from_config') + @patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_is_using_diskless_sbd(self, mock_service_is_active, mock_get_sbd_device_from_config): + mock_get_sbd_device_from_config.return_value = [] + mock_service_is_active.return_value = True + result = SBDUtils.is_using_diskless_sbd() + self.assertTrue(result) + + @patch('crmsh.sbd.ShellUtils.get_stdout_stderr') + def test_has_sbd_device_already_initialized(self, mock_get_stdout_stderr): + mock_get_stdout_stderr.return_value = (0, '', '') + result = SBDUtils.has_sbd_device_already_initialized('/dev/sbd_device') + self.assertTrue(result) + + @patch('crmsh.bootstrap.confirm') + @patch('crmsh.sbd.SBDUtils.has_sbd_device_already_initialized') + def test_no_overwrite_device_check(self, mock_has_sbd_device_already_initialized, mock_confirm): + mock_has_sbd_device_already_initialized.return_value = True + mock_confirm.return_value = False + result = SBDUtils.no_overwrite_device_check('/dev/sbd_device') + self.assertTrue(result) + + @patch('crmsh.sbd.SBDUtils.get_sbd_device_metadata') + def test_check_devices_metadata_consistent_single_device(self, mock_get_sbd_device_metadata): + dev_list = ['/dev/sbd_device'] + result = SBDUtils.check_devices_metadata_consistent(dev_list) + self.assertTrue(result) + + @patch('crmsh.sbd.SBDUtils.get_sbd_device_metadata') + def test_check_devices_metadata_consistent_multiple_devices_consistent(self, mock_get_sbd_device_metadata): + dev_list = ['/dev/sbd_device1', '/dev/sbd_device2'] + mock_get_sbd_device_metadata.side_effect = ['metadata1', 'metadata1'] + result = SBDUtils.check_devices_metadata_consistent(dev_list) + self.assertTrue(result) + + @patch('crmsh.sbd.SBDUtils.get_sbd_device_metadata') + @patch('logging.Logger.warning') + def test_check_devices_metadata_consistent_multiple_devices_inconsistent(self, mock_logger_warning, mock_get_sbd_device_metadata): + dev_list = ['/dev/sbd_device1', '/dev/sbd_device2'] + mock_get_sbd_device_metadata.side_effect = ['metadata1', 'metadata2'] + result = SBDUtils.check_devices_metadata_consistent(dev_list) + self.assertFalse(result) + mock_logger_warning.assert_called() -class TestSBDManager(unittest.TestCase): +class TestSBDTimeout(unittest.TestCase): """ - Unitary tests for crmsh.sbd.SBDManager + Unitary tests for crmsh.sbd.SBDTimeout """ @classmethod @@ -303,10 +156,6 @@ def setUp(self): """ Test setUp. """ - self.sbd_inst = sbd.SBDManager(mock.Mock(sbd_devices=["/dev/sdb1", "/dev/sdc1"], diskless_sbd=False)) - self.sbd_inst_devices_gt_3 = sbd.SBDManager(mock.Mock(sbd_devices=["/dev/sdb1", "/dev/sdc1", "/dev/sdd1", "/dev/sde1"])) - self.sbd_inst_interactive = sbd.SBDManager(mock.Mock(sbd_devices=[], diskless_sbd=False)) - self.sbd_inst_diskless = sbd.SBDManager(mock.Mock(sbd_devices=[], diskless_sbd=True)) def tearDown(self): """ @@ -318,578 +167,3 @@ def tearDownClass(cls): """ Global tearDown. """ - - @mock.patch('logging.Logger.warning') - def test_get_sbd_device_interactive_yes_to_all(self, mock_warn): - self.sbd_inst._context = mock.Mock(yes_to_all=True) - self.sbd_inst._get_sbd_device_interactive() - mock_warn.assert_called_once_with(sbd.SBDManager.SBD_WARNING) - - @mock.patch('crmsh.bootstrap.confirm') - @mock.patch('logging.Logger.info') - @mock.patch('logging.Logger.warning') - def test_get_sbd_device_interactive_not_confirm(self, mock_warn, mock_status, mock_confirm): - self.sbd_inst._context.yes_to_all = False - mock_confirm.return_value = False - self.sbd_inst._get_sbd_device_interactive() - mock_status.assert_called_once_with(sbd.SBDManager.SBD_STATUS_DESCRIPTION) - mock_warn.assert_called_once_with("Not configuring SBD - STONITH will be disabled.") - - @mock.patch('crmsh.sbd.SBDManager._no_overwrite_check') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.bootstrap.confirm') - @mock.patch('logging.Logger.info') - def test_get_sbd_device_interactive_already_configured(self, mock_status, mock_confirm, mock_from_config, mock_no_overwrite): - self.sbd_inst._context = mock.Mock(yes_to_all=False) - mock_confirm.return_value = True - mock_from_config.return_value = ["/dev/sda1"] - mock_no_overwrite.return_value = True - - res = self.sbd_inst._get_sbd_device_interactive() - self.assertEqual(res, ["/dev/sda1"]) - - mock_status.assert_called_once_with(sbd.SBDManager.SBD_STATUS_DESCRIPTION) - mock_confirm.assert_has_calls([ - mock.call("Do you wish to use SBD?"), - ]) - mock_status.assert_called_once_with(sbd.SBDManager.SBD_STATUS_DESCRIPTION) - mock_from_config.assert_called_once_with() - - @mock.patch('crmsh.bootstrap.prompt_for_string') - @mock.patch('crmsh.sbd.SBDManager._no_overwrite_check') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.bootstrap.confirm') - @mock.patch('logging.Logger.info') - def test_get_sbd_device_interactive_diskless(self, mock_status, mock_confirm, mock_from_config, mock_no_overwrite, mock_prompt): - self.sbd_inst._context = mock.Mock(yes_to_all=False) - mock_confirm.return_value = True - mock_no_overwrite.return_value = False - mock_from_config.return_value = [] - mock_prompt.return_value = "none" - - self.sbd_inst._get_sbd_device_interactive() - - mock_status.assert_called_once_with(sbd.SBDManager.SBD_STATUS_DESCRIPTION) - mock_from_config.assert_called_once_with() - mock_prompt.assert_called_once_with('Path to storage device (e.g. /dev/disk/by-id/...), or "none" for diskless sbd, use ";" as separator for multi path', 'none|\\/.*') - - @mock.patch('crmsh.bootstrap.prompt_for_string') - @mock.patch('crmsh.sbd.SBDManager._no_overwrite_check') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.bootstrap.confirm') - @mock.patch('logging.Logger.info') - def test_get_sbd_device_interactive_null_and_diskless(self, mock_status, mock_confirm, mock_from_config, mock_no_overwrite, mock_prompt): - self.sbd_inst._context = mock.Mock(yes_to_all=False) - mock_confirm.return_value = True - mock_no_overwrite.return_value = False - mock_from_config.return_value = [] - mock_prompt.return_value = "none" - - self.sbd_inst._get_sbd_device_interactive() - - mock_status.assert_called_once_with(sbd.SBDManager.SBD_STATUS_DESCRIPTION) - mock_confirm.assert_called_once_with("Do you wish to use SBD?") - mock_from_config.assert_called_once_with() - mock_prompt.assert_has_calls([ - mock.call('Path to storage device (e.g. /dev/disk/by-id/...), or "none" for diskless sbd, use ";" as separator for multi path', 'none|\\/.*') - ]) - - @mock.patch('crmsh.utils.re_split_string') - @mock.patch('logging.Logger.warning') - @mock.patch('logging.Logger.error') - @mock.patch('crmsh.sbd.SBDManager._verify_sbd_device') - @mock.patch('crmsh.bootstrap.prompt_for_string') - @mock.patch('crmsh.sbd.SBDManager._no_overwrite_check') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.bootstrap.confirm') - @mock.patch('logging.Logger.info') - def test_get_sbd_device_interactive(self, mock_status, mock_confirm, mock_from_config, mock_no_overwrite, mock_prompt, mock_verify, mock_error_msg, mock_warn, mock_split): - self.sbd_inst._context = mock.Mock(yes_to_all=False) - mock_confirm.side_effect = [True, False, True] - mock_from_config.return_value = [] - mock_no_overwrite.return_value = False - mock_prompt.side_effect = ["/dev/test1", "/dev/sda1", "/dev/sdb1"] - mock_split.side_effect = [["/dev/test1"], ["/dev/sda1"], ["/dev/sdb1"]] - mock_verify.side_effect = [ValueError("/dev/test1 error"), None, None] - - res = self.sbd_inst._get_sbd_device_interactive() - self.assertEqual(res, ["/dev/sdb1"]) - - mock_status.assert_called_once_with(sbd.SBDManager.SBD_STATUS_DESCRIPTION) - mock_confirm.assert_has_calls([ - mock.call("Do you wish to use SBD?"), - mock.call("Are you sure you wish to use this device?") - ]) - mock_from_config.assert_called_once_with() - mock_error_msg.assert_called_once_with("/dev/test1 error") - mock_warn.assert_has_calls([ - mock.call("All data on /dev/sda1 will be destroyed!"), - mock.call("All data on /dev/sdb1 will be destroyed!") - ]) - mock_prompt.assert_has_calls([ - mock.call('Path to storage device (e.g. /dev/disk/by-id/...), or "none" for diskless sbd, use ";" as separator for multi path', 'none|\\/.*') for x in range(3) - ]) - mock_split.assert_has_calls([ - mock.call(sbd.SBDManager.PARSE_RE, "/dev/test1"), - mock.call(sbd.SBDManager.PARSE_RE, "/dev/sda1"), - mock.call(sbd.SBDManager.PARSE_RE, "/dev/sdb1"), - ]) - - def test_verify_sbd_device_gt_3(self): - assert self.sbd_inst_devices_gt_3.sbd_devices_input == ["/dev/sdb1", "/dev/sdc1", "/dev/sdd1", "/dev/sde1"] - dev_list = self.sbd_inst_devices_gt_3.sbd_devices_input - with self.assertRaises(ValueError) as err: - self.sbd_inst_devices_gt_3._verify_sbd_device(dev_list) - self.assertEqual("Maximum number of SBD device is 3", str(err.exception)) - - @mock.patch('crmsh.sbd.SBDManager._compare_device_uuid') - @mock.patch('crmsh.utils.is_block_device') - def test_verify_sbd_device_not_block(self, mock_block_device, mock_compare): - assert self.sbd_inst.sbd_devices_input == ["/dev/sdb1", "/dev/sdc1"] - dev_list = self.sbd_inst.sbd_devices_input - mock_block_device.side_effect = [True, False] - - with self.assertRaises(ValueError) as err: - self.sbd_inst._verify_sbd_device(dev_list) - self.assertEqual("/dev/sdc1 doesn't look like a block device", str(err.exception)) - - mock_block_device.assert_has_calls([mock.call("/dev/sdb1"), mock.call("/dev/sdc1")]) - mock_compare.assert_called_once_with("/dev/sdb1", []) - - @mock.patch('crmsh.sbd.SBDManager._verify_sbd_device') - def test_get_sbd_device_from_option(self, mock_verify): - self.sbd_inst._get_sbd_device() - mock_verify.assert_called_once_with(['/dev/sdb1', '/dev/sdc1']) - - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_interactive') - def test_get_sbd_device_from_interactive(self, mock_interactive): - mock_interactive.return_value = ["/dev/sdb1", "/dev/sdc1"] - self.sbd_inst_interactive._get_sbd_device() - mock_interactive.assert_called_once_with() - - def test_get_sbd_device_diskless(self): - self.sbd_inst_diskless._get_sbd_device() - - @mock.patch('crmsh.sbd.SBDTimeout') - @mock.patch('logging.Logger.info') - def test_initialize_sbd_return(self, mock_info, mock_sbd_timeout): - mock_inst = mock.Mock() - mock_sbd_timeout.return_value = mock_inst - self.sbd_inst_diskless._context = mock.Mock(profiles_dict={}) - self.sbd_inst_diskless._initialize_sbd() - mock_info.assert_called_once_with("Configuring diskless SBD") - mock_inst.initialize_timeout.assert_called_once_with() - - @mock.patch('crmsh.utils.fatal') - @mock.patch('crmsh.bootstrap.invoke') - @mock.patch('crmsh.sbd.SBDTimeout') - @mock.patch('logging.Logger.info') - def test_initialize_sbd(self, mock_info, mock_sbd_timeout, mock_invoke, mock_error): - mock_inst = mock.Mock(sbd_msgwait=10, sbd_watchdog_timeout=5) - mock_sbd_timeout.return_value = mock_inst - mock_inst.set_sbd_watchdog_timeout = mock.Mock() - mock_inst.set_sbd_msgwait = mock.Mock() - self.sbd_inst._sbd_devices = ["/dev/sdb1", "/dev/sdc1"] - mock_invoke.side_effect = [(True, None, None), (False, None, "error")] - mock_error.side_effect = ValueError - - with self.assertRaises(ValueError): - self.sbd_inst._initialize_sbd() - - mock_invoke.assert_has_calls([ - mock.call("sbd -4 10 -1 5 -d /dev/sdb1 create"), - mock.call("sbd -4 10 -1 5 -d /dev/sdc1 create") - ]) - mock_error.assert_called_once_with("Failed to initialize SBD device /dev/sdc1: error") - - @mock.patch('crmsh.bootstrap.sync_file') - @mock.patch('crmsh.utils.sysconfig_set') - @mock.patch('shutil.copyfile') - def test_update_configuration(self, mock_copy, mock_sysconfig, mock_update): - self.sbd_inst._sbd_devices = ["/dev/sdb1", "/dev/sdc1"] - self.sbd_inst._watchdog_inst = mock.Mock(watchdog_device_name="/dev/watchdog") - self.sbd_inst.timeout_inst = mock.Mock(sbd_watchdog_timeout=15) - - self.sbd_inst._update_sbd_configuration() - - mock_copy.assert_called_once_with("/usr/share/fillup-templates/sysconfig.sbd", "/etc/sysconfig/sbd") - mock_sysconfig.assert_called_once_with("/etc/sysconfig/sbd", SBD_WATCHDOG_DEV='/dev/watchdog', SBD_DEVICE='/dev/sdb1;/dev/sdc1', SBD_WATCHDOG_TIMEOUT="15") - mock_update.assert_called_once_with("/etc/sysconfig/sbd") - - @mock.patch('crmsh.bootstrap.utils.parse_sysconfig') - def test_get_sbd_device_from_config_none(self, mock_parse): - mock_parse_inst = mock.Mock() - mock_parse.return_value = mock_parse_inst - mock_parse_inst.get.return_value = None - - res = self.sbd_inst._get_sbd_device_from_config() - assert res == [] - - mock_parse.assert_called_once_with("/etc/sysconfig/sbd") - mock_parse_inst.get.assert_called_once_with("SBD_DEVICE") - - @mock.patch('crmsh.utils.re_split_string') - @mock.patch('crmsh.bootstrap.utils.parse_sysconfig') - def test_get_sbd_device_from_config(self, mock_parse, mock_split): - mock_parse_inst = mock.Mock() - mock_parse.return_value = mock_parse_inst - mock_parse_inst.get.return_value = "/dev/sdb1;/dev/sdc1" - mock_split.return_value = ["/dev/sdb1", "/dev/sdc1"] - - res = self.sbd_inst._get_sbd_device_from_config() - assert res == ["/dev/sdb1", "/dev/sdc1"] - - mock_parse.assert_called_once_with("/etc/sysconfig/sbd") - mock_parse_inst.get.assert_called_once_with("SBD_DEVICE") - mock_split.assert_called_once_with(sbd.SBDManager.PARSE_RE, "/dev/sdb1;/dev/sdc1") - - @mock.patch('logging.Logger.warning') - @mock.patch('crmsh.utils.get_quorum_votes_dict') - def test_warn_diskless_sbd_diskless(self, mock_vote, mock_warn): - self.sbd_inst_diskless._context = mock.Mock(cluster_is_running=False) - self.sbd_inst_diskless._warn_diskless_sbd() - mock_vote.assert_not_called() - mock_warn.assert_called_once_with(sbd.SBDManager.DISKLESS_SBD_WARNING) - - @mock.patch('logging.Logger.warning') - @mock.patch('crmsh.utils.get_quorum_votes_dict') - def test_warn_diskless_sbd_peer(self, mock_vote, mock_warn): - mock_vote.return_value = {'Expected': '1'} - self.sbd_inst_diskless._warn_diskless_sbd("node2") - mock_vote.assert_called_once_with("node2") - mock_warn.assert_called_once_with(sbd.SBDManager.DISKLESS_SBD_WARNING) - - @mock.patch('crmsh.utils.package_is_installed') - def test_sbd_init_not_installed(self, mock_package): - mock_package.return_value = False - self.sbd_inst.sbd_init() - mock_package.assert_called_once_with("sbd") - - @mock.patch('crmsh.bootstrap.invoke') - @mock.patch('crmsh.sbd.SBDManager._update_sbd_configuration') - @mock.patch('crmsh.sbd.SBDManager._initialize_sbd') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device') - @mock.patch('crmsh.watchdog.Watchdog') - @mock.patch('crmsh.utils.package_is_installed') - def test_sbd_init_return(self, mock_package, mock_watchdog, mock_get_device, mock_initialize, mock_update, mock_invoke): - mock_package.return_value = True - self.sbd_inst._sbd_devices = None - self.sbd_inst.diskless_sbd = False - self.sbd_inst._context = mock.Mock(watchdog=None) - mock_watchdog_inst = mock.Mock() - mock_watchdog.return_value = mock_watchdog_inst - mock_watchdog_inst.init_watchdog = mock.Mock() - - self.sbd_inst.sbd_init() - - mock_package.assert_called_once_with("sbd") - mock_get_device.assert_called_once_with() - mock_initialize.assert_not_called() - mock_update.assert_not_called() - mock_watchdog.assert_called_once_with(_input=None) - mock_watchdog_inst.init_watchdog.assert_called_once_with() - mock_invoke.assert_called_once_with("systemctl disable sbd.service") - - @mock.patch('crmsh.sbd.SBDManager._enable_sbd_service') - @mock.patch('crmsh.sbd.SBDManager._warn_diskless_sbd') - @mock.patch('crmsh.sbd.SBDManager._update_sbd_configuration') - @mock.patch('crmsh.sbd.SBDManager._initialize_sbd') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device') - @mock.patch('crmsh.watchdog.Watchdog') - @mock.patch('crmsh.utils.package_is_installed') - def test_sbd_init(self, mock_package, mock_watchdog, mock_get_device, mock_initialize, mock_update, mock_warn, mock_enable_sbd): - mock_package.return_value = True - self.sbd_inst_diskless._context = mock.Mock(watchdog=None) - mock_watchdog_inst = mock.Mock() - mock_watchdog.return_value = mock_watchdog_inst - mock_watchdog_inst.init_watchdog = mock.Mock() - self.sbd_inst_diskless.sbd_init() - - mock_package.assert_called_once_with("sbd") - mock_get_device.assert_called_once_with() - mock_initialize.assert_called_once_with() - mock_update.assert_called_once_with() - mock_watchdog.assert_called_once_with(_input=None) - mock_watchdog_inst.init_watchdog.assert_called_once_with() - mock_warn.assert_called_once_with() - mock_enable_sbd.assert_called_once_with() - - @mock.patch('crmsh.sbd.SBDManager.configure_sbd_resource_and_properties') - @mock.patch('crmsh.bootstrap.wait_for_cluster') - @mock.patch('crmsh.utils.cluster_run_cmd') - @mock.patch('logging.Logger.info') - @mock.patch('crmsh.xmlutil.CrmMonXmlParser') - def test_restart_cluster_on_needed_no_ra_running(self, mock_parser, mock_status, mock_cluster_run, mock_wait, mock_config_sbd_ra): - mock_parser().is_any_resource_running.return_value = False - self.sbd_inst._restart_cluster_and_configure_sbd_ra() - mock_status.assert_called_once_with("Restarting cluster service") - mock_cluster_run.assert_called_once_with("crm cluster restart") - mock_wait.assert_called_once_with() - mock_config_sbd_ra.assert_called_once_with() - - @mock.patch('crmsh.sbd.SBDTimeout.get_stonith_timeout') - @mock.patch('logging.Logger.warning') - @mock.patch('crmsh.xmlutil.CrmMonXmlParser') - def test_restart_cluster_on_needed_diskless(self, mock_parser, mock_warn, mock_get_timeout): - mock_parser().is_any_resource_running.return_value = True - mock_get_timeout.return_value = 60 - self.sbd_inst_diskless.timeout_inst = mock.Mock(stonith_watchdog_timeout=-1) - self.sbd_inst_diskless._restart_cluster_and_configure_sbd_ra() - mock_warn.assert_has_calls([ - mock.call("To start sbd.service, need to restart cluster service manually on each node"), - mock.call("Then run \"crm configure property stonith-enabled=true stonith-watchdog-timeout=-1 stonith-timeout=60\" on any node") - ]) - - @mock.patch('crmsh.sbd.SBDManager.configure_sbd_resource_and_properties') - @mock.patch('logging.Logger.warning') - @mock.patch('crmsh.xmlutil.CrmMonXmlParser') - def test_restart_cluster_on_needed(self, mock_parser, mock_warn, mock_config_sbd_ra): - mock_parser().is_any_resource_running.return_value = True - self.sbd_inst._restart_cluster_and_configure_sbd_ra() - mock_warn.assert_has_calls([ - mock.call("To start sbd.service, need to restart cluster service manually on each node"), - ]) - - @mock.patch('crmsh.bootstrap.invoke') - def test_enable_sbd_service_init(self, mock_invoke): - self.sbd_inst._context = mock.Mock(cluster_is_running=False) - self.sbd_inst._enable_sbd_service() - mock_invoke.assert_called_once_with("systemctl enable sbd.service") - - @mock.patch('crmsh.sbd.SBDManager._restart_cluster_and_configure_sbd_ra') - @mock.patch('crmsh.utils.cluster_run_cmd') - def test_enable_sbd_service_restart(self, mock_cluster_run, mock_restart): - self.sbd_inst._context = mock.Mock(cluster_is_running=True) - self.sbd_inst._enable_sbd_service() - mock_cluster_run.assert_has_calls([ - mock.call("systemctl enable sbd.service"), - ]) - mock_restart.assert_called_once_with() - - @mock.patch('crmsh.utils.package_is_installed') - def test_configure_sbd_resource_and_properties_not_installed(self, mock_package): - mock_package.return_value = False - self.sbd_inst.configure_sbd_resource_and_properties() - mock_package.assert_called_once_with("sbd") - - @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - @mock.patch('crmsh.sbd.SBDTimeout.adjust_sbd_timeout_related_cluster_configuration') - @mock.patch('crmsh.utils.set_property') - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - @mock.patch('crmsh.xmlutil.CrmMonXmlParser') - @mock.patch('crmsh.service_manager.ServiceManager.service_is_enabled') - @mock.patch('crmsh.utils.package_is_installed') - def test_configure_sbd_resource_and_properties( - self, - mock_package, mock_enabled, mock_parser, mock_run, mock_set_property, sbd_adjust, mock_is_active, - ): - mock_package.return_value = True - mock_enabled.return_value = True - mock_parser().is_resource_configured.return_value = False - mock_is_active.return_value = False - self.sbd_inst._context = mock.Mock(cluster_is_running=True) - self.sbd_inst._get_sbd_device_from_config = mock.Mock() - self.sbd_inst._get_sbd_device_from_config.return_value = ["/dev/sda1"] - self.sbd_inst._sbd_devices = ["/dev/sda1"] - - self.sbd_inst.configure_sbd_resource_and_properties() - - mock_package.assert_called_once_with("sbd") - mock_enabled.assert_called_once_with("sbd.service") - mock_run.assert_called_once_with("crm configure primitive {} {} params devices=\"/dev/sda1\"".format(sbd.SBDManager.SBD_RA_ID, sbd.SBDManager.SBD_RA)) - mock_set_property.assert_called_once_with("stonith-enabled", "true") - - @mock.patch('crmsh.utils.package_is_installed') - def test_join_sbd_config_not_installed(self, mock_package): - mock_package.return_value = False - self.sbd_inst.join_sbd("alice", "node1") - mock_package.assert_called_once_with("sbd") - - @mock.patch('crmsh.bootstrap.invoke') - @mock.patch('os.path.exists') - @mock.patch('crmsh.utils.package_is_installed') - def test_join_sbd_config_not_exist(self, mock_package, mock_exists, mock_invoke): - mock_package.return_value = True - mock_exists.return_value = False - self.sbd_inst.join_sbd("alice", "node1") - mock_package.assert_called_once_with("sbd") - mock_exists.assert_called_once_with("/etc/sysconfig/sbd") - mock_invoke.assert_called_once_with("systemctl disable sbd.service") - - @mock.patch('crmsh.bootstrap.invoke') - @mock.patch('crmsh.service_manager.ServiceManager.service_is_enabled') - @mock.patch('os.path.exists') - @mock.patch('crmsh.utils.package_is_installed') - def test_join_sbd_config_disabled(self, mock_package, mock_exists, mock_enabled, mock_invoke): - mock_package.return_value = True - mock_exists.return_value = True - mock_enabled.return_value = False - - self.sbd_inst.join_sbd("alice", "node1") - - mock_package.assert_called_once_with("sbd") - mock_exists.assert_called_once_with("/etc/sysconfig/sbd") - mock_invoke.assert_called_once_with("systemctl disable sbd.service") - mock_enabled.assert_called_once_with("sbd.service", "node1") - - @mock.patch('logging.Logger.info') - @mock.patch('crmsh.sbd.SBDManager._verify_sbd_device') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.watchdog.Watchdog') - @mock.patch('crmsh.bootstrap.invoke') - @mock.patch('crmsh.service_manager.ServiceManager.service_is_enabled') - @mock.patch('os.path.exists') - @mock.patch('crmsh.utils.package_is_installed') - def test_join_sbd(self, mock_package, mock_exists, mock_enabled, mock_invoke, mock_watchdog, mock_get_device, mock_verify, mock_status): - mock_package.return_value = True - mock_exists.return_value = True - mock_enabled.return_value = True - mock_get_device.return_value = ["/dev/sdb1"] - mock_watchdog_inst = mock.Mock() - mock_watchdog.return_value = mock_watchdog_inst - mock_watchdog_inst.join_watchdog = mock.Mock() - - self.sbd_inst.join_sbd("alice", "node1") - - mock_package.assert_called_once_with("sbd") - mock_exists.assert_called_once_with("/etc/sysconfig/sbd") - mock_invoke.assert_called_once_with("systemctl enable sbd.service") - mock_get_device.assert_called_once_with() - mock_verify.assert_called_once_with(["/dev/sdb1"], ["node1"]) - mock_enabled.assert_called_once_with("sbd.service", "node1") - mock_status.assert_called_once_with("Got SBD configuration") - mock_watchdog.assert_called_once_with(remote_user="alice", peer_host="node1") - mock_watchdog_inst.join_watchdog.assert_called_once_with() - - @mock.patch('crmsh.utils.sysconfig_set') - @mock.patch('logging.Logger.info') - @mock.patch('crmsh.sbd.SBDManager._warn_diskless_sbd') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.watchdog.Watchdog') - @mock.patch('crmsh.bootstrap.invoke') - @mock.patch('crmsh.service_manager.ServiceManager.service_is_enabled') - @mock.patch('os.path.exists') - @mock.patch('crmsh.utils.package_is_installed') - def test_join_sbd_diskless(self, mock_package, mock_exists, mock_enabled, mock_invoke, mock_watchdog, mock_get_device, mock_warn, mock_status, mock_set): - mock_package.return_value = True - mock_exists.return_value = True - mock_enabled.return_value = True - mock_get_device.return_value = [] - mock_watchdog_inst = mock.Mock() - mock_watchdog.return_value = mock_watchdog_inst - mock_watchdog_inst.join_watchdog = mock.Mock() - - self.sbd_inst.join_sbd("alice", "node1") - - mock_package.assert_called_once_with("sbd") - mock_exists.assert_called_once_with("/etc/sysconfig/sbd") - mock_invoke.assert_called_once_with("systemctl enable sbd.service") - mock_get_device.assert_called_once_with() - mock_warn.assert_called_once_with("node1") - mock_enabled.assert_called_once_with("sbd.service", "node1") - mock_status.assert_called_once_with("Got diskless SBD configuration") - mock_watchdog.assert_called_once_with(remote_user="alice", peer_host="node1") - mock_watchdog_inst.join_watchdog.assert_called_once_with() - - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - def test_verify_sbd_device_classmethod_exception(self, mock_get_config): - mock_get_config.return_value = [] - with self.assertRaises(ValueError) as err: - sbd.SBDManager.verify_sbd_device() - self.assertEqual("No sbd device configured", str(err.exception)) - mock_get_config.assert_called_once_with() - - @mock.patch('crmsh.sbd.SBDManager._verify_sbd_device') - @mock.patch('crmsh.utils.list_cluster_nodes_except_me') - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - def test_verify_sbd_device_classmethod(self, mock_get_config, mock_list_nodes, mock_verify): - mock_get_config.return_value = ["/dev/sda1"] - mock_list_nodes.return_value = ["node1"] - sbd.SBDManager.verify_sbd_device() - mock_get_config.assert_called_once_with() - mock_verify.assert_called_once_with(["/dev/sda1"], ["node1"]) - - @mock.patch('crmsh.sbd.SBDManager._get_device_uuid') - def test_compare_device_uuid_return(self, mock_get_uuid): - self.sbd_inst._compare_device_uuid("/dev/sdb1", None) - mock_get_uuid.assert_not_called() - - @mock.patch('crmsh.sbd.SBDManager._get_device_uuid') - def test_compare_device_uuid(self, mock_get_uuid): - mock_get_uuid.side_effect = ["1234", "5678"] - with self.assertRaises(ValueError) as err: - self.sbd_inst._compare_device_uuid("/dev/sdb1", ["node1"]) - self.assertEqual("Device /dev/sdb1 doesn't have the same UUID with node1", str(err.exception)) - mock_get_uuid.assert_has_calls([mock.call("/dev/sdb1"), mock.call("/dev/sdb1", "node1")]) - - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - def test_get_device_uuid_not_match(self, mock_run): - mock_run.return_value = "data" - with self.assertRaises(ValueError) as err: - self.sbd_inst._get_device_uuid("/dev/sdb1") - self.assertEqual("Cannot find sbd device UUID for /dev/sdb1", str(err.exception)) - mock_run.assert_called_once_with("sbd -d /dev/sdb1 dump", None) - - @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') - def test_get_device_uuid(self, mock_run): - output = """ - ==Dumping header on disk /dev/sda1 - Header version : 2.1 - UUID : a2e9a92c-cc72-4ef9-ac55-ccc342f3546b - Number of slots : 255 - Sector size : 512 - Timeout (watchdog) : 5 - Timeout (allocate) : 2 - Timeout (loop) : 1 - Timeout (msgwait) : 10 - ==Header on disk /dev/sda1 is dumped - """ - mock_run.return_value = output - res = self.sbd_inst._get_device_uuid("/dev/sda1", node="node1") - self.assertEqual(res, "a2e9a92c-cc72-4ef9-ac55-ccc342f3546b") - mock_run.assert_called_once_with("sbd -d /dev/sda1 dump", "node1") - - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') - @mock.patch('crmsh.bootstrap.Context') - def test_is_using_diskless_sbd_true(self, mock_context, mock_is_active, mock_get_sbd): - context_inst = mock.Mock() - mock_context.return_value = context_inst - mock_get_sbd.return_value = [] - mock_is_active.return_value = True - assert sbd.SBDManager.is_using_diskless_sbd() is True - mock_context.assert_called_once_with() - mock_get_sbd.assert_called_once_with() - mock_is_active.assert_called_once_with("sbd.service") - - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.bootstrap.Context') - def test_is_using_diskless_sbd_false(self, mock_context, mock_get_sbd): - context_inst = mock.Mock() - mock_context.return_value = context_inst - mock_get_sbd.return_value = ["/dev/sda1"] - assert sbd.SBDManager.is_using_diskless_sbd() is False - mock_context.assert_called_once_with() - mock_get_sbd.assert_called_once_with() - - @mock.patch('crmsh.sbd.SBDManager._get_sbd_device_from_config') - @mock.patch('crmsh.bootstrap.Context') - def test_get_sbd_device_from_config_classmethod(self, mock_context, mock_get_sbd): - context_inst = mock.Mock() - mock_context.return_value = context_inst - mock_get_sbd.return_value = ["/dev/sda1"] - assert sbd.SBDManager.get_sbd_device_from_config() == ["/dev/sda1"] - mock_context.assert_called_once_with() - mock_get_sbd.assert_called_once_with() - - @mock.patch('crmsh.bootstrap.sync_file') - @mock.patch('crmsh.utils.sysconfig_set') - def test_update_configuration_static(self, mock_config_set, mock_csync2): - sbd_config_dict = { - "SBD_PACEMAKER": "yes", - "SBD_STARTMODE": "always", - "SBD_DELAY_START": "no", - } - self.sbd_inst.update_configuration(sbd_config_dict) - mock_config_set.assert_called_once_with(bootstrap.SYSCONFIG_SBD, **sbd_config_dict) - mock_csync2.assert_called_once_with(bootstrap.SYSCONFIG_SBD) diff --git a/test/unittests/test_utils.py b/test/unittests/test_utils.py index 997aad6b7..83986fc82 100644 --- a/test/unittests/test_utils.py +++ b/test/unittests/test_utils.py @@ -945,7 +945,7 @@ def test_has_disk_mounted(mock_run): mock_run.assert_called_once_with("mount") -@mock.patch('crmsh.sbd.SBDManager.is_using_diskless_sbd') +@mock.patch('crmsh.sbd.SBDUtils.is_using_diskless_sbd') @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') def test_has_stonith_running(mock_run, mock_diskless): mock_run.return_value = """ diff --git a/test/unittests/test_watchdog.py b/test/unittests/test_watchdog.py index 957f21ffb..0c505a2e3 100644 --- a/test/unittests/test_watchdog.py +++ b/test/unittests/test_watchdog.py @@ -5,6 +5,7 @@ except ImportError: import mock +from crmsh import sbd from crmsh import watchdog from crmsh import bootstrap from crmsh import constants @@ -46,7 +47,7 @@ def test_watchdog_device_name(self): @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') def test_verify_watchdog_device_ignore_error(self, mock_run): mock_run.return_value = (1, None, "error") - res = self.watchdog_inst._verify_watchdog_device("/dev/watchdog", True) + res = self.watchdog_inst.verify_watchdog_device("/dev/watchdog", True) self.assertEqual(res, False) mock_run.assert_called_once_with("wdctl /dev/watchdog") @@ -56,21 +57,21 @@ def test_verify_watchdog_device_error(self, mock_run, mock_error): mock_run.return_value = (1, None, "error") mock_error.side_effect = ValueError with self.assertRaises(ValueError) as err: - self.watchdog_inst._verify_watchdog_device("/dev/watchdog") + self.watchdog_inst.verify_watchdog_device("/dev/watchdog") mock_error.assert_called_once_with("Invalid watchdog device /dev/watchdog: error") mock_run.assert_called_once_with("wdctl /dev/watchdog") @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') def test_verify_watchdog_device(self, mock_run): mock_run.return_value = (0, None, None) - res = self.watchdog_inst._verify_watchdog_device("/dev/watchdog") + res = self.watchdog_inst.verify_watchdog_device("/dev/watchdog") self.assertEqual(res, True) - @mock.patch('crmsh.watchdog.invoke') - def test_load_watchdog_driver(self, mock_invoke): + @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') + def test_load_watchdog_driver(self, mock_run): self.watchdog_inst._load_watchdog_driver("softdog") - mock_invoke.assert_has_calls([ - mock.call("echo softdog > /etc/modules-load.d/watchdog.conf"), + mock_run.assert_has_calls([ + mock.call(f"echo softdog > {watchdog.Watchdog.WATCHDOG_CFG}"), mock.call("systemctl restart systemd-modules-load") ]) @@ -79,9 +80,9 @@ def test_get_watchdog_device_from_sbd_config(self, mock_parse): mock_parse_inst = mock.Mock() mock_parse.return_value = mock_parse_inst mock_parse_inst.get.return_value = "/dev/watchdog" - res = self.watchdog_inst._get_watchdog_device_from_sbd_config() + res = self.watchdog_inst.get_watchdog_device_from_sbd_config() self.assertEqual(res, "/dev/watchdog") - mock_parse.assert_called_once_with(bootstrap.SYSCONFIG_SBD) + mock_parse.assert_called_once_with(sbd.SBDManager.SYSCONFIG_SBD) @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') def test_driver_is_loaded(self, mock_run): @@ -128,7 +129,7 @@ def test_set_watchdog_info(self, mock_run): self.watchdog_inst._set_watchdog_info() self.assertEqual(self.watchdog_inst._watchdog_info_dict, {'/dev/watchdog': 'softdog', '/dev/watchdog0': 'softdog', '/dev/watchdog1': 'iTCO_wdt'}) - @mock.patch('crmsh.watchdog.Watchdog._verify_watchdog_device') + @mock.patch('crmsh.watchdog.Watchdog.verify_watchdog_device') def test_get_device_through_driver_none(self, mock_verify): self.watchdog_inst._watchdog_info_dict = {'/dev/watchdog': 'softdog', '/dev/watchdog0': 'softdog', '/dev/watchdog1': 'iTCO_wdt'} mock_verify.return_value = False @@ -136,7 +137,7 @@ def test_get_device_through_driver_none(self, mock_verify): self.assertEqual(res, None) mock_verify.assert_called_once_with("/dev/watchdog1") - @mock.patch('crmsh.watchdog.Watchdog._verify_watchdog_device') + @mock.patch('crmsh.watchdog.Watchdog.verify_watchdog_device') def test_get_device_through_driver(self, mock_verify): self.watchdog_inst._watchdog_info_dict = {'/dev/watchdog': 'softdog', '/dev/watchdog0': 'softdog', '/dev/watchdog1': 'iTCO_wdt'} mock_verify.return_value = True @@ -187,7 +188,7 @@ def test_get_first_unused_device_none(self): res = self.watchdog_inst._get_first_unused_device() self.assertEqual(res, None) - @mock.patch('crmsh.watchdog.Watchdog._verify_watchdog_device') + @mock.patch('crmsh.watchdog.Watchdog.verify_watchdog_device') def test_get_first_unused_device(self, mock_verify): mock_verify.return_value = True self.watchdog_inst._watchdog_info_dict = {'/dev/watchdog': 'softdog', '/dev/watchdog0': 'softdog', '/dev/watchdog1': 'iTCO_wdt'} @@ -196,8 +197,8 @@ def test_get_first_unused_device(self, mock_verify): mock_verify.assert_called_once_with("/dev/watchdog", ignore_error=True) @mock.patch('crmsh.watchdog.Watchdog._get_first_unused_device') - @mock.patch('crmsh.watchdog.Watchdog._verify_watchdog_device') - @mock.patch('crmsh.watchdog.Watchdog._get_watchdog_device_from_sbd_config') + @mock.patch('crmsh.watchdog.Watchdog.verify_watchdog_device') + @mock.patch('crmsh.watchdog.Watchdog.get_watchdog_device_from_sbd_config') def test_set_input_from_config(self, mock_from_config, mock_verify, mock_first): mock_from_config.return_value = "/dev/watchdog" mock_verify.return_value = True @@ -206,8 +207,8 @@ def test_set_input_from_config(self, mock_from_config, mock_verify, mock_first): mock_from_config.assert_called_once_with() @mock.patch('crmsh.watchdog.Watchdog._get_first_unused_device') - @mock.patch('crmsh.watchdog.Watchdog._verify_watchdog_device') - @mock.patch('crmsh.watchdog.Watchdog._get_watchdog_device_from_sbd_config') + @mock.patch('crmsh.watchdog.Watchdog.verify_watchdog_device') + @mock.patch('crmsh.watchdog.Watchdog.get_watchdog_device_from_sbd_config') def test_set_input(self, mock_from_config, mock_verify, mock_first): mock_from_config.return_value = None mock_first.return_value = None @@ -221,7 +222,7 @@ def test_valid_device_false(self): res = self.watchdog_inst._valid_device("test") self.assertEqual(res, False) - @mock.patch('crmsh.watchdog.Watchdog._verify_watchdog_device') + @mock.patch('crmsh.watchdog.Watchdog.verify_watchdog_device') def test_valid_device(self, mock_verify): mock_verify.return_value = True self.watchdog_inst._watchdog_info_dict = {'/dev/watchdog': 'softdog', '/dev/watchdog0': 'softdog', '/dev/watchdog1': 'iTCO_wdt'} @@ -229,7 +230,7 @@ def test_valid_device(self, mock_verify): self.assertEqual(res, True) @mock.patch('crmsh.utils.fatal') - @mock.patch('crmsh.watchdog.Watchdog._get_watchdog_device_from_sbd_config') + @mock.patch('crmsh.watchdog.Watchdog.get_watchdog_device_from_sbd_config') @mock.patch('crmsh.watchdog.Watchdog._set_watchdog_info') def test_join_watchdog_error(self, mock_set_info, mock_from_config, mock_error): mock_from_config.return_value = None @@ -238,12 +239,12 @@ def test_join_watchdog_error(self, mock_set_info, mock_from_config, mock_error): self.watchdog_join_inst.join_watchdog() mock_set_info.assert_called_once_with() mock_from_config.assert_called_once_with() - mock_error.assert_called_once_with("Failed to get watchdog device from {}".format(bootstrap.SYSCONFIG_SBD)) + mock_error.assert_called_once_with("Failed to get watchdog device from {}".format(sbd.SBDManager.SYSCONFIG_SBD)) @mock.patch('crmsh.watchdog.Watchdog._load_watchdog_driver') @mock.patch('crmsh.watchdog.Watchdog._get_driver_through_device_remotely') @mock.patch('crmsh.watchdog.Watchdog._valid_device') - @mock.patch('crmsh.watchdog.Watchdog._get_watchdog_device_from_sbd_config') + @mock.patch('crmsh.watchdog.Watchdog.get_watchdog_device_from_sbd_config') @mock.patch('crmsh.watchdog.Watchdog._set_watchdog_info') def test_join_watchdog(self, mock_set_info, mock_from_config, mock_valid, mock_get_driver_remotely, mock_load): mock_from_config.return_value = "/dev/watchdog" @@ -258,25 +259,26 @@ def test_join_watchdog(self, mock_set_info, mock_from_config, mock_valid, mock_g mock_get_driver_remotely.assert_called_once_with("/dev/watchdog") mock_load.assert_called_once_with("softdog") - @mock.patch('crmsh.watchdog.invokerc') + @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') @mock.patch('crmsh.watchdog.Watchdog._valid_device') @mock.patch('crmsh.watchdog.Watchdog._set_input') @mock.patch('crmsh.watchdog.Watchdog._set_watchdog_info') - def test_init_watchdog_valid(self, mock_set_info, mock_set_input, mock_valid, mock_invokerc): + def test_init_watchdog_valid(self, mock_set_info, mock_set_input, mock_valid, mock_run): mock_valid.return_value = True + mock_run.return_value = (0, None, None) self.watchdog_inst._input = "/dev/watchdog" self.watchdog_inst.init_watchdog() - mock_invokerc.assert_not_called() + mock_run.assert_not_called() mock_valid.assert_called_once_with("/dev/watchdog") @mock.patch('crmsh.utils.fatal') - @mock.patch('crmsh.watchdog.invokerc') + @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') @mock.patch('crmsh.watchdog.Watchdog._valid_device') @mock.patch('crmsh.watchdog.Watchdog._set_input') @mock.patch('crmsh.watchdog.Watchdog._set_watchdog_info') - def test_init_watchdog_error(self, mock_set_info, mock_set_input, mock_valid, mock_invokerc, mock_error): + def test_init_watchdog_error(self, mock_set_info, mock_set_input, mock_valid, mock_run, mock_error): mock_valid.return_value = False - mock_invokerc.return_value = False + mock_run.return_value = (1, None, None) self.watchdog_inst._input = "test" mock_error.side_effect = SystemExit @@ -284,27 +286,27 @@ def test_init_watchdog_error(self, mock_set_info, mock_set_input, mock_valid, mo self.watchdog_inst.init_watchdog() mock_valid.assert_called_once_with("test") - mock_invokerc.assert_called_once_with("modinfo test") + mock_run.assert_called_once_with("modinfo test") mock_error.assert_called_once_with("Should provide valid watchdog device or driver name by -w option") @mock.patch('crmsh.watchdog.Watchdog._get_device_through_driver') @mock.patch('crmsh.watchdog.Watchdog._load_watchdog_driver') @mock.patch('crmsh.watchdog.Watchdog._driver_is_loaded') - @mock.patch('crmsh.watchdog.invokerc') + @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') @mock.patch('crmsh.watchdog.Watchdog._valid_device') @mock.patch('crmsh.watchdog.Watchdog._set_input') @mock.patch('crmsh.watchdog.Watchdog._set_watchdog_info') - def test_init_watchdog(self, mock_set_info, mock_set_input, mock_valid, mock_invokerc, mock_is_loaded, mock_load, mock_get_device): + def test_init_watchdog(self, mock_set_info, mock_set_input, mock_valid, mock_run, mock_is_loaded, mock_load, mock_get_device): mock_valid.return_value = False self.watchdog_inst._input = "softdog" - mock_invokerc.return_value = True + mock_run.return_value = (0, None, None) mock_is_loaded.return_value = False mock_get_device.return_value = "/dev/watchdog" self.watchdog_inst.init_watchdog() mock_valid.assert_called_once_with("softdog") - mock_invokerc.assert_called_once_with("modinfo softdog") + mock_run.assert_called_once_with("modinfo softdog") mock_is_loaded.assert_called_once_with("softdog") mock_load.assert_called_once_with("softdog") mock_set_info.assert_has_calls([mock.call(), mock.call()])