Dev: ui_sbd: Configure crashdump watchdog timeout (jsc#PED-11931)

Author: liangxin1300

crmsh/cibquery.py

@@ -33,6 +33,27 @@ def has_primitive_filesystem_with_fstype(cib: lxml.etree.Element, fstype: str) -
         f'/instance_attributes/nvpair[@name="fstype" and @value="{fstype}"]'
     ))
 
+
+def has_primitive(cib: lxml.etree.Element, ra: ResourceAgent) -> list[str]:
+    """
+    Given cib and ResourceAgent instance, return id list of primitives that matched
+    consider provider as optional
+    """
+    provider_condition = f' and @provider="{ra.m_provider}"' if ra.m_provider else ""
+    return [e.get('id') for e in cib.xpath(
+        f'/cib/configuration/resources//primitive[@class="{ra.m_class}"{provider_condition} and @type="{ra.m_type}"]'
+    )]
+
+
+def get_parameter_value(cib: lxml.etree.Element, res_id: str, param_name: str) -> typing.Optional[str]:
+    return next((
+        e.get('value') for e in cib.xpath(
+            f'/cib/configuration/resources//primitive[@id="{res_id}"]'
+            f'/instance_attributes/nvpair[@name="{param_name}"]'
+        )
+    ), None)
+
+
 def get_cluster_nodes(cib: lxml.etree.Element) -> list[ClusterNode]:
     """Return a list of cluster nodes, excluding pacemaker-remote nodes"""
     result = list()

crmsh/sbd.py

@@ -559,7 +559,10 @@ def configure_sbd(self):
         Configure fence_sbd resource and related properties
         '''
         if self.diskless_sbd:
-            utils.set_property("stonith-watchdog-timeout", SBDTimeout.STONITH_WATCHDOG_TIMEOUT_DEFAULT)
+            if "stonith-watchdog" in self.timeout_dict:
+                utils.set_property("stonith-watchdog-timeout", self.timeout_dict.get("stonith-watchdog"), conditional=True)
+            else:
+                utils.set_property("stonith-watchdog-timeout", SBDTimeout.STONITH_WATCHDOG_TIMEOUT_DEFAULT)
         else:
             if utils.get_property("stonith-watchdog-timeout", get_default=False):
                 utils.delete_property("stonith-watchdog-timeout")

crmsh/ui_sbd.py

@@ -12,6 +12,7 @@
 from crmsh import sh
 from crmsh import xmlutil
 from crmsh import constants
+from crmsh import cibquery
 from crmsh.service_manager import ServiceManager
 
 
@@ -88,8 +89,8 @@ class SBD(command.UI):
     - sbd purge
     '''
     name = "sbd"
-    TIMEOUT_TYPES = ("watchdog", "allocate", "loop", "msgwait")
-    DISKLESS_TIMEOUT_TYPES = ("watchdog",)
+    TIMEOUT_TYPES = ("watchdog", "allocate", "loop", "msgwait", "crashdump-watchdog")
+    DISKLESS_TIMEOUT_TYPES = ("watchdog", "crashdump-watchdog")
     SHOW_TYPES = ("disk_metadata", "sysconfig", "property")
     DISKLESS_SHOW_TYPES = ("sysconfig", "property")
     PCMK_ATTRS = (
@@ -102,12 +103,15 @@ class SBD(command.UI):
     PCMK_ATTRS_DISKLESS = ('stonith-watchdog-timeout',)
     PARSE_RE = re.compile(
 		# Match keys with non-empty values, capturing possible suffix
-        r'(\w+)(?:-(\w+))?=("[^"]+"|[\w/\d;]+)'
+        r'([\w-]+)-([\w-]+)=([\w/\d]+)'
     )
 
     class SyntaxError(Exception):
         pass
 
+    class MissingRequiredException(Exception):
+        pass
+
     def __init__(self):
         self.device_list_from_config: list[str] = None
         self.device_meta_dict_runtime: dict[str, int] = None
@@ -271,43 +275,63 @@ def _parse_args(self, args: tuple[str, ...]) -> dict[str, int|str]:
         logger.debug("Parsed arguments: %s", parameter_dict)
         return parameter_dict
 
-    @staticmethod
-    def _adjust_timeout_dict(timeout_dict: dict) -> dict:
-        watchdog_timeout = timeout_dict.get("watchdog")
-        msgwait_timeout = timeout_dict.get("msgwait")
-        if watchdog_timeout and msgwait_timeout and msgwait_timeout < 2*watchdog_timeout:
-            logger.warning("It's recommended to set msgwait timeout >= 2*watchdog timeout")
-            return timeout_dict
-        if watchdog_timeout and not msgwait_timeout:
-            timeout_dict["msgwait"] = 2*watchdog_timeout
-            logger.info("No msgwait timeout specified, use 2*watchdog timeout: %s", 2*watchdog_timeout)
-            return timeout_dict
-        if msgwait_timeout and not watchdog_timeout:
-            watchdog_timeout = msgwait_timeout//2
-            timeout_dict["watchdog"] = watchdog_timeout
-            logger.info("No watchdog timeout specified, use msgwait timeout/2: %s", watchdog_timeout)
-            return timeout_dict
+    def set_crashdump_option(self):
+        '''
+        Set crashdump option for fence_sbd resource
+        '''
+        shell = sh.LocalShell()
+        cib = xmlutil.text2elem(shell.get_stdout_or_raise_error(None, 'crm configure show xml'))
+        ra = cibquery.ResourceAgent("stonith", "", "fence_sbd")
+        res_id_list = cibquery.has_primitive(cib, ra)
+        if not res_id_list:
+            logger.error("No fence_sbd resource found")
+            raise self.MissingRequiredException
+        crashdump_value = cibquery.get_parameter_value(cib, res_id_list[0], "crashdump")
+        if utils.is_boolean_false(crashdump_value):
+            cmd = f"crm resource param {res_id_list[0]} set crashdump 1"
+            shell.get_stdout_or_raise_error(None, cmd)
+        logger.info("Set crashdump option for fence_sbd resource")
+
+    def is_kdump_service_active(self) -> bool:
+        result = True
+        for node in self.cluster_nodes:
+            if not self.service_manager.service_is_active("kdump.service", node):
+                logger.error("Kdump service is not active on %s", node)
+                result = False
+        return result
 
     def _configure_diskbase(self, parameter_dict: dict):
         '''
         Configure disk-based SBD based on input parameters and runtime config
         '''
         update_dict = {}
+        timeout_dict = {
+            item: parameter_dict.get(item) or self.device_meta_dict_runtime.get(item)
+            for item in self.TIMEOUT_TYPES if item != "crashdump-watchdog"
+        }
+
+        crashdump_watchdog_timeout = parameter_dict.get("crashdump-watchdog")
+        if crashdump_watchdog_timeout:
+            if not self.is_kdump_service_active():
+                raise self.MissingRequiredException
+            self.set_crashdump_option()
+            timeout_dict["msgwait"] = 2*timeout_dict["watchdog"] + crashdump_watchdog_timeout
+            logger.info("Set msgwait timeout to 2*watchdog + crashdump-watchdog: %s", timeout_dict["msgwait"])
+            update_dict["SBD_TIMEOUT_ACTION"] = "flush,crashdump"
+            update_dict["SBD_OPTS"] = f"-C {crashdump_watchdog_timeout}"
+
+        if timeout_dict["msgwait"] < 2*timeout_dict["watchdog"]:
+            logger.warning("It's recommended to set msgwait timeout >= 2*watchdog timeout")
+            return
+
         watchdog_device = parameter_dict.get("watchdog-device")
         if watchdog_device != self.watchdog_device_from_config:
             update_dict["SBD_WATCHDOG_DEV"] = watchdog_device
-        timeout_dict = {k: v for k, v in parameter_dict.items() if k in self.TIMEOUT_TYPES}
-        is_subdict_timeout = utils.is_subdict(timeout_dict, self.device_meta_dict_runtime)
 
-        if is_subdict_timeout and not update_dict:
+        if timeout_dict == self.device_meta_dict_runtime and not update_dict:
             logger.info("No change in SBD configuration")
             return
 
-        if not is_subdict_timeout:
-            timeout_dict = self._adjust_timeout_dict(timeout_dict)
-            # merge runtime timeout dict into parameter timeout dict without overwriting
-            timeout_dict = {**self.device_meta_dict_runtime, **timeout_dict}
-
         sbd_manager = sbd.SBDManager(
             device_list_to_init=self.device_list_from_config,
             timeout_dict=timeout_dict,
@@ -320,17 +344,29 @@ def _configure_diskless(self, parameter_dict: dict):
         Configure diskless SBD based on input parameters and runtime config
         '''
         update_dict = {}
+        timeout_dict = {}
         watchdog_timeout = parameter_dict.get("watchdog")
         if watchdog_timeout and watchdog_timeout != self.watchdog_timeout_from_config:
             update_dict["SBD_WATCHDOG_TIMEOUT"] = str(watchdog_timeout)
         watchdog_device = parameter_dict.get("watchdog-device")
         if watchdog_device != self.watchdog_device_from_config:
             update_dict["SBD_WATCHDOG_DEV"] = watchdog_device
+        crashdump_watchdog_timeout = parameter_dict.get("crashdump-watchdog")
+        if crashdump_watchdog_timeout:
+            if not self.is_kdump_service_active():
+                raise self.MissingRequiredException
+            update_dict["SBD_TIMEOUT_ACTION"] = "flush,crashdump"
+            update_dict["SBD_OPTS"] = f"-C {crashdump_watchdog_timeout} -Z"
+            sbd_watchdog_timeout = watchdog_timeout or self.watchdog_timeout_from_config
+            stonith_watchdog_timeout = sbd_watchdog_timeout + crashdump_watchdog_timeout
+            logger.info("Set stonith-watchdog-timeout to SBD_WATCHDOG_TIMEOUT + crashdump-watchdog: %s", stonith_watchdog_timeout)
+            timeout_dict["stonith-watchdog"] = stonith_watchdog_timeout
         if not update_dict:
             logger.info("No change in SBD configuration")
             return
 
         sbd_manager = sbd.SBDManager(
+            timeout_dict=timeout_dict,
             update_dict=update_dict,
             diskless_sbd=True
         )
@@ -426,6 +462,7 @@ def do_configure(self, context, *args) -> bool:
             if args[0] == "show":
                 self._configure_show(args)
                 return True
+
             parameter_dict = self._parse_args(args)
             if sbd.SBDUtils.is_using_disk_based_sbd():
                 self._configure_diskbase(parameter_dict)
@@ -439,6 +476,8 @@ def do_configure(self, context, *args) -> bool:
             if usage:
                 print(usage)
             return False
+        except self.MissingRequiredException:
+            return False
 
     def do_purge(self, context) -> bool:
         '''

crmsh/utils.py

@@ -3202,11 +3202,4 @@ def strip_ansi_escape_sequences(text):
     """
     ansi_escape_pattern = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
     return ansi_escape_pattern.sub('', text)
-
-
-def is_subdict(sub_dict, main_dict):
-    """
-    Check if sub_dict is a sub-dictionary of main_dict
-    """
-    return all(main_dict.get(k) == v for k, v in sub_dict.items())
 # vim:ts=4:sw=4:et: