Use juju-run to invoke juju-reboot for unit reboot

Issue #921 has the context for the change: in order to avoid triggering
reboots asynchronously to juju hook executions (when `juju ssh reboot`
is done).

openstack-charmers/zaza-openstack-tests#921
(cherry picked from commit 3b74604)

unit_tests/utilities/test_zaza_utilities_generic.py

@@ -204,6 +204,13 @@ def test_reboot(self):
             ['juju', 'ssh', _unit,
              'sudo', 'reboot', '&&', 'exit'])
 
+    def test_juju_reboot(self):
+        _unit = "app/2"
+        generic_utils.juju_reboot(_unit)
+        self.subprocess.check_call.assert_called_once_with(
+            ['juju', 'ssh', _unit,
+             f'sudo juju-run -u {_unit} "juju-reboot --now"'])
+
     def test_run_via_ssh(self):
         _unit = "app/2"
         _cmd = "hostname"

unit_tests/utilities/test_zaza_utilities_machine_os.py

@@ -153,7 +153,8 @@ def test_reboot_hvs(self):
         unit = mock.MagicMock()
         unit.name = 'someApp/0'
         self.get_units.return_value = [unit]
-        self.patch_object(machine_os_utils.zaza.utilities.generic, 'reboot')
+        self.patch_object(machine_os_utils.zaza.utilities.generic,
+                          'juju_reboot')
         self.patch_object(machine_os_utils.zaza.model,
                           'block_until_unit_wl_status')
         self.patch_object(machine_os_utils.zaza.charm_lifecycle.utils,
@@ -164,17 +165,17 @@ def test_reboot_hvs(self):
                           'wait_for_application_states')
         machine_os_utils.reboot_hvs()
         self.get_units.assert_called_once_with('someApp')
-        self.reboot.assert_called_once_with('someApp/0')
+        self.juju_reboot.assert_called_once_with('someApp/0')
         self.wait_for_application_states.assert_called_once_with(
             states={'someDeployStatus': None})
 
         # Units provided as argument
         self.get_units.reset_mock()
-        self.reboot.reset_mock()
+        self.juju_reboot.reset_mock()
         self.wait_for_application_states.reset_mock()
         machine_os_utils.reboot_hvs(units=[unit])
         self.assertFalse(self.get_units.called)
-        self.reboot.assert_called_once_with('someApp/0')
+        self.juju_reboot.assert_called_once_with('someApp/0')
         self.wait_for_application_states.assert_called_once_with(
             states={'someDeployStatus': None})
 

zaza/utilities/generic.py

@@ -508,6 +508,25 @@ def reboot(unit_name):
         pass
 
 
+def juju_reboot(unit_name):
+    """Reboot a unit using juju-reboot.
+
+    As `juju run` does not allow running juju-reboot (see LP: #1990140), use
+    `juju ssh` to invoke juju-run with the unit context with the juju-reboot
+    command. This will trigger a Juju-controlled reboot during which Juju
+    will not attempt to run any hooks asynchronously which can cause an
+    unintended hook execution failure upon reboot (for example, this happens
+    with update-status hooks causing intermittent CI failures).
+    """
+    cmd = ['juju', 'ssh', unit_name,
+           f'sudo juju-run -u {unit_name} "juju-reboot --now"']
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        logging.info(e)
+        pass
+
+
 def set_dpkg_non_interactive_on_unit(
         unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"):
     """Set dpkg options on unit.

zaza/utilities/machine_os.py

@@ -200,7 +200,7 @@ def reboot_hvs(units=None):
         return
     units = units or zaza.model.get_units(hv_application)
     for unit in units:
-        zaza.utilities.generic.reboot(unit.name)
+        zaza.utilities.generic.juju_reboot(unit.name)
         zaza.model.block_until_unit_wl_status(unit.name, "unknown")
     target_deploy_status = zaza.charm_lifecycle.utils.get_charm_config().get(
         'target_deploy_status', {})