DPE-2662 single unit upgrade fixes (#340)

* fixes for single unit upgrade

* the mandatory version bump

* lib bump and test fix

* lib bump

* add CI step to free some space in the runner

we start seeing errors due to no space left on device

.github/workflows/ci.yaml

@@ -179,6 +179,13 @@ jobs:
             echo Skipping unstable tests
             echo "mark_expression=not unstable" >> "$GITHUB_OUTPUT"
           fi
+      - name: Free space in runner
+        run: |
+          # free space in the runner
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Run integration tests
         run: tox run -e integration -- "${{ matrix.groups.path_to_test_file }}" --group="${{ matrix.groups.group_number }}" -m '${{ steps.select-test-stability.outputs.mark_expression }}' --mysql-charm-series="${{ matrix.ubuntu-versions.series }}" --mysql-charm-bases-index="${{ matrix.ubuntu-versions.bases-index }}"
         env:

lib/charms/data_platform_libs/v0/upgrade.py

@@ -285,7 +285,7 @@ def restart(self, event) -> None:
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 13
+LIBPATCH = 14
 
 PYDEPS = ["pydantic>=1.10,<2", "poetry-core"]
 
@@ -895,6 +895,10 @@ def _on_upgrade_charm(self, event: UpgradeCharmEvent) -> None:
             self.charm.unit.status = WaitingStatus("other units upgrading first...")
             self.peer_relation.data[self.charm.unit].update({"state": "ready"})
 
+            if self.charm.app.planned_units() == 1:
+                # single unit upgrade, emit upgrade_granted event right away
+                getattr(self.on, "upgrade_granted").emit()
+
         else:
             # for k8s run version checks only on highest ordinal unit
             if (

lib/charms/mysql/v0/mysql.py

@@ -116,7 +116,7 @@ def wait_until_mysql_connection(self) -> None:
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 51
+LIBPATCH = 52
 
 UNIT_TEARDOWN_LOCKNAME = "unit-teardown"
 UNIT_ADD_LOCKNAME = "unit-add"
@@ -1848,12 +1848,15 @@ def verify_server_upgradable(self, instance: Optional[str] = None) -> None:
             MySQLServerUpgradableError: If the server is not upgradable
         """
         check_command = [
-            f"shell.connect_to_primary('{self.server_config_user}"
+            f"shell.connect('{self.server_config_user}"
             f":{self.server_config_password}@{instance or self.instance_address}')",
             "try:",
             "    util.check_for_server_upgrade(options={'outputFormat': 'JSON'})",
             "except ValueError:",  # ValueError is raised for same version check
-            "    print('SAME_VERSION')",
+            "    if session.run_sql('select @@version').fetch_all()[0][0].split('-')[0] == shell.version.split()[1]:",
+            "        print('SAME_VERSION')",
+            "    else:",
+            "        raise",
         ]
 
         def _strip_output(output: str):

src/upgrade.py

@@ -202,6 +202,27 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:
 
         self.charm.unit.status = MaintenanceStatus("recovering unit after upgrade")
 
+        try:
+            if self.charm.app.planned_units() > 1:
+                self._recover_multi_unit_cluster()
+            else:
+                self._recover_single_unit_cluster()
+
+            logger.debug("Upgraded unit is healthy. Set upgrade state to `completed`")
+            self.set_unit_completed()
+            # ensures leader gets it's own relation-changed when it upgrades
+            if self.charm.unit.is_leader():
+                logger.debug("Re-emitting upgrade-changed on leader...")
+                self.on_upgrade_changed(event)
+        except Exception:
+            logger.debug("Upgraded unit is not healthy")
+            self.set_unit_failed()
+            self.charm.unit.status = BlockedStatus(
+                "upgrade failed. Check logs for rollback instruction"
+            )
+
+    def _recover_multi_unit_cluster(self) -> None:
+        logger.debug("Recovering unit")
         try:
             for attempt in Retrying(
                 stop=stop_after_attempt(RECOVER_ATTEMPTS), wait=wait_fixed(10)
@@ -214,18 +235,13 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:
                             f" Retry {attempt.retry_state.attempt_number}/{RECOVER_ATTEMPTS}"
                         )
                         raise Exception
-                    logger.debug("Upgraded unit is healthy. Set upgrade state to `completed`")
-                    self.set_unit_completed()
-                    # ensures leader gets it's own relation-changed when it upgrades
-                    if self.charm.unit.is_leader():
-                        logger.debug("Re-emitting upgrade-changed on leader...")
-                        self.on_upgrade_changed(event)
         except RetryError:
-            logger.debug("Upgraded unit is not healthy")
-            self.set_unit_failed()
-            self.charm.unit.status = BlockedStatus(
-                "upgrade failed. Check logs for rollback instruction"
-            )
+            raise
+
+    def _recover_single_unit_cluster(self) -> None:
+        """Recover single unit cluster."""
+        logger.debug("Recovering single unit cluster")
+        self.charm._mysql.reboot_from_complete_outage()
 
     def _on_upgrade_changed(self, _) -> None:
         """Handle the upgrade changed event.
@@ -273,7 +289,11 @@ def _check_server_upgradeability(self) -> None:
         Raises:
             VersionError: If the server is not upgradeable.
         """
-        if len(self.upgrade_stack or []) < self.charm.app.planned_units():
+        planned_units = self.charm.app.planned_units()
+        if planned_units == 1:
+            # single unit upgrade, no need for check
+            return
+        if len(self.upgrade_stack or []) < planned_units:
             # check is done for first upgrading unit only
             return
 

tests/unit/test_mysql.py

@@ -1694,9 +1694,13 @@ def test_set_cluster_primary(self, _run_mysqlsh_script):
     def test_verify_server_upgradable(self, _run_mysqlsh_script):
         """Test is_server_upgradable."""
         commands = (
-            f"shell.connect_to_primary('{self.mysql.server_config_user}:{self.mysql.server_config_password}@127.0.0.1')",
+            f"shell.connect('{self.mysql.server_config_user}:{self.mysql.server_config_password}@127.0.0.1')",
             "try:\n    util.check_for_server_upgrade(options={'outputFormat': 'JSON'})",
-            "except ValueError:\n    print('SAME_VERSION')",
+            "except ValueError:",
+            "    if session.run_sql('select @@version').fetch_all()[0][0].split('-')[0] == shell.version.split()[1]:",
+            "        print('SAME_VERSION')",
+            "    else:",
+            "        raise",
         )
         _run_mysqlsh_script.return_value = (
             "Some info header to be stripped\n"