Delete LXD instance if LXD failed to remove ephemeral instance

charmcraft.yaml

@@ -7,6 +7,7 @@ parts:
     charm-python-packages:
       - setuptools  # for jinja2
     build-packages:
+      - git  # for installing git source of pylxd 
       - libffi-dev  # for cffi
       - libssl-dev  # for cryptography
       - rust-all  # for cryptography

requirements.txt

@@ -3,6 +3,6 @@ ops
 requests
 jinja2
 ghapi
-pylxd
+pylxd @ git+https://github.com/lxc/pylxd
 # Newer version does not work with default OpenSSL version on jammy.
 cryptography <= 38.0.4

src/runner.py

@@ -141,6 +141,18 @@ def remove(self, remove_token: str) -> None:
                         self.instance.stop(force=True)
                     except LxdError as err:
                         raise RunnerRemoveError(f"Unable to remove {self.config.name}") from err
+            else:
+                # Delete ephemeral instances that are in error status or stopped status that LXD
+                # failed to clean up.
+                logger.warning(
+                    "Found runner %s in status %s, forcing deletion",
+                    self.config.name,
+                    self.instance.status,
+                )
+                try:
+                    self.instance.delete(wait=True)
+                except LxdError as err:
+                    raise RunnerRemoveError(f"Unable to remove {self.config.name}") from err
 
         # The runner should cleanup itself.  Cleanup on GitHub in case of runner cleanup error.
         if isinstance(self.config.path, GitHubRepo):

src/utilities.py

@@ -52,10 +52,10 @@ def retry_decorator(
         """Decorate function with retry.
 
         Args:
-            fn (Callable[..., R]): The function to decorate.
+            fn: The function to decorate.
 
         Returns:
-            Callable[..., R]: The resulting function with retry added.
+            The resulting function with retry added.
         """
 
         @functools.wraps(func)