Fixed deleting a cluster while restoring/cloning (#539)

* Fixed deleting a cluster while restoring/cloning

CHANGES.rst

@@ -5,6 +5,9 @@ Changelog
 Unreleased
 ----------
 
+* Fixed a bug that lead to the namespace not being deleted after deleting a cluster
+  that had a snapshot restore/clone operation in progress.
+
 2.30.1 (2023-07-06)
 -------------------
 

crate/operator/main.py

@@ -52,14 +52,25 @@
     update_user_password_secret,
 )
 from crate.operator.kube_auth import login_via_kubernetes_asyncio
-from crate.operator.operations import update_sql_exporter_configmap
+from crate.operator.operations import (
+    is_namespace_terminating,
+    update_sql_exporter_configmap,
+)
 from crate.operator.restore_backup import is_valid_snapshot
 from crate.operator.utils import crate
 from crate.operator.webhooks import webhook_client
 
 NO_VALUE = object()
 
 
+async def raise_on_namespace_terminating(namespace: str):
+    # Ensure the namespace is not terminating. Otherwise end with a permanent error.
+    if await is_namespace_terminating(namespace):
+        raise kopf.PermanentError(
+            "The namespace for the target operation is terminating"
+        )
+
+
 @kopf.on.startup()
 async def startup(settings: kopf.OperatorSettings, **_kwargs):
     config.load()
@@ -129,6 +140,7 @@ async def cluster_create(
     """
     Handles creation of CrateDB Clusters.
     """
+    await raise_on_namespace_terminating(namespace)
     await create_cratedb(namespace, meta, spec, patch, status, logger)
 
 
@@ -153,6 +165,7 @@ async def cluster_update(
     """
     Handles updates to the CrateDB resource.
     """
+    await raise_on_namespace_terminating(namespace)
     await update_cratedb(namespace, name, patch, status, diff, started)
 
 
@@ -172,6 +185,7 @@ async def secret_update(
     """
     Handles changes to the password for a CrateDB cluster.
     """
+    await raise_on_namespace_terminating(namespace)
     await update_user_password_secret(namespace, name, diff, logger)
 
 
@@ -198,6 +212,8 @@ async def cluster_restore(
     """
     Handles field changes which trigger restoring data from a backup.
     """
+
+    await raise_on_namespace_terminating(namespace)
     await restore_backup(namespace, name, diff, new, patch, status, started, logger)
 
 
@@ -218,6 +234,7 @@ async def service_cidr_changes(
     """
     Handles updates to the list of allowed CIDRs, and updates the relevant k8s Service.
     """
+    await raise_on_namespace_terminating(namespace)
     await update_service_allowed_cidrs(namespace, name, diff, logger)
 
 
@@ -243,6 +260,7 @@ async def service_external_ip_update(
     This gets posted to the backend for further handling as a webhook
     (if webhooks are enabled).
     """
+    await raise_on_namespace_terminating(namespace)
     await external_ip_changed(namespace, diff, meta, logger)
 
 
@@ -263,6 +281,7 @@ async def service_backup_schedule_update(
     """
     Handles updates to the backup schedule for AWS s3 backups.
     """
+    await raise_on_namespace_terminating(namespace)
     await update_backup_schedule(namespace, name, diff, logger)
 
 
@@ -282,6 +301,7 @@ async def ping_cratedb(
     logger: logging.Logger,
     **_kwargs,
 ):
+    await raise_on_namespace_terminating(namespace)
     await ping_cratedb_status(namespace, name, spec["cluster"]["name"], patch, logger)
 
 
@@ -301,4 +321,5 @@ async def resume_sql_exporter_configmap(
     """
     Updates sql-exporter configmap in all namespaces to latest.
     """
+    await raise_on_namespace_terminating(namespace)
     await update_sql_exporter_configmap(namespace, name, logger)

crate/operator/operations.py

@@ -37,6 +37,7 @@
     V1CronJobList,
     V1JobList,
     V1JobStatus,
+    V1Namespace,
     V1PersistentVolumeClaimList,
     V1PodList,
     V1Service,
@@ -240,6 +241,31 @@ async def get_pods_in_deployment(
     return [{"uid": p.metadata.uid, "name": p.metadata.name} for p in all_pods.items]
 
 
+async def get_namespace_resource(namespace_name: str) -> V1Namespace:
+    """
+    Return the namespace for the given name.
+
+    :param namespace_name: The Kubernetes namespace name to look up.
+    """
+    async with ApiClient() as api_client:
+        core = CoreV1Api(api_client)
+        namespaces = await core.list_namespace()
+        for ns in namespaces.items:
+            if ns.metadata.name == namespace_name:
+                return ns
+        return None
+
+
+async def is_namespace_terminating(namespace_name: str) -> bool:
+    """
+    Determines if the namespace identified by the given name is terminating or not.
+
+    :param namespace_name: The Kubernetes namespace name to look up.
+    """
+    namespace_obj = await get_namespace_resource(namespace_name)
+    return namespace_obj and namespace_obj.status.phase == "Terminating"
+
+
 async def get_cratedb_resource(namespace: str, name: str) -> dict:
     """
     Return the CrateDB custom resource.