Code cleanup: Make kustomization updates happen in place (#629)

Simplify kustomization updates to prepare for adding dependency
tracking.

Issue #617

flux_local/git_repo.py

@@ -516,14 +516,8 @@ async def build_kustomization(
     selector: ResourceSelector,
     kustomize_flags: list[str],
     builder: CachableBuilder,
-) -> tuple[
-    Iterable[HelmRepository],
-    Iterable[HelmRelease],
-    Iterable[ClusterPolicy],
-    Iterable[ConfigMap],
-    Iterable[Secret],
-]:
-    """Build helm objects for the Kustomization."""
+) -> None:
+    """Build helm objects for the Kustomization and update state."""
 
     root: Path = selector.path.root
     kustomization_selector: MetadataSelector = selector.kustomization
@@ -537,7 +531,7 @@ async def build_kustomization(
         and not cluster_policy_selector.enabled
         and not selector.doc_visitor
     ):
-        return ([], [], [], [], [])
+        return
 
     with trace_context(f"Build '{kustomization.namespaced_name}'"):
         cmd = await builder.build(kustomization, root / kustomization.path)
@@ -576,7 +570,7 @@ async def build_kustomization(
         if selector.doc_visitor:
             kinds.extend(selector.doc_visitor.kinds)
         if not kinds:
-            return ([], [], [], [], [])
+            return None
 
         regexp = f"kind=^({'|'.join(kinds)})$"
         docs = await cmd.grep(regexp).objects(
@@ -590,38 +584,44 @@ async def build_kustomization(
                     continue
                 selector.doc_visitor.func(kustomization.namespaced_name, doc)
 
-        return (
+        kustomization.helm_repos = list(
             filter(
                 helm_repo_selector.predicate,
                 [
                     HelmRepository.parse_doc(doc)
                     for doc in docs
                     if doc.get("kind") == HELM_REPO_KIND
                 ],
-            ),
+            )
+        )
+        kustomization.helm_releases = list(
             filter(
                 helm_release_selector.predicate,
                 [
                     HelmRelease.parse_doc(doc)
                     for doc in docs
                     if doc.get("kind") == HELM_RELEASE_KIND
                 ],
-            ),
+            )
+        )
+        kustomization.cluster_policies = list(
             filter(
                 cluster_policy_selector.predicate,
                 [
                     ClusterPolicy.parse_doc(doc)
                     for doc in docs
                     if doc.get("kind") == CLUSTER_POLICY_KIND
                 ],
-            ),
-            [
+            )
+        )
+        kustomization.config_maps = [
                 ConfigMap.parse_doc(doc)
                 for doc in docs
                 if doc.get("kind") == CONFIG_MAP_KIND
-            ],
-            [Secret.parse_doc(doc) for doc in docs if doc.get("kind") == SECRET_KIND],
-        )
+        ]
+        kustomization.secrets = [
+            Secret.parse_doc(doc) for doc in docs if doc.get("kind") == SECRET_KIND
+        ]
 
 
 async def build_manifest(
@@ -674,22 +674,14 @@ async def update_kustomization(cluster: Cluster) -> None:
                         builder,
                     )
                 )
-            results = list(await asyncio.gather(*build_tasks))
-            for kustomization, (
-                helm_repos,
-                helm_releases,
-                cluster_policies,
-                config_maps,
-                secrets,
-            ) in zip(
-                cluster.kustomizations,
-                results,
-            ):
-                kustomization.helm_repos = list(helm_repos)
-                kustomization.helm_releases = list(helm_releases)
-                kustomization.cluster_policies = list(cluster_policies)
-                kustomization.config_maps = list(config_maps)
-                kustomization.secrets = list(secrets)
+            await asyncio.gather(*build_tasks)
+
+        # Validate all Kustomizations have valid dependsOn attribtues since later
+        # we'll be using them to order processing.
+        for cluster in clusters:
+            all_ks = set([ks.namespaced_name for ks in cluster.kustomizations])
+            for ks in cluster.kustomizations:
+                ks.validate_depends_on(all_ks)
 
         kustomization_tasks = []
         # Expand and visit Kustomizations

flux_local/manifest.py

@@ -6,6 +6,7 @@
 """
 
 import base64
+import logging
 from pathlib import Path
 from typing import Any, Optional, cast
 
@@ -28,6 +29,9 @@
     "ClusterPolicy",
 ]
 
+_LOGGER = logging.getLogger(__name__)
+
+
 # Match a prefix of apiVersion to ensure we have the right type of object.
 # We don't check specific versions for forward compatibility on upgrade.
 FLUXTOMIZE_DOMAIN = "kustomize.toolkit.fluxcd.io"
@@ -442,6 +446,9 @@ class Kustomization(BaseManifest):
     images: list[str] = Field(default_factory=list)
     """The list of images referenced in the kustomization."""
 
+    depends_on: list[str] | None = None
+    """A list of namespaced names that this Kustomization depends on."""
+
     @classmethod
     def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization":
         """Parse a partial Kustomization from a kubernetes resource."""
@@ -457,6 +464,14 @@ def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization":
         path = spec.get("path", "")
         source_path = metadata.get("annotations", {}).get("config.kubernetes.io/path")
         source_ref = spec.get("sourceRef", {})
+
+        depends_on = []
+        for dependency in spec.get("dependsOn", ()):
+            if not (dep_name := dependency.get("name")):
+                raise InputException(f"Invalid {cls} missing dependsOn.name: {doc}")
+            dep_namespace = dependency.get("namespace", namespace)
+            depends_on.append(f"{dep_namespace}/{dep_name}")
+
         return Kustomization(
             name=name,
             namespace=namespace,
@@ -467,6 +482,7 @@ def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization":
             source_namespace=source_ref.get("namespace", namespace),
             target_namespace=spec.get("targetNamespace"),
             contents=doc,
+            depends_on=depends_on,
         )
 
     @property
@@ -479,6 +495,18 @@ def namespaced_name(self) -> str:
         """Return the namespace and name concatenated as an id."""
         return f"{self.namespace}/{self.name}"
 
+
+    def validate_depends_on(self, all_ks: set[str]) -> None:
+        """Validate depends_on values are all correct given the list of Kustomizations."""
+        depends_on = set(self.depends_on or {})
+        if missing := (depends_on - all_ks):
+            _LOGGER.warning(
+                "Kustomization %s has dependsOn with invalid names: %s",
+                self.namespaced_name,
+                missing,
+            )
+            self.depends_on = list(depends_on - missing)
+
     @classmethod
     def compact_exclude_fields(cls) -> dict[str, Any]:
         """Return a dictionary of fields to exclude from compact_dict."""
@@ -501,6 +529,7 @@ def compact_exclude_fields(cls) -> dict[str, Any]:
             "source_kind": True,
             "target_namespace": True,
             "contents": True,
+            "depends_on": True,
         }