Merge branch 'main' into fix-deprecated-output

.devcontainer/Dockerfile

@@ -14,8 +14,8 @@ RUN apt-get update --fix-missing && \
 
 COPY --from=docker.io/bitnami/kubectl:1.30.2            /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
 COPY --from=registry.k8s.io/kustomize/kustomize:v5.4.2  /app/kustomize                   /usr/local/bin/kustomize
-COPY --from=ghcr.io/kyverno/kyverno-cli:v1.12.4         /ko-app/kubectl-kyverno          /usr/local/bin/kyverno
-COPY --from=docker.io/alpine/helm:3.15.2                /usr/bin/helm                    /usr/local/bin/helm
+COPY --from=ghcr.io/kyverno/kyverno-cli:v1.12.5         /ko-app/kubectl-kyverno          /usr/local/bin/kyverno
+COPY --from=docker.io/alpine/helm:3.15.3                /usr/bin/helm                    /usr/local/bin/helm
 COPY --from=ghcr.io/fluxcd/flux-cli:v2.3.0              /usr/local/bin/flux              /usr/local/bin/flux
 
 COPY requirements_dev.txt /src/

.pre-commit-config.yaml

@@ -4,15 +4,17 @@ repos:
     rev: v4.6.0
     hooks:
       - id: trailing-whitespace
+        exclude: '^tests/.*/__snapshots__/.*.ambr$'
       - id: end-of-file-fixer
       - id: check-yaml
+        exclude: '^tests/testdata/cluster8/apps/.*\.yaml|tests/testdata/cluster/apps/prod/.*\.yaml$'
       - id: check-added-large-files
   - repo: https://github.com/psf/black
     rev: 24.4.2
     hooks:
       - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.4.9
+    rev: v0.5.2
     hooks:
       - id: ruff
         args:

Dockerfile

@@ -10,10 +10,10 @@ COPY setup.cfg .
 RUN pip install -e .
 
 COPY --from=ghcr.io/fluxcd/flux-cli:v2.3.0              /usr/local/bin/flux              /usr/local/bin/flux
-COPY --from=docker.io/alpine/helm:3.15.2                /usr/bin/helm                    /usr/local/bin/helm
+COPY --from=docker.io/alpine/helm:3.15.3                /usr/bin/helm                    /usr/local/bin/helm
 COPY --from=docker.io/bitnami/kubectl:1.30.2            /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
 COPY --from=registry.k8s.io/kustomize/kustomize:v5.4.2  /app/kustomize                   /usr/local/bin/kustomize
-COPY --from=ghcr.io/kyverno/kyverno-cli:v1.12.4         /ko-app/kubectl-kyverno          /usr/local/bin/kyverno
+COPY --from=ghcr.io/kyverno/kyverno-cli:v1.12.5         /ko-app/kubectl-kyverno          /usr/local/bin/kyverno
 
 USER 1001
 ENTRYPOINT ["/usr/local/bin/flux-local"]

flux_local/git_repo.py

@@ -403,27 +403,54 @@ def remove(self, kustomization: Kustomization) -> None:
                 del self._cache[key]
 
 
+@dataclass
+class VisitResult:
+    """Result of visiting a kustomization."""
+
+    kustomizations: list[Kustomization]
+    config_maps: list[ConfigMap]
+    secrets: list[Secret]
+
+    def __post_init__(self) -> None:
+        """Validate the object"""
+        unique = {ks.namespaced_name for ks in self.kustomizations}
+        if len(unique) != len(self.kustomizations):
+            ks_names = [ks.namespaced_name for ks in self.kustomizations]
+            dupes = list(filter(lambda x: ks_names.count(x) > 1, ks_names))
+            raise FluxException(
+                f"Detected multiple Fluxtomizations with the same name: {dupes}. "
+                "This indicates either (1) an incorrect Kustomization which needs to be fixed "
+                "or (2) a multi-cluster setup which requires flux-local to run with a more strict --path."
+            )
+
+
 async def visit_kustomization(
     selector: PathSelector,
     builder: CachableBuilder,
     path: Path,
     visit_ks: Kustomization | None,
-) -> list[Kustomization]:
+) -> VisitResult:
     """Visit a path and return a list of Kustomizations."""
 
     _LOGGER.debug("Visiting path (%s) %s", selector.path, path)
     label = visit_ks.namespaced_name if visit_ks else str(path)
+
+    kinds = [CLUSTER_KUSTOMIZE_KIND, CONFIG_MAP_KIND, SECRET_KIND]
+
     with trace_context(f"Kustomization '{label}'"):
         cmd: kustomize.Kustomize
         if visit_ks is None:
-            cmd = kustomize.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}", selector.root / path)
+            cmd = kustomize.filter_resources(kinds, selector.root / path)
         else:
             cmd = await builder.build(visit_ks, selector.root / path)
-            cmd = cmd.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}")
-        cmd = cmd.grep(GREP_SOURCE_REF_KIND)
+            cmd = cmd.filter_resources(kinds)
+        cmd = await cmd.stash()
+        ks_cmd = cmd.grep(GREP_SOURCE_REF_KIND)
+        cfg_cmd = cmd.filter_resources([CONFIG_MAP_KIND, SECRET_KIND])
 
         try:
-            docs = await cmd.objects()
+            ks_docs = await ks_cmd.objects()
+            cfg_docs = await cfg_cmd.objects()
         except KustomizePathException as err:
             raise FluxException(err) from err
         except FluxException as err:
@@ -436,25 +463,26 @@ async def visit_kustomization(
                 f"Error building Fluxtomization '{visit_ks.namespaced_name}' "
                 f"path '{path}': {ERROR_DETAIL_BAD_KS} {err}"
             ) from err
-    kustomizations = list(
-        filter(
-            is_allowed_source(selector.sources or []),
-            [
-                Kustomization.parse_doc(doc)
-                for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, docs)
-            ],
-        )
+
+    return VisitResult(
+        kustomizations=list(
+            filter(
+                is_allowed_source(selector.sources or []),
+                [
+                    Kustomization.parse_doc(doc)
+                    for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, ks_docs)
+                ],
+            )
+        ),
+        config_maps=[
+            ConfigMap.parse_doc(doc)
+            for doc in cfg_docs
+            if doc.get("kind") == CONFIG_MAP_KIND
+        ],
+        secrets=[
+            Secret.parse_doc(doc) for doc in cfg_docs if doc.get("kind") == SECRET_KIND
+        ],
     )
-    unique = {ks.namespaced_name for ks in kustomizations}
-    if len(unique) != len(kustomizations):
-        ks_names = [ks.namespaced_name for ks in kustomizations]
-        dupes = list(filter(lambda x: ks_names.count(x) > 1, ks_names))
-        raise FluxException(
-            f"Detected multiple Fluxtomizations with the same name: {dupes}. "
-            "This indicates either (1) an incorrect Kustomization which needs to be fixed "
-            "or (2) a multi-cluster setup which requires flux-local to run with a more strict --path."
-        )
-    return kustomizations
 
 
 async def kustomization_traversal(
@@ -468,6 +496,7 @@ async def kustomization_traversal(
 
     path_queue: deque[tuple[Path, Kustomization | None]] = deque()
     path_queue.append((selector.relative_path, None))
+    cluster_config = values.cluster_config([], [])
     while path_queue:
         # Fully empty the queue, running all tasks in parallel
         tasks = []
@@ -484,7 +513,10 @@ async def kustomization_traversal(
         # Find new kustomizations
         kustomizations = []
         for result in await asyncio.gather(*tasks):
-            for ks in result:
+            cluster_config = values.merge_cluster_config(
+                cluster_config, result.secrets, result.config_maps
+            )
+            for ks in result.kustomizations:
                 if ks.namespaced_name in visited_ks:
                     continue
                 kustomizations.append(ks)
@@ -502,6 +534,12 @@ async def kustomization_traversal(
             if not (ks_path := adjust_ks_path(ks, selector)):
                 continue
             ks.path = str(ks_path)
+            if ks.postbuild_substitute_from:
+                values.expand_postbuild_substitute_reference(
+                    ks,
+                    cluster_config,
+                )
+
             path_queue.append((ks_path, ks))
             response_kustomizations.append(ks)
 
@@ -581,8 +619,7 @@ async def build_kustomization(
         if not kinds:
             return
 
-        regexp = f"kind=^({'|'.join(kinds)})$"
-        docs = await cmd.grep(regexp).objects(
+        docs = await cmd.filter_resources(kinds).objects(
             target_namespace=kustomization.target_namespace
         )
 

flux_local/helm.py

@@ -63,6 +63,7 @@
 
 
 HELM_BIN = "helm"
+DEFAULT_REGISTRY_CONFIG = "/dev/null"
 
 
 def _chart_name(release: HelmRelease, repo: HelmRepository | None) -> str:
@@ -126,10 +127,18 @@ class Options:
     api_versions: str | None = None
     """Value of the helm --api-versions flag."""
 
+    registry_config: str | None = None
+    """Value of the helm --registry-config flag."""
+
+    @property
+    def base_args(self) -> list[str]:
+        """Helm template CLI arguments built from the options."""
+        return ["--registry-config", self.registry_config or DEFAULT_REGISTRY_CONFIG]
+
     @property
     def template_args(self) -> list[str]:
         """Helm template CLI arguments built from the options."""
-        args = []
+        args = self.base_args
         if self.skip_crds:
             args.append("--skip-crds")
         if self.skip_tests:
@@ -159,8 +168,6 @@ def __init__(self, tmp_dir: Path, cache_dir: Path) -> None:
         self._tmp_dir = tmp_dir
         self._repo_config_file = self._tmp_dir / "repository-config.yaml"
         self._flags = [
-            "--registry-config",
-            "/dev/null",
             "--repository-cache",
             str(cache_dir),
             "--repository-config",
@@ -189,11 +196,10 @@ async def update(self) -> None:
         content = yaml.dump(RepositoryConfig(repos).config, sort_keys=False)
         async with aiofiles.open(str(self._repo_config_file), mode="w") as config_file:
             await config_file.write(content)
-        await command.run(
-            command.Command(
-                [HELM_BIN, "repo", "update"] + self._flags, exc=HelmException
-            )
-        )
+        args = [HELM_BIN, "repo", "update"]
+        args.extend(Options().base_args)
+        args.extend(self._flags)
+        await command.run(command.Command(args, exc=HelmException))
 
     async def template(
         self,
@@ -226,6 +232,7 @@ async def template(
             "--namespace",
             release.namespace,
         ]
+        args.extend(self._flags)
         args.extend(options.template_args)
         if release.chart.version:
             args.extend(
@@ -239,7 +246,7 @@ async def template(
             async with aiofiles.open(values_path, mode="w") as values_file:
                 await values_file.write(yaml.dump(release.values, sort_keys=False))
             args.extend(["--values", str(values_path)])
-        cmd = Kustomize([command.Command(args + self._flags, exc=HelmException)])
+        cmd = Kustomize([command.Command(args, exc=HelmException)])
         if options.skip_resources:
             cmd = cmd.skip_resources(options.skip_resources)
         return cmd

flux_local/kustomize.py

@@ -127,7 +127,12 @@ async def objects(
         self, target_namespace: str | None = None
     ) -> list[dict[str, Any]]:
         """Run the kustomize command and return the result cluster objects as a list."""
-        return [doc async for doc in self._docs(target_namespace=target_namespace)]
+        try:
+            return [doc async for doc in self._docs(target_namespace=target_namespace)]
+        except yaml.YAMLError as err:
+            raise KustomizeException(
+                f"Unable to parse command output: {self._cmds}: {err}"
+            ) from err
 
     def skip_resources(self, kinds: list[str]) -> "Kustomize":
         """Skip resources kinds of the specified types."""
@@ -136,6 +141,13 @@ def skip_resources(self, kinds: list[str]) -> "Kustomize":
         skip_re = "|".join(kinds)
         return self.grep(f"kind=^({skip_re})$", invert=True)
 
+    def filter_resources(self, kinds: list[str]) -> "Kustomize":
+        """Skip resources kinds of the specified types."""
+        if not kinds:
+            return self
+        skip_re = "|".join(kinds)
+        return self.grep(f"kind=^({skip_re})$", invert=False)
+
     async def validate_policies(self, policies: list[manifest.ClusterPolicy]) -> None:
         """Apply kyverno policies to objects built so far."""
         if not policies:
@@ -278,6 +290,12 @@ def grep(expr: str, path: Path, invert: bool = False) -> Kustomize:
     return Kustomize([Command(args, cwd=cwd, exc=KustomizeException)])
 
 
+def filter_resources(kinds: list[str], path: Path) -> Kustomize:
+    """Filter resources in the specified path based of a specific kind."""
+    regexp = f"kind=^({'|'.join(kinds)})$"
+    return grep(regexp, path)
+
+
 def update_namespace(doc: dict[str, Any], namespace: str) -> dict[str, Any]:
     """Update the namespace of the specified document.
 

flux_local/manifest.py

@@ -46,7 +46,7 @@
 SECRET_KIND = "Secret"
 CONFIG_MAP_KIND = "ConfigMap"
 DEFAULT_NAMESPACE = "flux-system"
-VALUE_PLACEHOLDER = "!!PLACEHOLDER!!"
+VALUE_PLACEHOLDER = "..PLACEHOLDER.."
 VALUE_B64_PLACEHOLDER = base64.b64encode(VALUE_PLACEHOLDER.encode())
 HELM_REPOSITORY = "HelmRepository"
 GIT_REPOSITORY = "GitRepository"

flux_local/tool/selector.py

@@ -185,6 +185,10 @@ def add_helm_options_flags(args: ArgumentParser) -> None:
         "-a",
         help="Kubernetes api versions used for helm Capabilities.APIVersions",
     )
+    args.add_argument(
+        "--registry-config",
+        help="Path to a helm registry config file",
+    )
 
 
 def build_helm_options(**kwargs) -> helm.Options:  # type: ignore[no-untyped-def]
@@ -198,6 +202,7 @@ def build_helm_options(**kwargs) -> helm.Options:  # type: ignore[no-untyped-def
         skip_secrets=kwargs["skip_secrets"],
         kube_version=kwargs.get("kube_version"),
         api_versions=kwargs.get("api_versions"),
+        registry_config=kwargs.get("registry_config"),
     )
 
 

flux_local/values.py

@@ -59,6 +59,16 @@ def cluster_config(
     )
 
 
+def merge_cluster_config(
+    config: ClusterConfig, secrets: list[Secret], config_maps: list[ConfigMap]
+) -> ClusterConfig:
+    """Create a ClusterConfig from a list of secrets and configmaps."""
+    return ClusterConfig(
+        lambda: list(config.secrets) + secrets,
+        lambda: list(config.config_maps) + config_maps,
+    )
+
+
 def ks_cluster_config(kustomizations: list[Kustomization]) -> ClusterConfig:
     """Create a ClusterConfig from a list of Kustomizations."""
 
@@ -262,7 +272,9 @@ def expand_postbuild_substitute_reference(
             continue
 
         if found_data is None:
-            if not ref.optional and not ref.kind == SECRET_KIND:  # Secrets are commonly filtered
+            if (
+                not ref.optional and not ref.kind == SECRET_KIND
+            ):  # Secrets are commonly filtered
                 _LOGGER.warning(
                     "Unable to find SubstituteReference for %s: %s",
                     ks.namespaced_name,
@@ -271,5 +283,6 @@ def expand_postbuild_substitute_reference(
             continue
 
         values.update(found_data)
+    _LOGGER.debug("update_postbuild_substitutions=%s", values)
     ks.update_postbuild_substitutions(values)
     return ks

requirements.txt

@@ -1,4 +1,4 @@
-aiofiles==23.2.1
+aiofiles==24.1.0
 GitPython==3.1.43
 mashumaro==3.13.1
 nest_asyncio==1.6.0