diff --git a/flux_local/resource_diff.py b/flux_local/resource_diff.py new file mode 100644 index 00000000..8c043c87 --- /dev/null +++ b/flux_local/resource_diff.py @@ -0,0 +1,167 @@ +"""Module for computing resource diffs. + +This is used internally, primarily by the diff tool. +""" + +from collections.abc import Iterable +from dataclasses import asdict +import difflib +import logging +import pathlib +import tempfile +from typing import Generator, Any, AsyncGenerator, TypeVar +import yaml + + +from . import command +from .visitor import ObjectOutput, ResourceKey + +_LOGGER = logging.getLogger(__name__) + +_TRUNCATE = "[Diff truncated by flux-local]" + +T = TypeVar("T") + + +def _unique_keys(k1: dict[T, Any], k2: dict[T, Any]) -> Iterable[T]: + """Return an ordered set.""" + return { + **{k: True for k in k1.keys()}, + **{k: True for k in k2.keys()}, + }.keys() + + +def perform_object_diff( + a: ObjectOutput, b: ObjectOutput, n: int, limit_bytes: int +) -> Generator[str, None, None]: + """Generate diffs between the two output objects.""" + for kustomization_key in _unique_keys(a.content, b.content): + _LOGGER.debug( + "Diffing results for Kustomization %s (n=%d)", kustomization_key, n + ) + a_resources = a.content.get(kustomization_key, {}) + b_resources = b.content.get(kustomization_key, {}) + for resource_key in _unique_keys(a_resources, b_resources): + diff_text = difflib.unified_diff( + a=a_resources.get(resource_key, []), + b=b_resources.get(resource_key, []), + fromfile=f"{kustomization_key.label} {resource_key.compact_label}", + tofile=f"{kustomization_key.label} {resource_key.compact_label}", + n=n, + ) + size = 0 + for line in diff_text: + size += len(line) + if limit_bytes and size > limit_bytes: + yield _TRUNCATE + break + yield line + + +async def perform_external_diff( + cmd: list[str], + a: ObjectOutput, + b: ObjectOutput, + limit_bytes: int, +) -> AsyncGenerator[str, None]: + """Generate diffs between the two output objects.""" + with tempfile.TemporaryDirectory() as tmpdir: + for kustomization_key in _unique_keys(a.content, b.content): + _LOGGER.debug( + "Diffing results for Kustomization %s", + kustomization_key, + ) + a_resources = a.content.get(kustomization_key, {}) + b_resources = b.content.get(kustomization_key, {}) + keys = _unique_keys(a_resources, b_resources) + + a_file = pathlib.Path(tmpdir) / "a.yaml" + a_file.write_text( + "\n".join( + [ + "\n".join(a_resources.get(resource_key, [])) + for resource_key in keys + ] + ) + ) + b_file = pathlib.Path(tmpdir) / "b.yaml" + b_file.write_text( + "\n".join( + [ + "\n".join(b_resources.get(resource_key, [])) + for resource_key in keys + ] + ) + ) + + out = await command.Command( + cmd + [str(a_file), str(b_file)], retcodes=[0, 1] + ).run() + if out: + result = out.decode("utf-8") + if limit_bytes and len(result) > limit_bytes: + result = result[:limit_bytes] + "\n" + _TRUNCATE + yield result + + +def _omit_none(obj: Any) -> dict[str, Any]: + """Creates a dictionary with None values missing.""" + return {k: v for k, v in obj if v is not None} + + +def perform_yaml_diff( + a: ObjectOutput, + b: ObjectOutput, + n: int, + limit_bytes: int, +) -> Generator[str, None, None]: + """Generate diffs between the two output objects.""" + + diffs = [] + for kustomization_key in _unique_keys(a.content, b.content): + _LOGGER.debug("Diffing results for %s (n=%d)", kustomization_key, n) + a_resources = a.content.get(kustomization_key, {}) + b_resources = b.content.get(kustomization_key, {}) + resource_diffs = [] + for resource_key in _unique_keys(a_resources, b_resources): + diff_text = difflib.unified_diff( + a=a_resources.get(resource_key, []), + b=b_resources.get(resource_key, []), + fromfile=f"{kustomization_key.label} {resource_key.compact_label}", + tofile=f"{kustomization_key.label} {resource_key.compact_label}", + n=n, + ) + diff_content = "\n".join(diff_text) + if not diff_content: + continue + if limit_bytes and len(diff_content) > limit_bytes: + diff_content = diff_content[:limit_bytes] + "\n" + _TRUNCATE + obj = { + **asdict(resource_key, dict_factory=_omit_none), + "diff": diff_content, + } + resource_diffs.append(obj) + if resource_diffs: + diffs.append( + { + **asdict(kustomization_key), + "diffs": resource_diffs, + } + ) + if diffs: + yield yaml.dump(diffs, sort_keys=False, explicit_start=True, default_style=None) + + +def get_helm_release_diff_keys(a: ObjectOutput, b: ObjectOutput) -> list[ResourceKey]: + """Return HelmRelease resource keys with diffs, by cluster.""" + results: list[ResourceKey] = [] + for kustomization_key in _unique_keys(a.content, b.content): + _LOGGER.debug("Diffing results for Kustomization %s", kustomization_key) + a_resources = a.content.get(kustomization_key, {}) + b_resources = b.content.get(kustomization_key, {}) + for resource_key in _unique_keys(a_resources, b_resources): + if resource_key.kind != "HelmRelease": + continue + if a_resources.get(resource_key) != b_resources.get(resource_key): + results.append(resource_key) + return results diff --git a/flux_local/tool/build.py b/flux_local/tool/build.py index cd50bde6..e8efae0d 100644 --- a/flux_local/tool/build.py +++ b/flux_local/tool/build.py @@ -12,9 +12,9 @@ from flux_local import git_repo +from flux_local.visitor import ContentOutput, HelmVisitor from . import selector -from .visitor import ContentOutput, HelmVisitor _LOGGER = logging.getLogger(__name__) diff --git a/flux_local/tool/diff.py b/flux_local/tool/diff.py index 4c6a945e..7476aaef 100644 --- a/flux_local/tool/diff.py +++ b/flux_local/tool/diff.py @@ -4,176 +4,30 @@ import functools import os from argparse import ArgumentParser, _SubParsersAction as SubParsersAction -from collections.abc import Iterable from contextlib import contextmanager -from dataclasses import asdict -import difflib import logging import pathlib import shlex import tempfile -from typing import cast, Generator, Any, AsyncGenerator, TypeVar -import yaml +from typing import cast, Generator, Any -from flux_local import git_repo, command +from flux_local import git_repo +from flux_local.visitor import HelmVisitor, ObjectOutput +from flux_local.resource_diff import ( + get_helm_release_diff_keys, + perform_yaml_diff, + perform_external_diff, + perform_object_diff, +) from . import selector -from .visitor import HelmVisitor, ObjectOutput, ResourceKey _LOGGER = logging.getLogger(__name__) # Type for command line flags of comma separated list _CSV = functools.partial(str.split, sep=",") -_TRUNCATE = "[Diff truncated by flux-local]" - -T = TypeVar("T") - - -def _unique_keys(k1: dict[T, Any], k2: dict[T, Any]) -> Iterable[T]: - """Return an ordered set.""" - return { - **{k: True for k in k1.keys()}, - **{k: True for k in k2.keys()}, - }.keys() - - -def perform_object_diff( - a: ObjectOutput, b: ObjectOutput, n: int, limit_bytes: int -) -> Generator[str, None, None]: - """Generate diffs between the two output objects.""" - for kustomization_key in _unique_keys(a.content, b.content): - _LOGGER.debug( - "Diffing results for Kustomization %s (n=%d)", kustomization_key, n - ) - a_resources = a.content.get(kustomization_key, {}) - b_resources = b.content.get(kustomization_key, {}) - for resource_key in _unique_keys(a_resources, b_resources): - diff_text = difflib.unified_diff( - a=a_resources.get(resource_key, []), - b=b_resources.get(resource_key, []), - fromfile=f"{kustomization_key.label} {resource_key.compact_label}", - tofile=f"{kustomization_key.label} {resource_key.compact_label}", - n=n, - ) - size = 0 - for line in diff_text: - size += len(line) - if limit_bytes and size > limit_bytes: - yield _TRUNCATE - break - yield line - - -async def perform_external_diff( - cmd: list[str], - a: ObjectOutput, - b: ObjectOutput, - limit_bytes: int, -) -> AsyncGenerator[str, None]: - """Generate diffs between the two output objects.""" - with tempfile.TemporaryDirectory() as tmpdir: - for kustomization_key in _unique_keys(a.content, b.content): - _LOGGER.debug( - "Diffing results for Kustomization %s", - kustomization_key, - ) - a_resources = a.content.get(kustomization_key, {}) - b_resources = b.content.get(kustomization_key, {}) - keys = _unique_keys(a_resources, b_resources) - - a_file = pathlib.Path(tmpdir) / "a.yaml" - a_file.write_text( - "\n".join( - [ - "\n".join(a_resources.get(resource_key, [])) - for resource_key in keys - ] - ) - ) - b_file = pathlib.Path(tmpdir) / "b.yaml" - b_file.write_text( - "\n".join( - [ - "\n".join(b_resources.get(resource_key, [])) - for resource_key in keys - ] - ) - ) - - out = await command.Command( - cmd + [str(a_file), str(b_file)], retcodes=[0, 1] - ).run() - if out: - result = out.decode("utf-8") - if limit_bytes and len(result) > limit_bytes: - result = result[:limit_bytes] + "\n" + _TRUNCATE - yield result - - -def omit_none(obj: Any) -> dict[str, Any]: - """Creates a dictionary with None values missing.""" - return {k: v for k, v in obj if v is not None} - - -def perform_yaml_diff( - a: ObjectOutput, - b: ObjectOutput, - n: int, - limit_bytes: int, -) -> Generator[str, None, None]: - """Generate diffs between the two output objects.""" - - diffs = [] - for kustomization_key in _unique_keys(a.content, b.content): - _LOGGER.debug("Diffing results for %s (n=%d)", kustomization_key, n) - a_resources = a.content.get(kustomization_key, {}) - b_resources = b.content.get(kustomization_key, {}) - resource_diffs = [] - for resource_key in _unique_keys(a_resources, b_resources): - diff_text = difflib.unified_diff( - a=a_resources.get(resource_key, []), - b=b_resources.get(resource_key, []), - fromfile=f"{kustomization_key.label} {resource_key.compact_label}", - tofile=f"{kustomization_key.label} {resource_key.compact_label}", - n=n, - ) - diff_content = "\n".join(diff_text) - if not diff_content: - continue - if limit_bytes and len(diff_content) > limit_bytes: - diff_content = diff_content[:limit_bytes] + "\n" + _TRUNCATE - obj = { - **asdict(resource_key, dict_factory=omit_none), - "diff": diff_content, - } - resource_diffs.append(obj) - if resource_diffs: - diffs.append( - { - **asdict(kustomization_key), - "diffs": resource_diffs, - } - ) - if diffs: - yield yaml.dump(diffs, sort_keys=False, explicit_start=True, default_style=None) - - -def get_helm_release_diff_keys(a: ObjectOutput, b: ObjectOutput) -> list[ResourceKey]: - """Return HelmRelease resource keys with diffs, by cluster.""" - results: list[ResourceKey] = [] - for kustomization_key in _unique_keys(a.content, b.content): - _LOGGER.debug("Diffing results for Kustomization %s", kustomization_key) - a_resources = a.content.get(kustomization_key, {}) - b_resources = b.content.get(kustomization_key, {}) - for resource_key in _unique_keys(a_resources, b_resources): - if resource_key.kind != "HelmRelease": - continue - if a_resources.get(resource_key) != b_resources.get(resource_key): - results.append(resource_key) - return results - def add_diff_flags(args: ArgumentParser) -> None: """Add shared diff flags.""" diff --git a/flux_local/tool/get.py b/flux_local/tool/get.py index 59e2cbb9..6f636eee 100644 --- a/flux_local/tool/get.py +++ b/flux_local/tool/get.py @@ -12,10 +12,10 @@ import tempfile from flux_local import git_repo, image, helm +from flux_local.visitor import HelmVisitor, ImageOutput from .format import PrintFormatter, YamlFormatter from . import selector -from .visitor import HelmVisitor, ImageOutput _LOGGER = logging.getLogger(__name__) diff --git a/flux_local/tool/visitor.py b/flux_local/visitor.py similarity index 95% rename from flux_local/tool/visitor.py rename to flux_local/visitor.py index a194b867..6d28bd68 100644 --- a/flux_local/tool/visitor.py +++ b/flux_local/visitor.py @@ -1,4 +1,8 @@ -"""Visitors used by multiple commands.""" +"""Module for holding state while visiting resources. + +This is used internally primarily for keeping state when building output +or computing diffs. +""" import asyncio from abc import ABC, abstractmethod @@ -32,9 +36,7 @@ ] -ResourceType = ( - Kustomization | HelmRelease | HelmRepository | OCIRepository -) +ResourceType = Kustomization | HelmRelease | HelmRepository | OCIRepository @dataclass(frozen=True, order=True) @@ -176,7 +178,9 @@ def update_manifest(self, manifest: Manifest) -> None: helm_release.images.sort() -def strip_resource_attributes(resource: dict[str, Any], strip_attributes: list[str]) -> None: +def strip_resource_attributes( + resource: dict[str, Any], strip_attributes: list[str] +) -> None: """Strip any annotations from kustomize that contribute to diff noise when objects are re-ordered in the output.""" strip_attrs(resource["metadata"], strip_attributes) # Remove common noisy labels in commonly used templates @@ -186,7 +190,11 @@ def strip_resource_attributes(resource: dict[str, Any], strip_attributes: list[s and (meta := templ.get("metadata")) ): strip_attrs(meta, strip_attributes) - if resource["kind"] == "List" and (items := resource.get("items")) and isinstance(items, list): + if ( + resource["kind"] == "List" + and (items := resource.get("items")) + and isinstance(items, list) + ): for item in items: if not (item_meta := item.get("metadata")): continue diff --git a/tests/tool/test_visitor.py b/tests/test_visitor.py similarity index 98% rename from tests/tool/test_visitor.py rename to tests/test_visitor.py index 81cbc22a..95ce852e 100644 --- a/tests/tool/test_visitor.py +++ b/tests/test_visitor.py @@ -6,7 +6,7 @@ import pytest -from flux_local.tool.visitor import strip_resource_attributes +from flux_local.visitor import strip_resource_attributes STRIP_ATTRIBUTES = [ "app.kubernetes.io/version",