Convert File.sensitive_data to a tri-state variable to allow overriding sensitivity auto-detection

.pre-commit-config.yaml

@@ -19,7 +19,11 @@ repos:
         exclude: |
             (?x)^(
                 src/batou/insecure-private.key|
-                src/batou/secrets/tests/fixture/age/id_ed25519
+                src/batou/secrets/tests/fixture/age/id_ed25519|
+                examples/sensitive-values/ssh-keys/client_ed25519|
+                examples/sensitive-values/ssh-keys/host_ed25519|
+                examples/sensitive-values/ssh-keys/host_rsa|
+                examples/sensitive-values/secrets.cfg.age.clear
             )$
 
 -   repo: https://github.com/pycqa/isort

CHANGES.md

@@ -4,7 +4,10 @@
 ## 2.5.1 (unreleased)
 ---------------------
 
-- Nothing changed yet.
+- `File` Component: Converted `sensitive_data` flag to a tri-state variable. This allows manual overriding of automatic sensitivity detection logic for file diffs. The new possible states are:
+  - `None`: Default automatic detection of sensitive data.
+  - `True`: Always mark the file as sensitive and avoid printing the diff.
+  - `False`: Always print the file diff, even if sensitive data is detected.
 
 
 ## 2.5.0 (2024-09-04)

doc/source/components/files.txt

@@ -90,8 +90,12 @@ Creates a file. The main parameter for File is the target path. A ``File`` insta
 
     Mark a file as sensitive so its content is not exposed by the
     (diff-)output of batou. This is useful in situations where the
-    rendered file contains a password or other sensitive data.
-    [Default: False]
+    rendered file contains a password or other sensitive data. If
+    unset, batou will automatically determine if file content is
+    sensitive if it shares words with secrets provided by the
+    environment. This attribute can be set to True or False to make
+    batou consider the file's content as always or never sensitive,
+    respectively. [Default: None]
 
 .. py:class:: batou.lib.file.BinaryFile(path)
 

examples/sensitive-values/.batou.json

@@ -0,0 +1 @@
+{"migration": {"version": 2400}}

examples/sensitive-values/appenv

@@ -0,0 +1 @@
+../../appenv.py

examples/sensitive-values/batou

@@ -0,0 +1 @@
+../../appenv.py

examples/sensitive-values/components/values/component.py

@@ -0,0 +1,49 @@
+from batou.component import Component
+from batou.lib.file import File
+
+
+class SensitiveValues(Component):
+
+    # SSH keys loaded from age-encrypted secrets
+    ssh_client_privkey = None
+    ssh_client_pubkey = None
+
+    # SSH keys loaded from plaintext configuration
+    ssh_host_rsa_pubkey = None
+    ssh_host_ed25519_pubkey = None
+
+    def configure(self):
+        # File content loaded from secrets automatically detected as
+        # sensitive.
+        self += File(
+            "client_ed25519.key",
+            content="{{component.ssh_client_privkey}}",
+        )
+        self += File(
+            "client_ed25519.pub",
+            content="{{component.ssh_client_pubkey}}",
+        )
+
+        # Content from non-secret configuration automatically marked
+        # sensitive when words overlap with words found in secret
+        # values.
+        self += File(
+            "hostkey_sensitive_auto_rsa.pub",
+            content="{{component.ssh_host_rsa_pubkey}}",
+        )
+        self += File(
+            "hostkey_sensitive_auto_ed25519.pub",
+            content="{{component.ssh_host_ed25519_pubkey}}",
+        )
+
+        # Override autodetection of file content sensitivity.
+        self += File(
+            "hostkey_sensitive_masked_rsa.pub",
+            content="{{component.ssh_host_rsa_pubkey}}",
+            sensitive_data=True,
+        )
+        self += File(
+            "hostkey_sensitive_clear_ed25519.pub",
+            content="{{component.ssh_host_ed25519_pubkey}}",
+            sensitive_data=False,
+        )

examples/sensitive-values/environments/local/age_keys.txt

@@ -0,0 +1,9 @@
+###########################################
+# This file is automatically generated by #
+# batou. It contains the public keys of   #
+# the members of the environment. It is   #
+# re-written every time a secrets file is #
+# encrypted (after a change).             #
+###########################################
+# plain ssh public key
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIACZ8++sQADp8fztgumfw2i+WSgzMHB7MgSpkM2y5pHi batou-ci-test-key

examples/sensitive-values/environments/local/environment.cfg

@@ -0,0 +1,9 @@
+[environment]
+connect_method = local
+
+[hosts]
+localhost = sensitivevalues
+
+[component:sensitivevalues]
+ssh_host_ed25519_pubkey = ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM63uA8ENkTbwfDsNHQKuQmh+D3fYtNlEvEVpW7q6LvM batou-example-host
+ssh_host_rsa_pubkey = ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChEpRy4ouSiSNvSyhCXTQOEhGM8hYj1EcW6xlzzE/FjiRoTyx3QFAP1tyBJ9cvUfyszwF4L4s/B+XyCQgW4ODdLw6xuXZD6QFEpMupgJ3Js8wKQFP9CKshxSi5oVgbJRDMjaRzJK5sRw3vM7RfJJ+mBH94AgyYhN6GvCPTDYCm50a55vLaePrydEgeh/L6jQ55Fp0rV6xmX6edszTDRkf5FicvwEM2FWxebWDyHsIV9vtkcxrvzeeQdHDq6fzpoZknH1EPxv5wEo9WxgRo/OCS0etXBZq5nvkv0e3ukjeIfbXL4VU8+zVpDTvGKctFrb/CzAoAQ/P2Ii86XtBpH5CDcaAIJuAEOcdiDGKyBr+52NRa2Y3F4JtgfNKU5MF+4o6t1YK8e/mDN8Sils1aoQubXef7ZhS4p6d9HRjKmz6MX4nsx4OlQDy97NUQzwSpsKjfBHo00hrPIc+w3OWwGN0PAZBHUNxKzI9l5rO/u93mBs9RCY8JUYeiQ1THZDjXSzE= batou-example-host

examples/sensitive-values/requirements.lock

@@ -0,0 +1,19 @@
+# appenv-requirements-hash: 92e251bc834f921c996b9af42180afdf00414e12ee57291b199a6127ff5e6897
+-e ../../
+ConfigUpdater==3.2
+Jinja2==3.1.4
+MarkupSafe==2.1.5
+PyYAML==6.0.1
+certifi==2024.8.30
+charset-normalizer==3.3.2
+execnet==2.0.2
+idna==3.8
+importlib-metadata==6.7.0
+importlib-resources==5.12.0
+py==1.11.0
+remote-pdb==2.1.0
+requests==2.31.0
+setuptools==68.0.0
+typing_extensions==4.7.1
+urllib3==2.0.7
+zipp==3.15.0

examples/sensitive-values/requirements.txt

@@ -0,0 +1,2 @@
+# appenv-python-preference: 3.7,3.8,3.9,3.10,3.11,3.12
+-e ../../

examples/sensitive-values/secrets.cfg.age.clear

@@ -0,0 +1,13 @@
+[batou]
+secret_provider = age
+members = ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIACZ8++sQADp8fztgumfw2i+WSgzMHB7MgSpkM2y5pHi batou-ci-test-key
+
+[component:sensitivevalues]
+ssh_client_pubkey = ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIECZgZf7e/kI9xZv1d5HjWVZNkcqJyZ4+qLcCwqOnqrp batou-example-client
+ssh_client_privkey = -----BEGIN OPENSSH PRIVATE KEY-----
+        b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+        QyNTUxOQAAACBAmYGX+3v5CPcWb9XeR41lWTZHKicmePqi3AsKjp6q6QAAAJjlUFDw5VBQ
+        8AAAAAtzc2gtZWQyNTUxOQAAACBAmYGX+3v5CPcWb9XeR41lWTZHKicmePqi3AsKjp6q6Q
+        AAAEAcrKfEtCi3303/2AWlSFnkssvZkdHo7q/TSXw8IKxoC0CZgZf7e/kI9xZv1d5HjWVZ
+        NkcqJyZ4+qLcCwqOnqrpAAAAFGJhdG91LWV4YW1wbGUtY2xpZW50AQ==
+        -----END OPENSSH PRIVATE KEY-----

examples/sensitive-values/ssh-keys/client_ed25519

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACBAmYGX+3v5CPcWb9XeR41lWTZHKicmePqi3AsKjp6q6QAAAJjlUFDw5VBQ
+8AAAAAtzc2gtZWQyNTUxOQAAACBAmYGX+3v5CPcWb9XeR41lWTZHKicmePqi3AsKjp6q6Q
+AAAEAcrKfEtCi3303/2AWlSFnkssvZkdHo7q/TSXw8IKxoC0CZgZf7e/kI9xZv1d5HjWVZ
+NkcqJyZ4+qLcCwqOnqrpAAAAFGJhdG91LWV4YW1wbGUtY2xpZW50AQ==
+-----END OPENSSH PRIVATE KEY-----

examples/sensitive-values/ssh-keys/client_ed25519.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIECZgZf7e/kI9xZv1d5HjWVZNkcqJyZ4+qLcCwqOnqrp batou-example-client

examples/sensitive-values/ssh-keys/host_ed25519

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACDOt7gPBDZE28Hw7DR0CrkJofg932LTZRLxFaVu6ui7zAAAAJiCRjiNgkY4
+jQAAAAtzc2gtZWQyNTUxOQAAACDOt7gPBDZE28Hw7DR0CrkJofg932LTZRLxFaVu6ui7zA
+AAAEC0xgp1logK9h0DJaI81CK1IkV32aZ/t3kTNJSbNP4G4s63uA8ENkTbwfDsNHQKuQmh
++D3fYtNlEvEVpW7q6LvMAAAAEmJhdG91LWV4YW1wbGUtaG9zdAECAw==
+-----END OPENSSH PRIVATE KEY-----

examples/sensitive-values/ssh-keys/host_ed25519.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM63uA8ENkTbwfDsNHQKuQmh+D3fYtNlEvEVpW7q6LvM batou-example-host

examples/sensitive-values/ssh-keys/host_rsa

@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEAoRKUcuKLkokjb0soQl00DhIRjPIWI9RHFusZc8xPxY4kaE8sd0BQ
+D9bcgSfXL1H8rM8BeC+LPwfl8gkIFuDg3S8Osbl2Q+kBRKTLqYCdybPMCkBT/QirIcUoua
+FYGyUQzI2kcySubEcN7zO0XySfpgR/eAIMmITehrwj0w2ApudGueby2nj68nRIHofy+o0O
+eRadK1esZl+nnbM0w0ZH+RYnL8BDNhVsXm1g8h7CFfb7ZHMa783nkHRw6un86aGZJx9RD8
+b+cBKPVsYEaPzgktHrVwWauZ75L9Ht7pI3iH21y+FVPPs1aQ07xinLRa2/wswKAEPz9iIv
+Ol7QaR+Qg3GgCCbgBDnHYgxisga/udjUWtmNxeCbYHzSlOTBfuKOrdWCvHv5gzfEopbNWq
+ELm13n+2YUuKenfR0Yyps+jF+J7MeDpUA8vezVEM8EqbCo3wR6NNIazyHPsNzlsBjdDwGQ
+R1DcSsyPZeazv7vd5gbPUQmPCVGHokNUx2Q410sxAAAFiPyj5Qb8o+UGAAAAB3NzaC1yc2
+EAAAGBAKESlHLii5KJI29LKEJdNA4SEYzyFiPURxbrGXPMT8WOJGhPLHdAUA/W3IEn1y9R
+/KzPAXgviz8H5fIJCBbg4N0vDrG5dkPpAUSky6mAncmzzApAU/0IqyHFKLmhWBslEMyNpH
+MkrmxHDe8ztF8kn6YEf3gCDJiE3oa8I9MNgKbnRrnm8tp4+vJ0SB6H8vqNDnkWnStXrGZf
+p52zNMNGR/kWJy/AQzYVbF5tYPIewhX2+2RzGu/N55B0cOrp/OmhmScfUQ/G/nASj1bGBG
+j84JLR61cFmrme+S/R7e6SN4h9tcvhVTz7NWkNO8Ypy0Wtv8LMCgBD8/YiLzpe0GkfkINx
+oAgm4AQ5x2IMYrIGv7nY1FrZjcXgm2B80pTkwX7ijq3Vgrx7+YM3xKKWzVqhC5td5/tmFL
+inp30dGMqbPoxfiezHg6VAPL3s1RDPBKmwqN8EejTSGs8hz7Dc5bAY3Q8BkEdQ3ErMj2Xm
+s7+73eYGz1EJjwlRh6JDVMdkONdLMQAAAAMBAAEAAAGABNvjoIeXAEekywGwaDgZjucaom
+7XHiOUNWvIK8cZDPOZw4/H3p0RDTlFE5xZEHNftPLVr4N3puIdHK0LEm2cOu/leJUIrUnF
+IQX7otRfbis/V3vTTMnLJ8yjyt3EI6V9mT4YnOSZYmjOUc30ff5D1qVCFyOwr5UqhVP9nK
+tGm0JUztzZrJ+Dqna5ijo9qTNCIYL+IMWXTMtL6iTyzYU8PJZffkBFhsckqsCP8R3eav01
+XjVetac3ehMZKO0AFSgrw1JqWEfEMjSUpno/lWfr0WRglX6bZL7VgpzxdRace/HBVIxW6Q
+RuueSr2wHWCXtHLZcVvs5CiTCKvL73I15dNCZqgLEBikxnPGqgwwskLgZPhb50BhK+/5A4
+pZ7O9Gl50z6+HtdBxaIUZPVcHRhXQ5dsYBb3y/JPzgXeyU5oxVmPOgiqiLhfv5PJRAELXc
+xTwCy/+8b1L1jAQmTDed1piFcHTFr16q6UgZrER5ZKB8unT+4qGyTgrDkp4RvOl7BFAAAA
+wAp1GCe3wpSz7dbEfNsLGcRLcVVmt74fdGhA1IPzFKvFPr85nWmZoi6D6GBK81ZJaVSeya
+0UPV24gAq1KZ9+otTGPkZRzWpy70XYCY3myWD59icxv/jHuqvIoVor8CaGJzSNgBlfVG1O
+Tys9TQw20coUbbXNAHAE/lP4HELWJI4x7XSU35bruHx3eFUXI8wBDdoqZJCdYX9MSLRwH9
+g6mbafmi66eyHKnqNA2Xqi0J94tJkzlOyAc9IscdI76T2tpAAAAMEAzwA3YgezEuspXTDR
+cbC1LGmAwfs5e7z2k7e3YGv+0BcLwvOmM9P53+WOyzXLsPOz19SU/LIeUHVDPDEyhKuDoT
+hOrjOn9T7ZZfviHrPxZQzpG4e83By/KZwezAOgB5sr26c4jdrpidWp552r5eFeNbHqpBmZ
+8LFV/GZwqZ3Hy2WaAfCYtvE1hdDlxVyui0E9zslU38YFKPcrRTW37OdCiPwRDqxWj4pmVj
+8muMLMvz5HNAbWskX8lROwZrB6HpxvAAAAwQDHMzbaaBYlwu73hiuhQFtmYfHqODvGE3Ei
+tl4sw7xWvDiQKGmVWHi972FmeBJQ1WmvXhk4Zop8OpeofrEknWmpKx5yvwE+xN0LfcSZjS
+05q3nOy/RTMvPe7RsjNDAy+3mEixYNTTQdqM1KepxIRm3TpgmTd3wWSgAj6XDxkl3LKcYD
++WryE2cMJKF/uFeQS7zV5WdNrIbIfU6uE/ds1se/UAADY5z00ReOY5WHF+9YB7Vb4W/JQb
+F/KGCeo3fTol8AAAASYmF0b3UtZXhhbXBsZS1ob3N0AQ==
+-----END OPENSSH PRIVATE KEY-----

examples/sensitive-values/ssh-keys/host_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChEpRy4ouSiSNvSyhCXTQOEhGM8hYj1EcW6xlzzE/FjiRoTyx3QFAP1tyBJ9cvUfyszwF4L4s/B+XyCQgW4ODdLw6xuXZD6QFEpMupgJ3Js8wKQFP9CKshxSi5oVgbJRDMjaRzJK5sRw3vM7RfJJ+mBH94AgyYhN6GvCPTDYCm50a55vLaePrydEgeh/L6jQ55Fp0rV6xmX6edszTDRkf5FicvwEM2FWxebWDyHsIV9vtkcxrvzeeQdHDq6fzpoZknH1EPxv5wEo9WxgRo/OCS0etXBZq5nvkv0e3ukjeIfbXL4VU8+zVpDTvGKctFrb/CzAoAQ/P2Ii86XtBpH5CDcaAIJuAEOcdiDGKyBr+52NRa2Y3F4JtgfNKU5MF+4o6t1YK8e/mDN8Sils1aoQubXef7ZhS4p6d9HRjKmz6MX4nsx4OlQDy97NUQzwSpsKjfBHo00hrPIc+w3OWwGN0PAZBHUNxKzI9l5rO/u93mBs9RCY8JUYeiQ1THZDjXSzE= batou-example-host

src/batou/lib/file.py

@@ -79,7 +79,7 @@ class File(Component):
     leading = False
 
     # Signal that the content is sensitive data.
-    sensitive_data = False
+    sensitive_data = None
 
     def configure(self):
         self._unmapped_path = self.path
@@ -379,7 +379,7 @@ class ManagedContentBase(FileComponent):
 
     content = None
     source = ""
-    sensitive_data = False
+    sensitive_data = None
 
     # If content is given as unicode (always the case with templates)
     # then require it to be encodable. We assume UTF-8 as a sensible default
@@ -474,19 +474,22 @@ def verify(self, predicting=False):
             output.annotate("Unknown content - can't predict diff.")
             raise batou.UpdateNeeded()
 
-        if self.encoding:
-            current_text = current.decode(self.encoding, errors="replace")
-            wanted_text = self.content.decode(self.encoding, errors="replace")
-
         if not self.encoding:
             output.annotate("Not showing diff for binary data.", yellow=True)
+            raise batou.UpdateNeeded()
         elif self.sensitive_data:
             output.annotate(
                 "Not showing diff as it contains sensitive data.", red=True
             )
-        else:
-            current_lines = current_text.splitlines()
-            wanted_lines = wanted_text.splitlines()
+            raise batou.UpdateNeeded()
+
+        current_text = current.decode(self.encoding, errors="replace")
+        wanted_text = self.content.decode(self.encoding, errors="replace")
+        current_lines = current_text.splitlines()
+        wanted_lines = wanted_text.splitlines()
+
+        contains_secrets = False
+        if self.sensitive_data is None:
             words = set(
                 itertools.chain(
                     *(x.split() for x in current_lines),
@@ -497,40 +500,41 @@ def verify(self, predicting=False):
                 self.environment.secret_data.intersection(words)
             )
 
-            diff = difflib.unified_diff(current_lines, wanted_lines)
-            if not os.path.exists(self.diff_dir):
-                os.makedirs(self.diff_dir)
-            diff, diff_too_long, diff_log = limited_buffer(
-                diff, self._max_diff, self._max_diff_lead, logdir=self.diff_dir
+        diff = difflib.unified_diff(current_lines, wanted_lines)
+        if not os.path.exists(self.diff_dir):
+            os.makedirs(self.diff_dir)
+        diff, diff_too_long, diff_log = limited_buffer(
+            diff, self._max_diff, self._max_diff_lead, logdir=self.diff_dir
+        )
+
+        if contains_secrets:
+            output.line(
+                "Not showing diff as it contains sensitive data,",
+                yellow=True,
+            )
+            output.line(f"see {diff_log} for the diff.".format(), yellow=True)
+            raise batou.UpdateNeeded()
+
+        if diff_too_long:
+            output.line(
+                f"More than {self._max_diff} lines of diff. Showing first "
+                f"and last {self._max_diff_lead} lines.",
+                yellow=True,
+            )
+            output.line(
+                f"see {diff_log} for the full diff.".format(), yellow=True
+            )
+
+        for line in diff:
+            line = line.replace("\n", "")
+            if not line.strip():
+                continue
+            output.annotate(
+                f"  {os.path.basename(self.path)} {line}",
+                red=line.startswith("-"),
+                green=line.startswith("+"),
             )
 
-            if diff_too_long:
-                output.line(
-                    f"More than {self._max_diff} lines of diff. Showing first "
-                    f"and last {self._max_diff_lead} lines.",
-                    yellow=True,
-                )
-                output.line(
-                    f"see {diff_log} for the full diff.".format(), yellow=True
-                )
-            if contains_secrets:
-                output.line(
-                    "Not showing diff as it contains sensitive data,",
-                    yellow=True,
-                )
-                output.line(
-                    f"see {diff_log} for the diff.".format(), yellow=True
-                )
-            else:
-                for line in diff:
-                    line = line.replace("\n", "")
-                    if not line.strip():
-                        continue
-                    output.annotate(
-                        f"  {os.path.basename(self.path)} {line}",
-                        red=line.startswith("-"),
-                        green=line.startswith("+"),
-                    )
         raise batou.UpdateNeeded()
 
     def update(self):