From dfca0d13c6a5cfc80437f25c333852f7d392bf22 Mon Sep 17 00:00:00 2001 From: rjhaverkamp Date: Tue, 31 Oct 2023 11:33:00 +0100 Subject: [PATCH 1/2] add sops as a secret reslover to himl. Sops code from: https://github.com/ansible-collections/community.sops/tree/main --- examples/secrets/default.yaml | 1 + himl/secret_resolvers.py | 11 ++- himl/simplesops.py | 126 ++++++++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 himl/simplesops.py diff --git a/examples/secrets/default.yaml b/examples/secrets/default.yaml index f0a9dc2..5e4adc9 100644 --- a/examples/secrets/default.yaml +++ b/examples/secrets/default.yaml @@ -1,3 +1,4 @@ --- secret_path_v2: "{{vault.path(/kv2_secret)}}" secret_key_v2: "{{vault.key(/kv2_secret/key)}}" +sops_secret: "{{ sops.secret_file(/home/user/secrets/secret_file.yaml).secret_key(['s3']['access_key']) }}" diff --git a/himl/secret_resolvers.py b/himl/secret_resolvers.py index 0558c36..c8fbe76 100644 --- a/himl/secret_resolvers.py +++ b/himl/secret_resolvers.py @@ -13,6 +13,7 @@ from .simplessm import SimpleSSM from .simples3 import SimpleS3 from .simplevault import SimpleVault +from .simplesops import SimpleSops class SecretResolver: @@ -68,6 +69,14 @@ def resolve(self, secret_type, secret_params): s3 = SimpleS3(aws_profile, region_name) return s3.get(bucket, path, base64Encode) +class SopsSecretResolver(SecretResolver): + def supports(self, secret_type): + return secret_type == "sops" + + def resolve(self, secret_type, secret_params): + file = self.get_param_or_exception("secret_file", secret_params) + sops = SimpleSops() + return sops.get(secret_file=file, secret_key=secret_params.get("secret_key")) class VaultSecretResolver(SecretResolver): def supports(self, secret_type): @@ -96,7 +105,7 @@ def resolve(self, secret_type, secret_params): class AggregatedSecretResolver(SecretResolver): def __init__(self, default_aws_profile=None): self.secret_resolvers = (SSMSecretResolver(default_aws_profile), S3SecretResolver(default_aws_profile), - VaultSecretResolver()) + VaultSecretResolver(), SopsSecretResolver()) def supports(self, secret_type): return any([resolver.supports(secret_type) for resolver in self.secret_resolvers]) diff --git a/himl/simplesops.py b/himl/simplesops.py new file mode 100644 index 0000000..d922824 --- /dev/null +++ b/himl/simplesops.py @@ -0,0 +1,126 @@ +# Copyright (c), Edoardo Tenani , 2018-2020 +# Simplified BSD License (see LICENSES/BSD-2-Clause.txt or https://opensource.org/licenses/BSD-2-Clause) +# SPDX-License-Identifier: BSD-2-Clause + +from __future__ import absolute_import, division, print_function + +import os, logging + +from subprocess import Popen, PIPE + +logger = logging.getLogger(__name__) + +# From https://github.com/getsops/sops/blob/master/cmd/sops/codes/codes.go +# Should be manually updated +SOPS_ERROR_CODES = { + 1: "ErrorGeneric", + 2: "CouldNotReadInputFile", + 3: "CouldNotWriteOutputFile", + 4: "ErrorDumpingTree", + 5: "ErrorReadingConfig", + 6: "ErrorInvalidKMSEncryptionContextFormat", + 7: "ErrorInvalidSetFormat", + 8: "ErrorConflictingParameters", + 21: "ErrorEncryptingMac", + 23: "ErrorEncryptingTree", + 24: "ErrorDecryptingMac", + 25: "ErrorDecryptingTree", + 49: "CannotChangeKeysFromNonExistentFile", + 51: "MacMismatch", + 52: "MacNotFound", + 61: "ConfigFileNotFound", + 85: "KeyboardInterrupt", + 91: "InvalidTreePathFormat", + 100: "NoFileSpecified", + 128: "CouldNotRetrieveKey", + 111: "NoEncryptionKeyFound", + 200: "FileHasNotBeenModified", + 201: "NoEditorFound", + 202: "FailedToCompareVersions", + 203: "FileAlreadyEncrypted", +} + + +class SopsError(Exception): + """Extend Exception class with sops specific information""" + + def __init__(self, filename, exit_code, message, decryption=True): + if exit_code in SOPS_ERROR_CODES: + exception_name = SOPS_ERROR_CODES[exit_code] + message = "error with file %s: %s exited with code %d: %s" % ( + filename, + exception_name, + exit_code, + message.decode("utf-8"), + ) + else: + message = ( + "could not %s file %s; Unknown sops error code: %s; message: %s" + % ( + "decrypt" if decryption else "encrypt", + filename, + exit_code, + message.decode("utf-8"), + ) + ) + super(SopsError, self).__init__(message) + + +class Sops: + """Utility class to perform sops CLI actions""" + + @staticmethod + def decrypt( + encrypted_file, + secret_key=None, + decode_output=True, + rstrip=True, + ): + command = ["sops"] + env = os.environ.copy() + if secret_key is None: + raise Exception( + "Error while getting secret for %s: secret key not supplied" + % encrypted_file + ) + command.extend(["--extract", secret_key]) + command.extend(["--decrypt", encrypted_file]) + print(command) + process = Popen( + command, + stdin=None, + stdout=PIPE, + stderr=PIPE, + env=env, + ) + (output, err) = process.communicate() + exit_code = process.returncode + + if decode_output: + # output is binary, we want UTF-8 string + output = output.decode("utf-8", errors="surrogate_or_strict") + # the process output is the decrypted secret; be cautious + if exit_code != 0: + raise SopsError(encrypted_file, exit_code, err, decryption=True) + + if rstrip: + output = output.rstrip() + + return output + + +class SimpleSops: + def __init__(self): + pass + + def get(self, secret_file, secret_key): + try: + logger.info( + "Resolving sops secret %s from file %s", secret_key, secret_file + ) + return Sops.decrypt(encrypted_file=secret_file, secret_key=secret_key) + except SopsError as e: + raise Exception( + "Error while trying to read sops value for file %s, key: %s - %s" + % (secret_file, secret_key, e) + ) diff --git a/setup.py b/setup.py index 37a125f..4ba3326 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name='himl', - version="0.15.0", + version="0.16.0", description='A hierarchical config using yaml', long_description=_readme + '\n\n', long_description_content_type='text/markdown', From 264ab2c297d99705ce1286e6220c58001533901b Mon Sep 17 00:00:00 2001 From: rjhaverkamp Date: Tue, 31 Oct 2023 13:47:45 +0100 Subject: [PATCH 2/2] add caching for sops resolver. This prevents having to touch the yubikey on every secret resolve --- himl/simplesops.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/himl/simplesops.py b/himl/simplesops.py index d922824..8fdf90f 100644 --- a/himl/simplesops.py +++ b/himl/simplesops.py @@ -3,8 +3,9 @@ # SPDX-License-Identifier: BSD-2-Clause from __future__ import absolute_import, division, print_function +from functools import lru_cache -import os, logging +import os, logging, yaml from subprocess import Popen, PIPE @@ -51,7 +52,7 @@ def __init__(self, filename, exit_code, message, decryption=True): filename, exception_name, exit_code, - message.decode("utf-8"), + message, ) else: message = ( @@ -60,7 +61,7 @@ def __init__(self, filename, exit_code, message, decryption=True): "decrypt" if decryption else "encrypt", filename, exit_code, - message.decode("utf-8"), + message, ) ) super(SopsError, self).__init__(message) @@ -69,23 +70,16 @@ def __init__(self, filename, exit_code, message, decryption=True): class Sops: """Utility class to perform sops CLI actions""" - @staticmethod + @lru_cache(maxsize=2048) def decrypt( encrypted_file, - secret_key=None, decode_output=True, rstrip=True, ): command = ["sops"] env = os.environ.copy() - if secret_key is None: - raise Exception( - "Error while getting secret for %s: secret key not supplied" - % encrypted_file - ) - command.extend(["--extract", secret_key]) + command.extend(["--decrypt", encrypted_file]) - print(command) process = Popen( command, stdin=None, @@ -105,8 +99,18 @@ def decrypt( if rstrip: output = output.rstrip() - - return output + return yaml.full_load(output) + + def get_keys(self, secret_file, secret_key): + result = Sops.decrypt(secret_file) + secret_key_path = secret_key.strip("[]") + keys = [key.strip("'") for key in secret_key_path.split("']['")] + try: + for key in keys: + result = result[key] + except KeyError as e: + raise SopsError(secret_file, 128, "Encountered KeyError parsing yaml for key: %s" % secret_key, decryption=True) + return result class SimpleSops: @@ -118,7 +122,7 @@ def get(self, secret_file, secret_key): logger.info( "Resolving sops secret %s from file %s", secret_key, secret_file ) - return Sops.decrypt(encrypted_file=secret_file, secret_key=secret_key) + return Sops().get_keys(secret_file=secret_file, secret_key=secret_key) except SopsError as e: raise Exception( "Error while trying to read sops value for file %s, key: %s - %s"