redacting secrets

checkmarx-ts · May 16, 2024 · 73270c4 · 73270c4
1 parent 6a9e5b9
commit 73270c4
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 30 deletions.
diff --git a/config/__init__.py b/config/__init__.py
@@ -12,6 +12,7 @@
 from api_utils import APISession
 from cxone_service import CxOneService
 from password_strength import PasswordPolicy
+from cxoneflow_logging import SecretRegistry
 
 class ConfigurationException(Exception):
 
@@ -73,26 +74,31 @@ def retrieve_services_by_route(clone_urls):
     @staticmethod
     def bootstrap(config_file_path = "./config.yaml"):
 
-        CxOneFlowConfig.log().info(f"Loading configuration from {config_file_path}")
+        try:
+            CxOneFlowConfig.log().info(f"Loading configuration from {config_file_path}")
 
-        with open(config_file_path, "rt") as cfg:
-            CxOneFlowConfig.__raw = yaml.safe_load(cfg)
+            with open(config_file_path, "rt") as cfg:
+                CxOneFlowConfig.__raw = yaml.safe_load(cfg)
 
-        if not "secret-root-path" in CxOneFlowConfig.__raw.keys():
-            raise ConfigurationException.missing_key_path("/secret-root-path")
-        else:
-            CxOneFlowConfig.__secret_root = CxOneFlowConfig.__raw['secret-root-path']
-
-        if len(CxOneFlowConfig.__raw.keys() - CxOneFlowConfig.__cloner_factories.keys()) == len(CxOneFlowConfig.__raw.keys()):
-            raise ConfigurationException.missing_at_least_one_key_path("/", CxOneFlowConfig.__cloner_factories.keys())
-
-        for scm in CxOneFlowConfig.__cloner_factories.keys():
-
-            if scm in CxOneFlowConfig.__raw.keys():
-                index = 0
-                for repo_config_dict in CxOneFlowConfig.__raw[scm]:
-                    CxOneFlowConfig.__setup_scm(CxOneFlowConfig.__cloner_factories[scm], CxOneFlowConfig.__auth_factories[scm], repo_config_dict, f"/{scm}[{index}]")
-                    index += 1
+            if not "secret-root-path" in CxOneFlowConfig.__raw.keys():
+                raise ConfigurationException.missing_key_path("/secret-root-path")
+            else:
+                CxOneFlowConfig.__secret_root = CxOneFlowConfig.__raw['secret-root-path']
+
+            if len(CxOneFlowConfig.__raw.keys() - CxOneFlowConfig.__cloner_factories.keys()) == len(CxOneFlowConfig.__raw.keys()):
+                raise ConfigurationException.missing_at_least_one_key_path("/", CxOneFlowConfig.__cloner_factories.keys())
+
+            for scm in CxOneFlowConfig.__cloner_factories.keys():
+
+                if scm in CxOneFlowConfig.__raw.keys():
+                    index = 0
+                    for repo_config_dict in CxOneFlowConfig.__raw[scm]:
+                        CxOneFlowConfig.__setup_scm(CxOneFlowConfig.__cloner_factories[scm], CxOneFlowConfig.__auth_factories[scm], repo_config_dict, f"/{scm}[{index}]")
+                        index += 1
+        except Exception as ex:
+            CxOneFlowConfig.log().exception(ex)
+            raise
+
 
     @staticmethod
     def __get_value_for_key_or_fail(config_path, key, config_dict):
@@ -104,13 +110,13 @@ def __get_value_for_key_or_fail(config_path, key, config_dict):
     @staticmethod
     def __get_secret_from_value_of_key_or_default(config_dict, key, default):
         if not key in config_dict.keys():
-            return default
+            return SecretRegistry.register(default)
         else:
             if not os.path.isfile(Path(CxOneFlowConfig.__secret_root) / Path(config_dict[key])):
-                return default
+                return SecretRegistry.register(default)
             else:
                 with open(Path(CxOneFlowConfig.__secret_root) / Path(config_dict[key]), "rt") as secret:
-                    return secret.readline().strip()
+                    return SecretRegistry.register(secret.readline().strip())
 
     @staticmethod
     def __get_secret_from_value_of_key_or_fail(config_path, key, config_dict):

diff --git a/cxoneflow_logging/__init__.py b/cxoneflow_logging/__init__.py
@@ -1,12 +1,34 @@
 import logging, logging.config, json, os, pathlib, re
+from threading import Lock
 
-class RedactingStreamHandler(logging.StreamHandler):
+class SecretRegistry:
+     __lock = Lock()
+     __default_regex = "Authorization: .+ (?P<header>.*)$"
+     __compiled_regex = re.compile(__default_regex,  re.RegexFlag.I | re.RegexFlag.M)
+     __secrets = []
+
+     @staticmethod
+     def register(secret : str) -> str:
+        if secret is not None:
+            with SecretRegistry.__lock:
+                if not secret in SecretRegistry.__secrets:
+                    SecretRegistry.__secrets.append(re.escape(secret.replace("\n", "").replace("\r", "")))
+                    SecretRegistry.__compiled_regex = re.compile(f"{SecretRegistry.__default_regex}|(?P<any>{'|'.join(SecretRegistry.__secrets)})", \
+                                                                 re.RegexFlag.I | re.RegexFlag.M)
+        return secret
+
 
-    __secret_matcher = re.compile("Authorization: .+ (?P<header>.*)$|http[s]?://(?P<url>.*?:.*?)@", re.RegexFlag.I | re.RegexFlag.M)
+     @staticmethod
+     def get_match_iter(logmsg : str) -> re.Match:
+          with SecretRegistry.__lock:
+              return SecretRegistry.__compiled_regex.finditer(logmsg)
+
 
+class RedactingStreamHandler(logging.StreamHandler):
+
     def format(self, record):
         msg = super().format(record)
-        for secret in RedactingStreamHandler.__secret_matcher.finditer(msg):
+        for secret in SecretRegistry.get_match_iter(msg):
             for m in secret.groupdict().keys():
                 msg = msg[0:secret.start(m)] + ('*' * (secret.end(m) - secret.start(m))) + msg[secret.end(m):]
 

diff --git a/scm_services/cloner.py b/scm_services/cloner.py
@@ -1,5 +1,5 @@
 import os, tempfile, shutil, shlex, subprocess, asyncio, logging, urllib, base64, re
-
+from cxoneflow_logging import SecretRegistry
 
 class Cloner:
 
@@ -19,7 +19,7 @@ def log():
     @staticmethod
     def __insert_creds_in_url(url, username, password):
         split = urllib.parse.urlsplit(url)
-        new_netloc = f"{urllib.parse.quote(username, safe='') if username is not None else 'git'}:{urllib.parse.quote(password, safe='')}@{split.netloc}"
+        new_netloc = f"{urllib.parse.quote(username, safe='') if username is not None else 'git'}:{SecretRegistry.register(urllib.parse.quote(password, safe=''))}@{split.netloc}"
         return urllib.parse.urlunsplit((split.scheme, new_netloc, split.path, split.query, split.fragment))
 
     @staticmethod
@@ -31,7 +31,7 @@ def using_basic_auth(username, password, in_header=False):
         retval.__supported_protocols = Cloner.__http_protocols
         retval.__port = None
         retval.__username = username
-        retval.__password = password
+        retval.__password = SecretRegistry.register(password)
 
         if not in_header:
             retval.__clone_cmd_stub = ["git", "clone"]
@@ -96,10 +96,10 @@ def clone(self, clone_url):
         Cloner.log().debug(f"Clone Execution for: {clone_url}")
 
         fixed_clone_url = self.__fix_clone_url(clone_url)
-
         clone_output_loc = tempfile.TemporaryDirectory(delete=False)
-        thread = asyncio.to_thread(subprocess.run, self.__clone_cmd_stub + [fixed_clone_url, clone_output_loc.name], \
-                                   capture_output=True, env=self.__env, check=True)
+        cmd = self.__clone_cmd_stub + [fixed_clone_url, clone_output_loc.name]
+        Cloner.log().debug(cmd)
+        thread = asyncio.to_thread(subprocess.run, cmd, capture_output=True, env=self.__env, check=True)
 
         return Cloner.__clone_worker(thread, clone_output_loc)