ghe config

config/__init__.py

@@ -6,9 +6,11 @@
 from scm_services import \
     bitbucketdc_cloner_factory, \
     adoe_cloner_factory, \
+    gh_cloner_factory, \
     adoe_api_auth_factory, \
     bbdc_api_auth_factory, \
-    SCMService, ADOEService, BBDCService
+    github_api_auth_factory, \
+    SCMService, ADOEService, BBDCService, GHService
 from api_utils import APISession
 from cxone_service import CxOneService
 from password_strength import PasswordPolicy
@@ -18,6 +20,7 @@
 from workflows import ResultSeverity, ResultStates
 from typing import Tuple, List
 from multiprocessing import cpu_count
+from typing import Dict, List
 
 
 def get_workers_count():
@@ -64,10 +67,23 @@ def missing_at_least_one_key_path(key_path, keys):
 
     @staticmethod
     def mutually_exclusive(key_path, keys):
-        return ConfigurationException(f"Only one of these keys should be defined: {["/".join([key_path, x]) for x in keys]}")
+        report_list = []
+        for k in keys:
+            if isinstance(k, str):
+                report_list.append("/".join([key_path, k]))
+
+            if isinstance(k, Tuple) or isinstance(k, List):
+                report_list.append(f"{key_path}/({",".join(k)})")
+
+
+        return ConfigurationException(f"Only one should be defined: {report_list}")
+
+    @staticmethod
+    def key_mismatch(key_path, provided, needed):
+        return ConfigurationException(f"{key_path} invalid: Needed {needed} but provided {provided}.")
 
     @staticmethod
-    def invalid_keys(key_path, keys):
+    def invalid_keys(key_path, keys : List):
         return ConfigurationException(f"These keys are invalid: {["/".join([key_path, x]) for x in keys]}")
 
 class RouteNotFoundException(Exception):
@@ -195,7 +211,7 @@ def __get_secret_from_value_of_key_or_default(config_dict, key, default):
                 return SecretRegistry.register(default)
             else:
                 with open(Path(CxOneFlowConfig.__secret_root) / Path(config_dict[key]), "rt") as secret:
-                    return SecretRegistry.register(secret.readline().strip())
+                    return SecretRegistry.register(secret.read().strip())
 
     @staticmethod
     def __get_secret_from_value_of_key_or_fail(config_path, key, config_dict):
@@ -330,45 +346,62 @@ def __cxone_client_factory(config_path, **kwargs):
     __ordered_scm_config_tuples = {}
     __scm_config_tuples_by_service_moniker = {}
 
-    __minimum_api_auth_keys = ['token', 'password']
-    __basic_auth_keys = ['username', 'password']
-    __all_possible_api_auth_keys = list(set(__minimum_api_auth_keys + __basic_auth_keys))
+    __oauth_auth_keys = [tuple(sorted(('oauth-secret', 'oauth-id')))]
+    __basic_auth_keys = [tuple(sorted(('username', 'password')))]
+    __token_auth_keys = [('token',)]
+    __all_possible_api_auth_keys = list(set(__token_auth_keys + __basic_auth_keys + __oauth_auth_keys))
 
-    __minimum_clone_auth_keys = __minimum_api_auth_keys + ['ssh']
-    __all_possible_clone_auth_keys = list(set(__minimum_clone_auth_keys + __basic_auth_keys + ['ssh-port']))
+    __minimum_clone_auth_keys = __all_possible_api_auth_keys + [('ssh',)]
+    __all_possible_clone_auth_keys = list(set(__minimum_clone_auth_keys + [('ssh-port',)]))
 
     @staticmethod
     def __scm_api_auth_factory(api_auth_factory, config_dict, config_path):
+        if config_dict is not None and len(config_dict.keys()) > 0:
+
+            CxOneFlowConfig.__validate_auth_keys(config_dict, CxOneFlowConfig.__all_possible_api_auth_keys, config_path)
 
-        CxOneFlowConfig.__validate_no_extra_auth_keys(config_dict, CxOneFlowConfig.__all_possible_api_auth_keys, config_path)
-
-        if len(CxOneFlowConfig.__validate_minimum_auth_keys(config_dict, CxOneFlowConfig.__minimum_api_auth_keys, config_path)) > 0:
             return api_auth_factory(CxOneFlowConfig.__get_secret_from_value_of_key_or_default(config_dict, "username", None),
                                     CxOneFlowConfig.__get_secret_from_value_of_key_or_default(config_dict, "password", None),
-                                    CxOneFlowConfig.__get_secret_from_value_of_key_or_default(config_dict, "token", None))
+                                    CxOneFlowConfig.__get_secret_from_value_of_key_or_default(config_dict, "token", None),
+                                    CxOneFlowConfig.__get_secret_from_value_of_key_or_default(config_dict, "oauth-secret", None),
+                                    CxOneFlowConfig.__get_secret_from_value_of_key_or_default(config_dict, "oauth-id", None))
 
         raise ConfigurationException(f"{config_path} SCM API authorization configuration is invalid!")
 
     @staticmethod
-    def __validate_minimum_auth_keys(config_dict, valid_keys, config_path):
-        auth_type_keys = [x for x in config_dict.keys() if x in valid_keys]
-        if len(auth_type_keys) > 1:
-            raise ConfigurationException.mutually_exclusive(config_path, auth_type_keys)
-        return auth_type_keys
-
+    def __validate_auth_keys(config_dict : Dict, valid_keys : List, config_path : str):
+        found_count = {}
+
+        for k in config_dict.keys():
+            found = False
+            for valid_tuple in valid_keys:
+                if k in valid_tuple:
+                    if valid_tuple not in found_count.keys():
+                        found_count[valid_tuple] = 1
+                    else:
+                        found_count[valid_tuple] += 1
+                    found = True
+                    break
+            if not found:
+                raise ConfigurationException.invalid_keys(config_path, [k])
+
+        # Check that only one set of keys was provided.
+        if len(found_count.keys()) > 1:
+            raise ConfigurationException.mutually_exclusive(config_path, found_count.keys())
+
+        # Check for missing keys in found sets.
+        for fk in found_count.keys():
+            if found_count[fk] != len(fk):
+                raise ConfigurationException.missing_keys(config_path, fk)
+
 
-    @staticmethod
-    def __validate_no_extra_auth_keys(config_dict, valid_keys, config_path):
-        extra_passed_keys = config_dict.keys() - valid_keys
 
-        if len(extra_passed_keys) > 0:
-            raise ConfigurationException.invalid_keys(config_path, extra_passed_keys)
 
 
     @staticmethod
     def __cloner_factory(scm_cloner_factory, clone_auth_dict, config_path):
 
-        CxOneFlowConfig.__validate_no_extra_auth_keys(clone_auth_dict, CxOneFlowConfig.__all_possible_clone_auth_keys, config_path)
+        CxOneFlowConfig.__validate_auth_keys(clone_auth_dict, CxOneFlowConfig.__all_possible_clone_auth_keys, config_path)
 
         ssh_secret = CxOneFlowConfig.__get_value_for_key_or_default('ssh', clone_auth_dict, None)
         if ssh_secret is not None:
@@ -378,7 +411,9 @@ def __cloner_factory(scm_cloner_factory, clone_auth_dict, config_path):
                                   CxOneFlowConfig.__get_secret_from_value_of_key_or_default(clone_auth_dict, 'password', None),
                                   CxOneFlowConfig.__get_secret_from_value_of_key_or_default(clone_auth_dict, 'token', None),
                                   ssh_secret,
-                                  CxOneFlowConfig.__get_value_for_key_or_default('ssh-port', clone_auth_dict, None))
+                                  CxOneFlowConfig.__get_value_for_key_or_default('ssh-port', clone_auth_dict, None),
+                                  CxOneFlowConfig.__get_value_for_key_or_default('oauth-secret', clone_auth_dict, None),
+                                  CxOneFlowConfig.__get_value_for_key_or_default('oauth-id', clone_auth_dict, None))
 
         if retval is None:
             raise ConfigurationException(f"{config_path} SCM clone authorization configuration is invalid!")
@@ -436,15 +471,21 @@ def __setup_scm(cloner_factory, api_auth_factory, scm_class, config_dict, config
 
     __cloner_factories = {
         'bbdc' : bitbucketdc_cloner_factory,
-        'adoe' : adoe_cloner_factory }
+        'adoe' : adoe_cloner_factory,
+        'gh' : gh_cloner_factory
+        }
 
     __auth_factories = {
         'bbdc' : bbdc_api_auth_factory,
-        'adoe' : adoe_api_auth_factory }
+        'adoe' : adoe_api_auth_factory,
+        'gh' : github_api_auth_factory
+        }
 
     __scm_factories = {
         'bbdc' : BBDCService,
-        'adoe' : ADOEService }
+        'adoe' : ADOEService,
+        'gh' : BBDCService
+        }
 
 
 

orchestration/__init__.py

@@ -1,6 +1,7 @@
 from api_utils import verify_signature
 from .bbdc import BitBucketDataCenterOrchestrator
 from .adoe import AzureDevOpsEnterpriseOrchestrator
+from .gh import GithubOrchestrator
 import logging
 from config import CxOneFlowConfig
 
@@ -25,7 +26,7 @@ async def execute(orchestrator):
         if await orchestrator.is_signature_valid(scm_service.shared_secret):
             return await orchestrator.execute(cxone_service, scm_service, workflow_service)
         else:
-            OrchestrationDispatch.log().warn(f"Payload signature validation failed, webhook payload ignored.")
+            OrchestrationDispatch.log().warning(f"Payload signature validation failed, webhook payload ignored.")
 
 
 

orchestration/gh.py

@@ -0,0 +1,42 @@
+from .base import OrchestratorBase
+from api_utils import signature
+import logging
+
+class GithubOrchestrator(OrchestratorBase):
+
+
+    @staticmethod
+    def log():
+        return logging.getLogger("BitBucketDataCenterOrchestrator")
+
+    @property
+    def config_key(self):
+        return "gh"
+
+    @property
+    def is_diagnostic(self) -> bool:
+        return self.__isdiagnostic
+
+
+    def __init__(self, headers : dict, webhook_payload : dict):
+        OrchestratorBase.__init__(self, headers, webhook_payload)
+
+        self.__isdiagnostic = False
+
+        self.__event = self.get_header_key_safe('X-GitHub-Event') 
+
+        if not self.__event is None and self.__event == "ping":
+            self.__isdiagnostic = True
+            return
+
+
+    async def is_signature_valid(self, shared_secret : str) -> bool:
+        sig = self.get_header_key_safe('X-Hub-Signature-256')
+        if sig is None:
+            GithubOrchestrator.log().warning("X-Hub-Signature-256 header is missing, rejecting.")
+            return False
+
+        hashalg,hash = sig.split("=")
+        payload_hash = signature.get(hashalg, shared_secret, self._webhook_payload)
+
+        return hash == payload_hash

scm_services/__init__.py

@@ -3,10 +3,17 @@
 from .scm import SCMService
 from .adoe import ADOEService
 from .bbdc import BBDCService
+from .gh import GHService
 from api_utils import auth_basic, auth_bearer
 
+class ScmCloneAuthSupportException(Exception):
+        pass
+
+
+def bitbucketdc_cloner_factory(username=None, password=None, token=None, ssh_path=None, ssh_port=None, oauth_secret : str=None, oauth_id : str=None) -> Cloner:
+        if oauth_secret is not None or oauth_id is not None:
+                raise ScmCloneAuthSupportException("OAuth clone authentication not supported for BBDC")
 
-def bitbucketdc_cloner_factory(username=None, password=None, token=None, ssh_path=None, ssh_port=None) -> Cloner:
         if username is not None and password is not None:
                 return Cloner.using_basic_auth(username, password, True) 
 
@@ -18,7 +25,10 @@ def bitbucketdc_cloner_factory(username=None, password=None, token=None, ssh_pat
 
         return None        
 
-def adoe_cloner_factory(username=None, password=None, token=None, ssh_path=None, ssh_port=None) -> Cloner:
+def adoe_cloner_factory(username=None, password=None, token=None, ssh_path=None, ssh_port=None, oauth_secret : str=None, oauth_id : str=None) -> Cloner:
+        if oauth_secret is not None or oauth_id is not None:
+                raise ScmCloneAuthSupportException("OAuth clone authentication not supported for ADO")
+
         if username is not None and password is not None:
                 return Cloner.using_basic_auth(username, password, True) 
 
@@ -29,15 +39,44 @@ def adoe_cloner_factory(username=None, password=None, token=None, ssh_path=None,
                 return Cloner.using_ssh_auth(ssh_path, ssh_port)
 
 
-def adoe_api_auth_factory(username=None, password=None, token=None) -> AuthBase:
+def gh_cloner_factory(username=None, password=None, token=None, ssh_path=None, ssh_port=None, oauth_secret : str=None, oauth_id : str=None) -> Cloner:
+        return Cloner()
+
+
+class ScmApiAuthSupportException(Exception):
+        pass
+
+
+def adoe_api_auth_factory(username : str=None, password : str=None, token : str=None, oauth_secret : str=None, oauth_id : str=None) -> AuthBase:
+        if oauth_secret is not None or oauth_id is not None:
+                raise ScmApiAuthSupportException("OAuth API authentication not supported for ADO")
+
         if token is not None:
                 return auth_basic("", token)
-        else:
+        elif username is not None and password is not None:
                 return auth_basic(username, password)
+        else:
+                raise ScmApiAuthSupportException("Unable to determine API auth method.")
+
 
+def bbdc_api_auth_factory(username=None, password=None, token=None, oauth_secret : str=None, oauth_id : str=None) -> AuthBase:
+        if oauth_secret is not None or oauth_id is not None:
+                raise ScmApiAuthSupportException("OAuth API authentication not supported for BBDC")
 
-def bbdc_api_auth_factory(username=None, password=None, token=None) -> AuthBase:
         if token is not None:
                 return auth_bearer(token)
+        elif username is not None and password is not None:
+                return auth_basic(username, password)
         else:
+                raise ScmApiAuthSupportException("Unable to determine API auth method.")
+
+def github_api_auth_factory(username=None, password=None, token=None, oauth_secret : str=None, oauth_id : str=None) -> AuthBase:
+        if token is not None:
+                return auth_bearer(token)
+        elif username is not None and password is not None:
                 return auth_basic(username, password)
+        elif oauth_id is not None and oauth_secret is not None:
+                pass
+        else:
+                raise ScmApiAuthSupportException("Unable to determine API auth method.")
+

scm_services/gh.py

@@ -0,0 +1,5 @@
+from .scm import SCMService
+
+
+class GHService(SCMService):
+    pass

wsgi.py

@@ -6,7 +6,8 @@
 """
 from _agent import __agent__
 from flask import Flask, request, Response, send_from_directory
-from orchestration import OrchestrationDispatch, BitBucketDataCenterOrchestrator, AzureDevOpsEnterpriseOrchestrator
+from orchestration import OrchestrationDispatch, BitBucketDataCenterOrchestrator, \
+    AzureDevOpsEnterpriseOrchestrator, GithubOrchestrator
 import json, logging, asyncio, os
 from config import CxOneFlowConfig, ConfigurationException, get_config_path
 from time import perf_counter_ns
@@ -49,6 +50,26 @@ async def bbdc_webhook_endpoint():
         __log.exception(ex)
         return Response(status=400)
 
+@app.post("/gh")
+async def github_webhook_endpoint():
+    __log.info("Received hook for Github")
+    __log.debug(f"github webhook: headers: [{request.headers}] body: [{json.dumps(request.json)}]")
+    try:
+        orch = GithubOrchestrator(request.headers, request.data)
+
+        if not orch.is_diagnostic:
+            TaskManager.in_background(OrchestrationDispatch.execute(orch))
+            return Response(status=204)
+        else:
+            # The ping has no route URL associated, so check if any route matches.
+            for service in CxOneFlowConfig.retrieve_scm_services(orch.config_key):
+                if await orch.is_signature_valid(service.shared_secret):
+                    return Response(status=200)
+            return Response(status=401)
+
+    except Exception as ex:
+        __log.exception(ex)
+        return Response(status=400)
 
 @app.post("/adoe")
 async def adoe_webhook_endpoint():