Handle 'None' string in fence config

fence/config.py

@@ -1,3 +1,4 @@
+import collections
 import os
 from yaml import safe_load as yaml_load
 import urllib.parse
@@ -6,7 +7,6 @@
 from gen3config import Config
 
 from cdislogging import get_logger
-from collections import Counter
 from typing import List, Dict, Any
 
 logger = get_logger(__name__)
@@ -155,7 +155,66 @@ def post_process(self):
         self._validate_dbgap_config(self._configs["dbGaP"])
 
     @staticmethod
-    def find_duplicates(collection):
+    def _get_parent_studies_safely(dbgap_config):
+        """
+        This will get a list of parent id's from the dbgap config's mapping property
+        or return and empty array if the entry doesn't exist
+        Args:
+            dbgap_config: { parent_to_child_studies_mapping: { k1: v1, ... } }
+        Returns:
+            [k1, k2, ...]
+        """
+        study_mapping = dbgap_config.get('parent_to_child_studies_mapping', {})
+        # study mapping could be 'None'
+        safe_studies = list(study_mapping.keys()) if isinstance(study_mapping, dict) else []
+        return safe_studies
+
+    # TODO(fix-it): These functions should go in utils.py, but cannot be migrated until
+    # the variable DEFAULT_BACKOFF_SETTINGS is migrated here
+    @staticmethod
+    def _some(predicate, iterable, found_nothing_value=None):
+        """ Returns the first element that satisfies the predicate, else fail value
+            Expects predicate to be a boolean lambda and iterable to be a collection
+        """
+        for item in iterable:
+            result = predicate(item)
+            if result:
+                return item
+        return found_nothing_value
+
+    @staticmethod
+    def _coerce_to_array(unknown_data_structure, exception_message="Unrecognized data structure, aborting!"):
+        """
+        Given a dubious data structure, coerces it into a list
+        depending on the data type.
+        Currently, handles list, dict, and None
+        Args:
+            exception_message: any string for more context
+            unknown_data_structure: either a list or dict, fails otherwise
+
+        Returns: the data structure if it's a list,
+        or a list with the data structure in it if it's a dictionary
+
+        note: fails if the unknown data structure is any other type
+        """
+        # TODO: these three functions can be move out on their own when migrated to utils.py
+        identity = lambda v: v
+        wrap_in_array = lambda v: [v]
+        empty_array = lambda _: []
+        # END TODO
+        data_is = lambda data_type: isinstance(unknown_data_structure, data_type)
+        type_to_coercion = {
+            list: identity,
+            dict: wrap_in_array,
+            type(None): empty_array,
+        }
+        data_type_of_parameter = FenceConfig._some(data_is, list(type_to_coercion.keys()), TypeError)
+        if data_type_of_parameter is TypeError:
+            raise ValueError(exception_message + f" | Type: {type(unknown_data_structure)}")
+        return type_to_coercion[data_type_of_parameter](unknown_data_structure)
+
+    @staticmethod
+    def _find_duplicates(collection):
         """
 
         Args:
@@ -164,29 +223,17 @@ def find_duplicates(collection):
         Returns:
             a set of items that were found more than once in the collection
 
-        note: if collection is a set, this function will never find duplicates
-        (per the definition of a set)
+        note: if collection is a dictionary, counter will treat it as a
+        duplicates mapping!
         """
-        item_with_occurrences = Counter(collection)
+        item_with_occurrences = collections.Counter(collection)
         duplicates = {item
                       for item, occurrences in item_with_occurrences.items()
                       if occurrences > 1}
         return duplicates
 
-    @staticmethod
-    def get_parent_studies_safely(dbgap_config):
-        """
-        This will get a list of parent id's from the dbgap config's mapping property
-        or return and empty array if the entry doesn't exist
-        Args:
-            dbgap_config: { parent_to_child_studies_mapping: { k1: v1, ... } }
-        Returns:
-            [k1, k2, ...]
-        """
-        study_mapping = dbgap_config.get('parent_to_child_studies_mapping', {})
-        # study mapping could be 'None'
-        safe_studies = list(study_mapping.keys()) if isinstance(study_mapping, dict) else []
-        return safe_studies
+    # END TODO
+
 
     @staticmethod
     def _validate_parent_child_studies(dbgap_configs: List[Dict[str, Any]]) -> None:
@@ -200,28 +247,19 @@ def _validate_parent_child_studies(dbgap_configs: List[Dict[str, Any]]) -> None:
         """
         safe_list_of_parent_studies = []
         for dbgap_config in dbgap_configs:
-            safe_parent_studies = FenceConfig.get_parent_studies_safely(dbgap_config)
+            safe_parent_studies = FenceConfig._get_parent_studies_safely(dbgap_config)
             safe_list_of_parent_studies.extend(safe_parent_studies)
 
-        duplicates = FenceConfig.find_duplicates(safe_list_of_parent_studies)
+        duplicates = FenceConfig._find_duplicates(safe_list_of_parent_studies)
         if len(duplicates) > 0:
             raise Exception(
                 f"{duplicates} are duplicate parent study ids found in parent_to_child_studies_mapping for "
                 f"multiple dbGaP configurations.")
 
-    @staticmethod
-    def enforce_is_array(dbgap_configs):
-        if isinstance(dbgap_configs, list):
-            corrected_dbgap_configs = dbgap_configs
-        elif isinstance(dbgap_configs, dict):
-            corrected_dbgap_configs = [dbgap_configs]
-        else:
-            raise Exception("The dbgap configuration is not a recognized data structure, aborting!")
-        return corrected_dbgap_configs
-
     @staticmethod
     def _validate_dbgap_config(dbgap_configs):
-        corrected_dbgap_configs = FenceConfig.enforce_is_array(dbgap_configs)
+        error_message = "The dbgap configuration is not a recognized data structure, aborting!"
+        corrected_dbgap_configs = FenceConfig._coerce_to_array(dbgap_configs, error_message)
         FenceConfig._validate_parent_child_studies(corrected_dbgap_configs)
 
 

fence/scripting/fence_create.py

@@ -57,10 +57,9 @@
 from fence.config import config
 from fence.sync.sync_users import UserSyncer
 from fence.utils import (
-    create_client,
     get_valid_expiration,
     generate_client_credentials,
-    get_SQLAlchemyDriver,
+    get_SQLAlchemyDriver, logger,
 )
 from sqlalchemy.orm.attributes import flag_modified
 from gen3authz.client.arborist.client import ArboristClient
@@ -1824,3 +1823,69 @@ def access_token_polling_job(
     with driver.session as db_session:
         loop = asyncio.get_event_loop()
         loop.run_until_complete(job.update_tokens(db_session))
+
+
+def create_client(
+    DB,
+    username=None,
+    urls=[],
+    name="",
+    description="",
+    auto_approve=False,
+    is_admin=False,
+    grant_types=None,
+    confidential=True,
+    arborist=None,
+    policies=None,
+    allowed_scopes=None,
+    expires_in=None,
+):
+    client_id, client_secret, hashed_secret = generate_client_credentials(confidential)
+    if arborist is not None:
+        arborist.create_client(client_id, policies)
+    driver = get_SQLAlchemyDriver(DB)
+    auth_method = "client_secret_basic" if confidential else "none"
+
+    allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"]
+    if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])):
+        raise ValueError(
+            "Each allowed scope must be one of: {}".format(
+                config["CLIENT_ALLOWED_SCOPES"]
+            )
+        )
+
+    if "openid" not in allowed_scopes:
+        allowed_scopes.append("openid")
+        logger.warning('Adding required "openid" scope to list of allowed scopes.')
+
+    with driver.session as s:
+        user = None
+        if username:
+            user = query_for_user(session=s, username=username)
+            if not user:
+                user = User(username=username, is_admin=is_admin)
+                s.add(user)
+
+        if s.query(Client).filter(Client.name == name).first():
+            if arborist is not None:
+                arborist.delete_client(client_id)
+            raise Exception("client {} already exists".format(name))
+
+        client = Client(
+            client_id=client_id,
+            client_secret=hashed_secret,
+            user=user,
+            redirect_uris=urls,
+            allowed_scopes=" ".join(allowed_scopes),
+            description=description,
+            name=name,
+            auto_approve=auto_approve,
+            grant_types=grant_types,
+            is_confidential=confidential,
+            token_endpoint_auth_method=auth_method,
+            expires_in=expires_in,
+        )
+        s.add(client)
+        s.commit()
+
+    return client_id, client_secret

fence/scripting/google_monitor.py

@@ -7,6 +7,7 @@
 """
 import traceback
 
+import requests
 from gen3cirrus.google_cloud.iam import GooglePolicyMember
 from gen3cirrus import GoogleCloudManager
 from gen3cirrus.google_cloud.errors import GoogleAPIError
@@ -37,7 +38,7 @@
 from fence import utils
 from fence.config import config
 from fence.models import User
-from fence.errors import Unauthorized
+from fence.errors import Unauthorized, NotFound
 
 logger = get_logger(__name__)
 
@@ -445,6 +446,52 @@ def _get_user_email_list_from_google_project_with_owner_role(project_id):
         )
 
 
+def send_email(from_email, to_emails, subject, text, smtp_domain):
+    """
+    Send email to group of emails using mail gun api.
+
+    https://app.mailgun.com/
+
+    Args:
+        from_email(str): from email
+        to_emails(list): list of emails to receive the messages
+        text(str): the text message
+        smtp_domain(dict): smtp domain server
+
+            {
+                "smtp_hostname": "smtp.mailgun.org",
+                "default_login": "[email protected]",
+                "api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net",
+                "smtp_password": "password", # pragma: allowlist secret
+                "api_key": "api key" # pragma: allowlist secret
+            }
+
+    Returns:
+        Http response
+
+    Exceptions:
+        KeyError
+
+    """
+    if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get(
+        smtp_domain
+    ).get("smtp_password"):
+        raise NotFound(
+            "SMTP Domain '{}' does not exist in configuration for GUN_MAIL or "
+            "smtp_password was not provided. "
+            "Cannot send email.".format(smtp_domain)
+        )
+
+    api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "")
+    email_url = config["GUN_MAIL"][smtp_domain].get("api_url", "") + "/messages"
+
+    return requests.post(
+        email_url,
+        auth=("api", api_key),
+        data={"from": from_email, "to": to_emails, "subject": subject, "text": text},
+    )
+
+
 def _send_emails_informing_service_account_removal(
     to_emails, invalid_service_account_reasons, invalid_project_reasons, project_id
 ):