Skip to content

Commit

Permalink
Merge pull request #740 from isb-cgc/isb-cgc-prod-sp
Browse files Browse the repository at this point in the history 
Sprint 29
  • Loading branch information
@s-paquette
s-paquette committed Sep 20, 2018
2 parents f57263f + 27c00e1 commit 96c6908
Show file tree
Hide file tree
Showing 10 changed files with 980 additions and 58 deletions.
278 changes: 272 additions & 6 deletions accounts/dcf_support.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@

DCF_TOKEN_URL = settings.DCF_TOKEN_URL
DCF_GOOGLE_URL = settings.DCF_GOOGLE_URL

DCF_GOOGLE_SA_REGISTER_URL = settings.DCF_GOOGLE_SA_REGISTER_URL
DCF_GOOGLE_SA_VERIFY_URL = settings.DCF_GOOGLE_SA_VERIFY_URL
DCF_GOOGLE_SA_MONITOR_URL = settings.DCF_GOOGLE_SA_MONITOR_URL
DCF_GOOGLE_SA_URL = settings.DCF_GOOGLE_SA_URL

class DCFCommFailure(Exception):
"""Thrown if we have problems communicating with DCF """
Expand Down Expand Up @@ -104,6 +107,260 @@ def drop_dcf_token(user_id):
return None


def unregister_sa_via_dcf(user_id, sa_id):
"""
Delete the given service account
"""
try:
full_url = '{0}{1}'.format(DCF_GOOGLE_SA_URL, sa_id)
resp = _dcf_call(full_url, user_id, mode='delete')
except (TokenFailure, InternalTokenError, RefreshTokenExpired, DCFCommFailure) as e:
logger.error("[ERROR] Attempt to contact DCF for SA information (user {})".format(user_id))
raise e
except Exception as e:
logger.error("[ERROR] Attempt to contact DCF for SA information failed (user {})".format(user_id))
raise e

success = False
messages = None
if resp.status_code == 200:
success = True
messages = ["Service account {} was dropped".format(sa_id)]
elif resp.status_code == 400:
messages = ["Service account {} was not found".format(sa_id)]
elif resp.status_code == 403:
messages = ["User cannot delete service account {}".format(sa_id)]
else:
messages = ["Unexpected response '{}' from Data Commons while dropping service account: {}".format(resp.status_code, sa_id)]

return success, messages


def service_account_info_from_dcf_for_project(user_id, proj):
"""
Get all service accounts tied to a project
"""
retval = []

try:
full_url = '{0}?google_project_ids={1}'.format(DCF_GOOGLE_SA_URL, proj)
logger.info("[INFO] Calling DCF URL {}".format(full_url))
resp = _dcf_call(full_url, user_id, mode='get')
except (TokenFailure, InternalTokenError, RefreshTokenExpired, DCFCommFailure) as e:
logger.error("[ERROR] Attempt to contact DCF for SA information (user {})".format(user_id))
raise e
except Exception as e:
logger.error("[ERROR] Attempt to contact DCF for SA information failed (user {})".format(user_id))
raise e

messages = None
if resp.status_code == 200:
response_dict = json_loads(resp.text)
sa_list = response_dict['service_accounts']
for sa in sa_list:
ret_entry = {
'gcp_id': sa['google_project_id'],
'sa_dataset_ids': sa['project_access'],
'sa_name': sa['service_account_email'],
'sa_exp': sa['project_access_exp']
}
retval.append(ret_entry)
elif resp.status_code == 403:
messages = ["User is not a member of Google project {}".format(proj)]
elif resp.status_code == 401: # Have seen this when the google sa scope was not requested in key
messages = ["User does not have permissions for this operation on Google project {}".format(proj)]
elif resp.status_code == 400: # If they don't like the request, say it was empty:
logger.info("[INFO] DCF response of 400 for URL {}".format(full_url))
else:
messages = ["Unexpected response from Data Commons: {}".format(resp.status_code)]

return retval, messages


def service_account_info_from_dcf(user_id, proj_list):
"""
Get all service accounts tied to the list of projects
"""
try:
proj_string = ','.join(proj_list)
full_url = '{0}?google_project_ids={1}'.format(DCF_GOOGLE_SA_URL, proj_string)
resp = _dcf_call(full_url, user_id, mode='get')
except (TokenFailure, InternalTokenError, RefreshTokenExpired, DCFCommFailure) as e:
logger.error("[ERROR] Attempt to contact DCF for SA information (user {})".format(user_id))
raise e
except Exception as e:
logger.error("[ERROR] Attempt to contact DCF for SA information failed (user {})".format(user_id))
raise e

retval = {}
messages = None
response_dict = json_loads(resp.text)
if resp.status_code == 200:
sa_list = response_dict['service_accounts']
for sa in sa_list:
ret_entry = {
'gcp_id': sa['google_project_id'],
'sa_dataset_ids': sa['project_access'],
'sa_name': sa['service_account_email'],
'sa_exp': sa['project_access_exp']
}
retval[sa['service_account_email']] = ret_entry
elif resp.status_code == 403:
messages = ["User is not a member on one or more of these Google projects: {}".format(proj_string)]
else:
messages = ["Unexpected response from Data Commons: {}".format(resp.status_code)]

return retval, messages


def verify_sa_at_dcf(user_id, gcp_id, service_account_id, datasets):
"""
:raise TokenFailure:
:raise InternalTokenError:
:raise DCFCommFailure:
:raise RefreshTokenExpired:
"""

sa_data = {
"service_account_email": service_account_id,
"google_project_id": gcp_id,
"project_access": datasets
}

#
# Call DCF to see if there would be problems with the service account registration.
#

try:
resp = _dcf_call(DCF_GOOGLE_SA_VERIFY_URL, user_id, mode='post', post_body=sa_data)
except (TokenFailure, InternalTokenError, RefreshTokenExpired, DCFCommFailure) as e:
logger.error("[ERROR] Attempt to contact DCF for SA verification failed (user {})".format(user_id))
raise e
except Exception as e:
logger.error("[ERROR] Attempt to contact DCF for SA verification failed (user {})".format(user_id))
raise e

messages = []

if resp:
logger.info("[INFO] DCF SA verification response code was {} with body: {} ".format(resp.status_code, resp.text))
response_dict = json_loads(resp.text)
if resp.status_code == 200:
messages = []
success = response_dict['success']
if not success:
logger.error("[ERROR] Inconsistent success response from DCF! Code: {} Text: {}".format(resp.status_code, success))
else:
messages.append("Service account {}: was verified".format(service_account_id))
elif resp.status_code == 400:
messages = []
error_info = response_dict['errors']
sa_error_info = error_info['service_account_email']
if sa_error_info['status'] == 200:
messages.append("Service account {}: no issues".format(service_account_id))
else:
messages.append("Service account {} error ({}): {}".format(service_account_id,
sa_error_info['error'],
sa_error_info['error_description']))
gcp_error_info = error_info['google_project_id']
if gcp_error_info['status'] == 200:
messages.append("Google cloud project {}: no issues".format(gcp_id))
else:
messages.append("Google cloud project {} error ({}): {}".format(gcp_id,
gcp_error_info['error'],
gcp_error_info['error_description']))
project_access_error_info = error_info['project_access']
messages.append("Requested projects:")
for project_name in project_access_error_info:
project = project_access_error_info[project_name]
if project['status'] == 200:
messages.append("Dataset {}: no issues".format(project_name))
else:
messages.append("Dataset {} error ({}): {}".format(project_name,
project['error'],
project['error_description']))
else:
logger.error("[ERROR] Unexpected response from DCF: {}".format(resp.status_code))

return messages


def register_sa_at_dcf(user_id, gcp_id, service_account_id, datasets):
"""
:raise TokenFailure:
:raise InternalTokenError:
:raise DCFCommFailure:
:raise RefreshTokenExpired:
"""

sa_data = {
"service_account_email": service_account_id,
"google_project_id": gcp_id,
"project_access": datasets
}

#
# Call DCF to see if there would be problems with the service account registration.
#

try:
logger.info("[INFO] Calling DCF at {}".format(json_dumps(sa_data)))
resp = _dcf_call(DCF_GOOGLE_SA_REGISTER_URL, user_id, mode='post', post_body=sa_data)
logger.info("[INFO] Just called DCF at {}".format(DCF_GOOGLE_SA_REGISTER_URL))
except (TokenFailure, InternalTokenError, RefreshTokenExpired, DCFCommFailure) as e:
logger.error("[ERROR] Attempt to contact DCF for SA registration failed (user {})".format(user_id))
raise e
except Exception as e:
logger.error("[ERROR] Attempt to contact DCF for SA registration failed (user {})".format(user_id))
raise e

messages = []

if resp:
logger.info("[INFO] DCF SA registration response code was {} with body: {} ".format(resp.status_code, resp.text))
response_dict = json_loads(resp.text)
if resp.status_code == 200:
messages = []
success = response_dict['success']
if not success:
logger.error("[ERROR] Inconsistent success response from DCF! Code: {} Text: {}".format(resp.status_code, success))
else:
messages.append("Service account {}: was verified".format(service_account_id))
elif resp.status_code == 400:
messages = []
error_info = response_dict['errors']
sa_error_info = error_info['service_account_email']
if sa_error_info['status'] == 200:
messages.append("Service account {}: no issues".format(service_account_id))
else:
messages.append("Service account {} error ({}): {}".format(service_account_id,
sa_error_info['error'],
sa_error_info['error_description']))
gcp_error_info = error_info['google_project_id']
if gcp_error_info['status'] == 200:
messages.append("Google cloud project {}: no issues".format(gcp_id))
else:
messages.append("Google cloud project {} error ({}): {}".format(gcp_id,
gcp_error_info['error'],
gcp_error_info['error_description']))
project_access_error_info = error_info['project_access']
messages.append("Requested projects:")
for project_name in project_access_error_info:
project = project_access_error_info[project_name]
if project['status'] == 200:
messages.append("Dataset {}: no issues".format(project_name))
else:
messages.append("Dataset {} error ({}): {}".format(project_name,
project['error'],
project['error_description']))
else:
logger.error("[ERROR] Unexpected response from DCF: {}".format(resp.status_code))
else:
logger.error("[ERROR] No response from DCF for registration")

return messages


def get_auth_elapsed_time(user_id):
"""
There is benefit in knowing when the user did their NIH login at DCF, allowing us to e.g. estimate
Expand Down Expand Up @@ -410,7 +667,7 @@ def refresh_at_dcf(user_id):
#

try:
resp = dcf_call(DCF_GOOGLE_URL, user_id, mode='patch')
resp = _dcf_call(DCF_GOOGLE_URL, user_id, mode='patch')
except (TokenFailure, InternalTokenError, RefreshTokenExpired, DCFCommFailure) as e:
throw_later = e
except Exception as e:
Expand Down Expand Up @@ -565,7 +822,7 @@ def _decode_token(token):
return decode_token_chunk(token, 1)


def dcf_call(full_url, user_id, mode='get', post_body=None, force_token=False):
def _dcf_call(full_url, user_id, mode='get', post_body=None, force_token=False, params_dict=None):
"""
All the stuff around a DCF call that handles token management and refreshes.
Expand Down Expand Up @@ -612,7 +869,7 @@ def token_storage_for_user(my_token_dict):

try:
resp = dcf.request(mode, full_url, client_id=client_id,
client_secret=client_secret, data=post_body)
client_secret=client_secret, data=post_body, params=params_dict)
except (TokenFailure, RefreshTokenExpired) as e:
# bubbles up from token_storage_for_user call
logger.error("[ERROR] _dcf_call {} aborted: {}".format(full_url, str(e)))
Expand Down Expand Up @@ -728,7 +985,16 @@ def refresh_token_storage(token_dict, decoded_jwt, user_token, nih_username_from
# "sub": "The users's DCF ID"
# }

refresh_expire_time = pytz.utc.localize(datetime.datetime.utcfromtimestamp(refresh_token_dict['exp']))
dcf_expire_timestamp = refresh_token_dict['exp']

#
# For testing purposes ONLY, we want the refresh token to expire in two days, not in 30. So mess with the returned
# value:
#

#dcf_expire_timestamp -= (28 * 86400) # ONLY USE THIS HACK FOR TESTING

refresh_expire_time = pytz.utc.localize(datetime.datetime.utcfromtimestamp(dcf_expire_timestamp))

# This refers to the *access key* expiration (~20 minutes)
if token_dict.has_key('expires_at'):
Expand Down Expand Up @@ -787,7 +1053,7 @@ def unlink_at_dcf(user_id, do_refresh):
#

try:
resp = dcf_call(DCF_GOOGLE_URL, user_id, mode='delete') # can raise TokenFailure, DCFCommFailure
resp = _dcf_call(DCF_GOOGLE_URL, user_id, mode='delete') # can raise TokenFailure, DCFCommFailure
except (TokenFailure, InternalTokenError, RefreshTokenExpired, DCFCommFailure) as e:
throw_later = e # hold off so we can try a refresh first...
except Exception as e:
Expand Down
2 changes: 1 addition & 1 deletion accounts/dcf_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def oauth2_login(request):

# Found that 'user' scope had to be included to be able to do the user query on callback, and the data scope
# to do data queries. Starting to recognize a pattern here...
oauth = OAuth2Session(client_id, redirect_uri=full_callback, scope=['openid', 'user'])
oauth = OAuth2Session(client_id, redirect_uri=full_callback, scope=['openid', 'user', 'google_service_account'])
authorization_url, state = oauth.authorization_url(DCF_AUTH_URL)

# stash the state string in the session!
Expand Down
Loading

0 comments on commit 96c6908

Please sign in to comment.