From c9a667a83e3915d7b2e5b20f5d9bf01ee4e98f3d Mon Sep 17 00:00:00 2001 From: chengshifan <545591082@qq.com> Date: Wed, 21 Feb 2024 10:49:44 +0800 Subject: [PATCH 1/7] add retry when invoke API from vault --- src/api/handlers/job_api.py | 66 ++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index 9ed6cce3..496c0967 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -1,6 +1,7 @@ #pylint: disable=too-many-lines,too-few-public-methods,too-many-locals,too-many-statements,too-many-branches import os import json +import time import uuid import copy import urllib.request, urllib.parse, urllib.error @@ -43,6 +44,49 @@ def delete_file(path): logger.warning("Failed to delete file: %s", error) +def get_token_by_app_role(app_role_url, role_id, secret_id, retry_times=3): + app_role = {'role_id': role_id, 'secret_id': secret_id} + json_data = json.dumps(app_role) + attempt = 0 + while retry_times > 0: + res = requests.post(url=app_role_url, data=json_data, verify=False) + attempt += 1 + if res.status_code == 200: + json_res = json.loads(res.content) + token = json_res['auth']['client_token'] + return token + else: + if retry_times > 1: + logger.warning("Vault Token returned code is not 200, will retry after %s seconds..." % 10 * pow(2, attempt)) + time.sleep(10 * pow(2, attempt)) + else: + logger.error("Getting token from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'" % attempt, app_role_url, res.status_code, res.text) + retry_times -= 1 + abort(400, "Getting token from vault error") + + +def get_value_from_vault(url, token, secret_key, verify, retry_times=3): + attempt = 0 + while retry_times > 0: + response = requests.get(url=url, headers={'X-Vault-Token': token}, verify=verify) + attempt += 1 + if response.status_code == 200: + json_res = json.loads(response.content) + if json_res['data'].get('data') and isinstance(json_res['data'].get('data'), dict): + value = json_res['data'].get('data').get(secret_key) + else: + value = json_res['data'].get(secret_key) + return value + else: + if retry_times > 1: + logger.warning("Vault value returned code is not 200, will retry after %s seconds..." % 10 * pow(2, attempt)) + time.sleep(10 * pow(2, attempt)) + else: + logger.error("Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'" % retry_times, url, response.status_code, response.text) + retry_times -= 1 + abort(400, "Getting data from vault error") + + @api.route("/api/job/job", doc=False) class Job(Resource): @@ -295,34 +339,18 @@ def get_secret(name): if validate_res == 'token': logger.info('validate way is token') elif validate_res == 'appRole': - app_role = {'role_id': role_id, 'secret_id': secret_id} - json_data = json.dumps(app_role) app_role_url = result[0] + '/v1/' + namespace + '/auth/approle/login' if namespace else result[0] + '/v1/auth/approle/login' - res = requests.post(url=app_role_url, data=json_data, verify=False) - if res.status_code == 200: - json_res = json.loads(res.content) - token = json_res['auth']['client_token'] - else: - abort(400, "Getting value from vault error: url is '%s', validate way is appRole; API response: '%s'" % (app_role_url, res.text)) + token = get_token_by_app_role(app_role_url, role_id, secret_id) else: abort(400, "Validate way is '%s' ! result is '%s' " % (validate_res, result)) if not ca: - res = requests.get(url=url, headers={'X-Vault-Token': token}, verify=False) + return get_value_from_vault(url, token, secret_key, False) else: with tempfile.NamedTemporaryFile(delete=False) as f: f.write(ca) f.flush() # ensure all data written - res = requests.get(url=url, headers={'X-Vault-Token': token}, verify=f.name) - if res.status_code == 200: - json_res = json.loads(res.content) - if json_res['data'].get('data') and isinstance(json_res['data'].get('data'), dict): - value = json_res['data'].get('data').get(secret_key) - else: - value = json_res['data'].get(secret_key) - return value - else: - abort(400, "Getting value from vault error: url is '%s', token is '%s' " % (url, result)) + return get_value_from_vault(url, token, secret_key, f.name) else: if is_fork: abort(400, 'Access to secret %s is not allowed from a fork' % name) From 470b37f21d04af1f523dad06e732657322adc2b7 Mon Sep 17 00:00:00 2001 From: chengshifan <545591082@qq.com> Date: Wed, 21 Feb 2024 11:21:18 +0800 Subject: [PATCH 2/7] adjust retry time --- src/api/handlers/job_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index 496c0967..ec0c7a3e 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -58,7 +58,7 @@ def get_token_by_app_role(app_role_url, role_id, secret_id, retry_times=3): else: if retry_times > 1: logger.warning("Vault Token returned code is not 200, will retry after %s seconds..." % 10 * pow(2, attempt)) - time.sleep(10 * pow(2, attempt)) + time.sleep(1 * pow(2, attempt)) else: logger.error("Getting token from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'" % attempt, app_role_url, res.status_code, res.text) retry_times -= 1 @@ -80,9 +80,9 @@ def get_value_from_vault(url, token, secret_key, verify, retry_times=3): else: if retry_times > 1: logger.warning("Vault value returned code is not 200, will retry after %s seconds..." % 10 * pow(2, attempt)) - time.sleep(10 * pow(2, attempt)) + time.sleep(1 * pow(2, attempt)) else: - logger.error("Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'" % retry_times, url, response.status_code, response.text) + logger.error("Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'" % attempt, url, response.status_code, response.text) retry_times -= 1 abort(400, "Getting data from vault error") From 4f9593f6d3b20f80129cfd707062a59c765f9cf7 Mon Sep 17 00:00:00 2001 From: chengshifan <545591082@qq.com> Date: Wed, 21 Feb 2024 14:06:20 +0800 Subject: [PATCH 3/7] bug fix --- src/api/handlers/job_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index ec0c7a3e..4d1cb135 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -57,10 +57,10 @@ def get_token_by_app_role(app_role_url, role_id, secret_id, retry_times=3): return token else: if retry_times > 1: - logger.warning("Vault Token returned code is not 200, will retry after %s seconds..." % 10 * pow(2, attempt)) + logger.warning("Vault Token returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) time.sleep(1 * pow(2, attempt)) else: - logger.error("Getting token from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'" % attempt, app_role_url, res.status_code, res.text) + logger.error("Getting token from Vault error even though retried '%s' times, url is '%s', API response is '%s':'%s'", attempt, app_role_url, res.status_code, res.text) retry_times -= 1 abort(400, "Getting token from vault error") @@ -79,10 +79,10 @@ def get_value_from_vault(url, token, secret_key, verify, retry_times=3): return value else: if retry_times > 1: - logger.warning("Vault value returned code is not 200, will retry after %s seconds..." % 10 * pow(2, attempt)) + logger.warning("Vault value returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) time.sleep(1 * pow(2, attempt)) else: - logger.error("Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'" % attempt, url, response.status_code, response.text) + logger.error("Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'", attempt, url, response.status_code, response.text) retry_times -= 1 abort(400, "Getting data from vault error") From b0f589cd3ab7877dc93e76602bea9e24300f40ae Mon Sep 17 00:00:00 2001 From: chengshifan <545591082@qq.com> Date: Wed, 21 Feb 2024 14:52:37 +0800 Subject: [PATCH 4/7] enhance it --- src/api/handlers/job_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index 4d1cb135..51eafaef 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -58,11 +58,11 @@ def get_token_by_app_role(app_role_url, role_id, secret_id, retry_times=3): else: if retry_times > 1: logger.warning("Vault Token returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) - time.sleep(1 * pow(2, attempt)) + time.sleep(10 * pow(2, attempt)) else: logger.error("Getting token from Vault error even though retried '%s' times, url is '%s', API response is '%s':'%s'", attempt, app_role_url, res.status_code, res.text) retry_times -= 1 - abort(400, "Getting token from vault error") + abort(400, "Getting token from Vault error even though retried '%s' times, url is '%s', API response is '%s':'%s'", attempt, app_role_url, res.status_code, res.text) def get_value_from_vault(url, token, secret_key, verify, retry_times=3): @@ -80,11 +80,11 @@ def get_value_from_vault(url, token, secret_key, verify, retry_times=3): else: if retry_times > 1: logger.warning("Vault value returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) - time.sleep(1 * pow(2, attempt)) + time.sleep(10 * pow(2, attempt)) else: logger.error("Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'", attempt, url, response.status_code, response.text) retry_times -= 1 - abort(400, "Getting data from vault error") + abort(400, "Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'", attempt, url, response.status_code, response.text) @api.route("/api/job/job", doc=False) From 2b3b253645442d373bf06db0bb4a4172170588ea Mon Sep 17 00:00:00 2001 From: chengshifan <545591082@qq.com> Date: Wed, 21 Feb 2024 16:23:36 +0800 Subject: [PATCH 5/7] bug fix --- src/api/handlers/job_api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index 51eafaef..22e75d0c 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -48,6 +48,7 @@ def get_token_by_app_role(app_role_url, role_id, secret_id, retry_times=3): app_role = {'role_id': role_id, 'secret_id': secret_id} json_data = json.dumps(app_role) attempt = 0 + err_msg = 'Getting token from Vault error' while retry_times > 0: res = requests.post(url=app_role_url, data=json_data, verify=False) attempt += 1 @@ -60,13 +61,15 @@ def get_token_by_app_role(app_role_url, role_id, secret_id, retry_times=3): logger.warning("Vault Token returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) time.sleep(10 * pow(2, attempt)) else: - logger.error("Getting token from Vault error even though retried '%s' times, url is '%s', API response is '%s':'%s'", attempt, app_role_url, res.status_code, res.text) + err_msg = "Getting token from Vault error even though retried {} times, url is {}, API response is {}:{}".format(attempt, app_role_url, res.status_code, res.text) + logger.error(err_msg) retry_times -= 1 - abort(400, "Getting token from Vault error even though retried '%s' times, url is '%s', API response is '%s':'%s'", attempt, app_role_url, res.status_code, res.text) + abort(400, err_msg) def get_value_from_vault(url, token, secret_key, verify, retry_times=3): attempt = 0 + err_msg = 'Getting value from Vault error' while retry_times > 0: response = requests.get(url=url, headers={'X-Vault-Token': token}, verify=verify) attempt += 1 @@ -82,9 +85,10 @@ def get_value_from_vault(url, token, secret_key, verify, retry_times=3): logger.warning("Vault value returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) time.sleep(10 * pow(2, attempt)) else: - logger.error("Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'", attempt, url, response.status_code, response.text) + err_msg = "Getting value from Vault error even though retried {} times, url is {}, API response is {}:{}".format(attempt, url, response.status_code, response.text) + logger.error(err_msg) retry_times -= 1 - abort(400, "Getting value from Vault error even though retried %s times, url is '%s', API response is '%s':'%s'", attempt, url, response.status_code, response.text) + abort(400, err_msg) @api.route("/api/job/job", doc=False) From b5aa15bc5f0cba01c1204ddd2339548e8f323379 Mon Sep 17 00:00:00 2001 From: chengshifan <545591082@qq.com> Date: Wed, 21 Feb 2024 17:12:22 +0800 Subject: [PATCH 6/7] optimize code --- src/api/handlers/job_api.py | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index 22e75d0c..e472e869 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -44,35 +44,22 @@ def delete_file(path): logger.warning("Failed to delete file: %s", error) -def get_token_by_app_role(app_role_url, role_id, secret_id, retry_times=3): +def get_token_by_app_role(app_role_url, role_id, secret_id): app_role = {'role_id': role_id, 'secret_id': secret_id} json_data = json.dumps(app_role) - attempt = 0 - err_msg = 'Getting token from Vault error' - while retry_times > 0: + for i in range(0, 10): res = requests.post(url=app_role_url, data=json_data, verify=False) - attempt += 1 if res.status_code == 200: json_res = json.loads(res.content) token = json_res['auth']['client_token'] return token - else: - if retry_times > 1: - logger.warning("Vault Token returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) - time.sleep(10 * pow(2, attempt)) - else: - err_msg = "Getting token from Vault error even though retried {} times, url is {}, API response is {}:{}".format(attempt, app_role_url, res.status_code, res.text) - logger.error(err_msg) - retry_times -= 1 + err_msg = "Getting token from Vault error even tried 10 times, url is {}, API response is {}:{}".format(app_role_url, res.status_code, res.text) abort(400, err_msg) -def get_value_from_vault(url, token, secret_key, verify, retry_times=3): - attempt = 0 - err_msg = 'Getting value from Vault error' - while retry_times > 0: +def get_value_from_vault(url, token, secret_key, verify): + for i in range(0, 10): response = requests.get(url=url, headers={'X-Vault-Token': token}, verify=verify) - attempt += 1 if response.status_code == 200: json_res = json.loads(response.content) if json_res['data'].get('data') and isinstance(json_res['data'].get('data'), dict): @@ -80,14 +67,7 @@ def get_value_from_vault(url, token, secret_key, verify, retry_times=3): else: value = json_res['data'].get(secret_key) return value - else: - if retry_times > 1: - logger.warning("Vault value returned code is not 200, will retry after %s seconds...", 10 * pow(2, attempt)) - time.sleep(10 * pow(2, attempt)) - else: - err_msg = "Getting value from Vault error even though retried {} times, url is {}, API response is {}:{}".format(attempt, url, response.status_code, response.text) - logger.error(err_msg) - retry_times -= 1 + err_msg = "Getting value from Vault error even tried 10 times, url is {}, API response is {}:{}".format(url, response.status_code, response.text) abort(400, err_msg) From 3ccaf7a3151dd3f50c6a59c3a94f262994517c83 Mon Sep 17 00:00:00 2001 From: chengshifan <545591082@qq.com> Date: Wed, 21 Feb 2024 17:41:40 +0800 Subject: [PATCH 7/7] add sleep --- src/api/handlers/job_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index e472e869..39143c14 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -53,6 +53,7 @@ def get_token_by_app_role(app_role_url, role_id, secret_id): json_res = json.loads(res.content) token = json_res['auth']['client_token'] return token + time.sleep(5) err_msg = "Getting token from Vault error even tried 10 times, url is {}, API response is {}:{}".format(app_role_url, res.status_code, res.text) abort(400, err_msg) @@ -67,6 +68,7 @@ def get_value_from_vault(url, token, secret_key, verify): else: value = json_res['data'].get(secret_key) return value + time.sleep(5) err_msg = "Getting value from Vault error even tried 10 times, url is {}, API response is {}:{}".format(url, response.status_code, response.text) abort(400, err_msg)