From b08b3d5f886094f78cc4e0c27e5244846bb726f5 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 31 May 2017 18:35:40 +0300 Subject: [PATCH 01/10] Added sleep moded, fixed bridge.py tests accordingly --- openprocurement/bot/identification/client.py | 2 +- .../bot/identification/databridge/bridge.py | 82 ++- .../identification/databridge/constants.py | 1 + .../identification/databridge/edr_handler.py | 43 +- .../databridge/journal_msg_ids.py | 1 + .../bot/identification/tests/bridge.py | 590 +++++++++++++++++- .../bot/identification/tests/edr_handler.py | 1 + .../bot/identification/tests/utils.py | 29 + 8 files changed, 732 insertions(+), 17 deletions(-) diff --git a/openprocurement/bot/identification/client.py b/openprocurement/bot/identification/client.py index 3d5a806..b4f9c7d 100644 --- a/openprocurement/bot/identification/client.py +++ b/openprocurement/bot/identification/client.py @@ -21,7 +21,7 @@ def verify(self, param, code, headers): return response - def details(self, id, headers): + def details(self, id, headers={}): """ Send request to Proxy server to get details.""" url = '{url}/{id}'.format(url=self.details_url, id=id) response = self.session.get(url=url, auth=(self.user, self.password), timeout=self.timeout, headers=headers) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index 8a34727..1a28d08 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from gevent import monkey +from openprocurement.bot.identification.databridge.constants import test_x_edr_internal_id + monkey.patch_all() import logging @@ -11,7 +13,7 @@ from functools import partial from yaml import load from gevent.queue import Queue -from restkit import request, RequestError +from restkit import request, RequestError, ResourceError from requests import RequestException from openprocurement_client.client import TendersClientSync, TendersClient @@ -36,7 +38,7 @@ def __init__(self, config): self.config = config api_server = self.config_get('tenders_api_server') - api_version = self.config_get('tenders_api_version') + self.api_version = self.config_get('tenders_api_version') ro_api_server = self.config_get('public_tenders_api_server') or api_server buffers_size = self.config_get('buffers_size') or 500 self.delay = self.config_get('delay') or 15 @@ -46,8 +48,8 @@ def __init__(self, config): self.doc_service_port = self.config_get('doc_service_port') or 6555 # init clients - self.tenders_sync_client = TendersClientSync('', host_url=ro_api_server, api_version=api_version) - self.client = TendersClient(self.config_get('api_token'), host_url=api_server, api_version=api_version) + self.tenders_sync_client = TendersClientSync('', host_url=ro_api_server, api_version=self.api_version) + self.client = TendersClient(self.config_get('api_token'), host_url=api_server, api_version=self.api_version) self.proxyClient = ProxyClient(host=self.config_get('proxy_server'), user=self.config_get('proxy_user'), password=self.config_get('proxy_password'), @@ -107,6 +109,7 @@ def __init__(self, config): processing_items=self.processing_items, doc_service_client=self.doc_service_client, delay=self.delay) + self.is_sleeping = False def config_get(self, name): return self.config.get('main').get(name) @@ -116,7 +119,7 @@ def check_doc_service(self): Separated to allow for possible granular checks """ try: - request("{host}:{port}/".format(host=self.doc_service_host, port=self.doc_service_port)) + r = request("{host}:{port}/".format(host=self.doc_service_host, port=self.doc_service_port)) except RequestError as e: logger.info('DocService connection error, message {}'.format(e), extra=journal_context({"MESSAGE_ID": DATABRIDGE_DOC_SERVICE_CONN_ERROR}, {})) @@ -124,11 +127,26 @@ def check_doc_service(self): else: return True + def check_openprocurement_api(self): + """Makes request to the TendersClient, returns True if it's up, raises RequestError otherwise + Separated to allow for possible granular checks + """ + try: + logger.debug("Checking status of openprocurement api") + self.client.head('/api/{}/spore'.format(self.api_version)) + except RequestError as e: + logger.info('TendersServer connection error, message {}'.format(e), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_DOC_SERVICE_CONN_ERROR}, {})) + raise e + else: + return True + def check_proxy(self): """Makes request to the EDR proxy, returns True if it's up, raises RequestError otherwise Separated to allow for possible granular checks """ try: + logger.debug("Checking status of proxy") self.proxyClient.health() except RequestException as e: logger.info('Proxy server connection error, message {}'.format(e), @@ -137,6 +155,53 @@ def check_proxy(self): else: return True + def check_paid_requests(self): + """Makes request to the EDR proxy, returns True if we have paid requests, otherwise return False""" + response = self.proxyClient.details(id=test_x_edr_internal_id) + if response.status_code == 402: # status of not having paid requests + logger.info("Checking for paid requests failed") + raise RequestException("Not enough money") + return True + + def set_exit_status(self, new_status): + for job in self.jobs.values(): + job.exit = new_status + + def check_services_and_start(self): + logger.info("Checking status of all services to try restarting") + try: + self.check_proxy() and self.check_paid_requests() and self.check_openprocurement_api() and self.check_doc_service() + except Exception as e: + logger.warning("Service is still unavailable") + return "Services are still unavailable" + else: + logger.warning("All services have become available, starting all workers") + self.set_exit_status(False) + self.is_sleeping = False + return "All services have become available, starting all workers" + + def check_services_and_stop(self): + """ + Check all services and workers; if at least one service is unavailable or at least one of the workers is + sleeping (has self.exit=True), sleep all the workers and set self.is_sleeping to True + :return: True - Stopped, False = Not stopped + """ + logger.info("Checking status of all services to stop bot if necessary") + try: + self.check_proxy() and self.check_doc_service() and self.check_openprocurement_api() + except Exception as e: + logger.info("Service is down, stopping all bots") + self.is_sleeping = True + self.set_exit_status(True) + return True + else: + for job in self.jobs.values(): + if job.exit: + self.is_sleeping = True + self.set_exit_status(True) + return True + return False + def _start_jobs(self): self.jobs = {'scanner': self.scanner(), 'filter_tender': self.filter_tender(), @@ -150,7 +215,10 @@ def run(self): try: while True: gevent.sleep(self.delay) - if counter == 20: + self.check_services_and_stop() + if counter == 1: + if self.is_sleeping: + self.check_services_and_start() logger.info('Current state: Filtered tenders {}; Edrpou codes queue {}; Retry edrpou codes queue {}; ' 'Edr ids queue {}; Retry edr ids queue {}; Upload to doc service {}; Retry upload to doc service {}; ' 'Upload to tender {}; Retry upload to tender {}'.format( @@ -188,7 +256,7 @@ def main(): config = load(config_file_obj.read()) logging.config.dictConfig(config) bridge = EdrDataBridge(config) - bridge.check_proxy() and bridge.check_doc_service() + bridge.check_proxy() and bridge.check_doc_service() and bridge.check_paid_requests() and bridge.check_openprocurement_api() bridge.run() else: logger.info('Invalid configuration file. Exiting...') diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index 3d04f0c..4439569 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,3 +4,4 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" +test_x_edr_internal_id = 1050844 \ No newline at end of file diff --git a/openprocurement/bot/identification/databridge/edr_handler.py b/openprocurement/bot/identification/databridge/edr_handler.py index 14bee5d..44f39d4 100644 --- a/openprocurement/bot/identification/databridge/edr_handler.py +++ b/openprocurement/bot/identification/databridge/edr_handler.py @@ -13,12 +13,12 @@ from openprocurement.bot.identification.databridge.journal_msg_ids import ( DATABRIDGE_GET_TENDER_FROM_QUEUE, DATABRIDGE_START_EDR_HANDLER, DATABRIDGE_SUCCESS_CREATE_FILE, - DATABRIDGE_EMPTY_RESPONSE -) + DATABRIDGE_EMPTY_RESPONSE, DATABRIDGE_PROXY_SERVER_CONN_ERROR) from openprocurement.bot.identification.databridge.utils import ( Data, journal_context, validate_param, RetryException, check_add_suffix ) from openprocurement.bot.identification.databridge.constants import version +from requests import RequestException logger = logging.getLogger(__name__) @@ -31,6 +31,7 @@ class EdrHandler(Greenlet): def __init__(self, proxyClient, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, processing_items, delay=15): super(EdrHandler, self).__init__() self.exit = False + self.has_paid_requests = True self.start_time = datetime.now() # init clients @@ -170,7 +171,7 @@ def retry_get_edr_id(self): tender_data.item_id)) gevent.sleep(0) - @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) + @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1) def get_edr_id_request(self, param, code, document_id): """Execute request to EDR Api for retry queue objects.""" response = self.proxyClient.verify(param, code, headers={'X-Client-Request-ID': document_id}) @@ -184,6 +185,7 @@ def get_edr_details(self): """Get data from edr_ids_queue; make request to EDR Api for detailed info; Required fields is put to Data.file_content variable, Data object is put to upload_to_doc_service_queue.""" while not self.exit: + logger.info("get_edr_details iteration") try: tender_data = self.edr_ids_queue.peek() except LoopExit: @@ -195,11 +197,14 @@ def get_edr_details(self): self.until_too_many_requests_event.wait() meta_id = tender_data.file_content['meta']['id'] for edr_id in tender_data.edr_ids: + logger.info("edr_id: {}".format(edr_id)) # if more then 1 instance add amount and number of document to document_id document_id = check_add_suffix(tender_data.edr_ids, meta_id, tender_data.edr_ids.index(edr_id) + 1) tender_data.file_content['meta']['id'] = document_id response = self.proxyClient.details(id=edr_id, headers={'X-Client-Request-ID': document_id}) + logger.info("Status: {}".format(response.status_code)) if response.status_code == 200: + logger.info("response.json(): {}".format(response.json())) if not isinstance(response.json(), dict): file_content = tender_data.file_content file_content['meta']['sourceRequests'].append(response.headers['X-Request-ID']) @@ -220,6 +225,9 @@ def get_edr_details(self): tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) + elif response.status_code == 402: + logger.info("NOT ENOUGH MONEY") + self.exit = True else: file_content = tender_data.file_content @@ -280,7 +288,7 @@ def retry_get_edr_details(self): params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) gevent.sleep(0) - @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) + @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1) def get_edr_details_request(self, edr_id, document_id): """Execute request to EDR Api to get detailed info for retry queue objects.""" response = self.proxyClient.details(id=edr_id, headers={'X-Client-Request-ID': document_id}) @@ -304,6 +312,25 @@ def handle_status_response(self, response, tender_id): 'Description: {err}'.format(err=response.text), extra=journal_context(params={"TENDER_ID": tender_id})) + def check_proxy(self): + """Makes request to the EDR proxy, returns True if it's up, raises RequestError otherwise + Separated to allow for possible granular checks + """ + try: + self.proxyClient.health() + except RequestException as e: + logger.info('Proxy server connection error, message {}'.format(e), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_PROXY_SERVER_CONN_ERROR}, {})) + raise e + else: + return True + + def check_edr_api(self): + try: + self.proxyClient.details(999186) + except RequestException as e: + logger.info("") + def _run(self): logger.info('Start EDR Handler', extra=journal_context({"MESSAGE_ID": DATABRIDGE_START_EDR_HANDLER}, {})) self.immortal_jobs = {'get_edr_id': spawn(self.get_edr_id), @@ -313,7 +340,15 @@ def _run(self): try: while not self.exit: + # if not(self.check_proxy() and self.check_edr_api()): + # logger.warning("Waiting because one of the services is unavailable: Proxy: {}; EDR API: {}".format( + # self.check_proxy(), self.check_edr_api() + # )) + # for name, job in self.immortal_jobs.items(): + # if not job.dead: + # self.immortal_jobs[name].wait(self.delay) gevent.sleep(self.delay) + logger.info("iteration of run") for name, job in self.immortal_jobs.items(): if job.dead: logger.warning("EDR handler worker {} dead try restart".format(name), diff --git a/openprocurement/bot/identification/databridge/journal_msg_ids.py b/openprocurement/bot/identification/databridge/journal_msg_ids.py index 8930eb6..3e0e4e0 100644 --- a/openprocurement/bot/identification/databridge/journal_msg_ids.py +++ b/openprocurement/bot/identification/databridge/journal_msg_ids.py @@ -37,6 +37,7 @@ DATABRIDGE_RESTART_RETRY_GET_EDR_ID = 'edr_databridge_restart_retry_get_edr_id' DATABRIDGE_RESTART_RETRY_GET_EDR_DETAILS = 'edr_databridge_restart_retry_get_edr_details' DATABRIDGE_DOC_SERVICE_CONN_ERROR = 'edr_databridge_doc_service_conn_error' +DATABRIDGE_TENDERS_SERVER_CONN_ERROR = 'edr_databridge_tenders_server_conn_error' DATABRIDGE_PROXY_SERVER_CONN_ERROR = 'edr_databridge_proxy_server_conn_error' DATABRIDGE_422_UPLOAD_TO_TENDER = 'edr_databridge_422_upload_to_tender' DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING = 'edr_databridge_item_status_changed_while_processing' diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index b519c50..f07d106 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -2,6 +2,8 @@ import unittest import os +import logging +import time from requests import RequestException @@ -14,13 +16,17 @@ from openprocurement.bot.identification.databridge.bridge import EdrDataBridge from openprocurement_client.client import TendersClientSync, TendersClient from openprocurement.bot.identification.client import DocServiceClient, ProxyClient +from openprocurement.bot.identification.databridge.constants import test_x_edr_internal_id +from openprocurement.bot.identification.tests.utils import MockLoggingHandler + + config = { 'main': { 'tenders_api_server': 'http://127.0.0.1:20604', - 'tenders_api_version': '0', + 'tenders_api_version': '2.3', 'public_tenders_api_server': 'http://127.0.0.1:20605', 'doc_service_server': 'http://127.0.0.1', 'doc_service_port': 20606, @@ -35,7 +41,8 @@ 'full_stack_sync_delay': 15, 'empty_stack_sync_delay': 101, 'on_error_sleep_delay': 5, - 'api_token': "api_token" + 'api_token': "api_token", + # 'delay': 0.1 } } @@ -64,7 +71,7 @@ def setUpClass(cls): cls.proxy_server_bottle = Bottle() cls.doc_server_bottle = Bottle() cls.api_server = WSGIServer(('127.0.0.1', 20604), cls.api_server_bottle, log=None) - setup_routing(cls.api_server_bottle, response_spore) + setup_routing(cls.api_server_bottle, response_spore, method='HEAD') cls.public_api_server = WSGIServer(('127.0.0.1', 20605), cls.api_server_bottle, log=None) cls.doc_server = WSGIServer(('127.0.0.1', 20606), cls.doc_server_bottle, log=None) setup_routing(cls.doc_server_bottle, doc_response, path='/') @@ -77,6 +84,12 @@ def setUpClass(cls): cls.doc_server.start() cls.proxy_server.start() + # mock logging + logger = logging.getLogger("openprocurement.bot.identification.databridge.bridge") + cls.log_handler = MockLoggingHandler(level='DEBUG') + logger.addHandler(cls.log_handler) + cls.log_messages = cls.log_handler.messages + @classmethod def tearDownClass(cls): cls.api_server.close() @@ -84,13 +97,25 @@ def tearDownClass(cls): cls.doc_server.close() cls.proxy_server.close() + def setUp(self): + super(BaseServersTest, self).setUp() + self.log_handler.reset() + def tearDown(self): + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/health') del self.worker -def setup_routing(app, func, path='/api/0/spore', method='GET'): +def setup_routing(app, func, path='/api/2.3/spore', method='GET'): + app.routes = [] app.route(path, method, func) +def setup_routing_multiple(app, func, paths=None, method='GET'): + if paths is None: + paths = ['/api/2.3/spore'] + app.routes = [] + for path in paths: + app.route(path, method, func) def response_spore(): response.set_cookie("SERVER_ID", ("a7afc9b1fc79e640f2487ba48243ca071c07a823d27" @@ -102,10 +127,14 @@ def response_spore(): def doc_response(): return response - def proxy_response(): return response +def proxy_response_402(): + res = response + res.status = "402 Payment required" + return res + class TestBridgeWorker(BaseServersTest): @@ -199,6 +228,8 @@ def test_run(self, sleep): self.assertEqual(edr_handler.call_count, 1) self.assertEqual(upload_file.call_count, 1) + # region testing checks of separate services + def test_proxy_server_failure(self): self.proxy_server.stop() self.worker = EdrDataBridge(config) @@ -222,3 +253,552 @@ def test_doc_service_failure(self): def test_doc_service_success(self): self.worker = EdrDataBridge(config) self.assertEqual(self.worker.check_doc_service(), True) + + def test_api_server_failure(self): + self.worker = EdrDataBridge(config) + self.api_server.stop() + with self.assertRaises(RequestError) as err_context: + self.worker.check_openprocurement_api() + self.assertEqual(err_context.exception.message, "socket.error: [Errno 111] Connection refused") + self.api_server.start() + self.assertEqual(self.worker.check_openprocurement_api(), True) + + def test_api_server_success(self): + self.worker = EdrDataBridge(config) + self.assertEqual(self.worker.check_openprocurement_api(), True) + + def test_paid_requests_failure(self): + setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.worker = EdrDataBridge(config) + with self.assertRaises(RequestException) as err: + self.assertEqual(self.worker.check_paid_requests(), True) + self.assertEqual(err.exception.message, "Not enough money") + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.assertEqual(self.worker.check_paid_requests(), True) + + def test_paid_requests_success(self): + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.worker = EdrDataBridge(config) + self.assertEqual(self.worker.check_paid_requests(), True) + + # endregion + + # region check_and_stop + + def test_check_and_stop_did_not_stop(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker.check_proxy = MagicMock() + self.worker.check_proxy.return_value = True + self.worker.check_doc_service = MagicMock() + self.worker.check_doc_service.return_value = True + self.worker.check_openprocurement_api = MagicMock() + self.worker.check_openprocurement_api.return_value = True + self.worker._start_jobs() + self.assertEqual(self.worker.is_sleeping, False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(self.worker.check_proxy.call_count, 1) + self.assertEqual(self.worker.check_doc_service.call_count, 1) + self.assertEqual(self.worker.check_openprocurement_api.call_count, 1) + self.assertEqual(has_stopped, False) + self.assertEqual(self.worker.is_sleeping, False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + + def test_check_and_stop_stops_after_exit(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + temp = self.worker.set_exit_status + self.worker.set_exit_status = MagicMock(side_effect=temp) + self.assertEqual(self.worker.set_exit_status.call_count, 0) + self.worker._start_jobs() + self.assertEqual(self.worker.is_sleeping, False) + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(self.worker.set_exit_status.call_count, 0) + self.assertEqual(has_stopped, False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + scanner.return_value.exit = True + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(self.worker.set_exit_status.call_count, 1) + self.assertEqual(has_stopped, True) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.worker.set_exit_status.assert_called_with(True) + self.assertEqual(self.worker.set_exit_status.call_count, 1) + + def test_check_and_stop_not_stops_mock_jobs(self): + self.worker = EdrDataBridge(config) + temp = self.worker.set_exit_status + self.worker.set_exit_status = MagicMock(side_effect=temp) + self.assertEqual(self.worker.set_exit_status.call_count, 0) + self.worker.is_sleeping = False + mock_job = MagicMock() + self.worker.jobs = {'test': mock_job} + + mock_job.exit = False + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, False) + self.assertEqual(self.worker.is_sleeping, False) + self.assertEqual(self.worker.set_exit_status.call_count, 0) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + + def test_check_and_stop_stops_after_exit_mock_jobs(self): + self.worker = EdrDataBridge(config) + temp = self.worker.set_exit_status + self.worker.set_exit_status = MagicMock(side_effect=temp) + self.assertEqual(self.worker.set_exit_status.call_count, 0) + self.worker.is_sleeping = False + mock_job = MagicMock() + self.worker.jobs = {'test': mock_job} + + mock_job.exit = True + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, True) + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(self.worker.set_exit_status.call_count, 1) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.worker.set_exit_status.assert_called_with(True) + + def test_check_and_stop_stops_with_failed_proxy(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + self.assertEqual(self.worker.check_services_and_stop(), False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') + self.assertEqual(self.worker.check_services_and_stop(), True) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/health') + + def test_check_and_stop_stops_with_failed_doc_service(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + self.doc_server.stop() + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, True) + self.assertEqual(self.worker.is_sleeping, True) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.doc_server.start() + + def test_check_and_stop_stops_with_failed_api(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + self.api_server.stop() + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, True) + self.assertEqual(self.worker.is_sleeping, True) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.api_server.start() + + # endregion + + # region check_and_start + + def test_check_and_start_call(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + self.worker.set_exit_status(True) + setup_routing_multiple(self.proxy_server_bottle, proxy_response, + paths=['/api/1.0/health', '/api/1.0/details/{}'.format(test_x_edr_internal_id)]) + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.worker.check_services_and_start() + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + + def test_check_and_start_does_not_start_with_failed_proxy(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + # setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') + self.proxy_server.stop() + self.worker.set_exit_status(True) + self.worker.is_sleeping = True + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.proxy_server.start() + + def test_check_and_start_does_not_start_without_paid_requests(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.worker.set_exit_status(True) + self.worker.is_sleeping = True + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + + def test_check_and_start_does_not_start_with_failed_doc_service(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + self.doc_server.stop() + self.worker.set_exit_status(True) + self.worker.is_sleeping = True + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.doc_server.start() + + def test_check_and_start_does_not_start_with_failed_api(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + self.api_server.stop() + self.worker.set_exit_status(True) + self.worker.is_sleeping = True + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.api_server.start() + + def test_check_and_start_starts_with_all_up(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker.check_proxy = MagicMock() + self.worker.check_proxy.return_value = True + self.worker.check_paid_requests = MagicMock() + self.worker.check_paid_requests.return_value = True + self.worker.check_doc_service = MagicMock() + self.worker.check_doc_service.return_value = True + self.worker.check_openprocurement_api = MagicMock() + self.worker.check_openprocurement_api.return_value = True + self.worker._start_jobs() + self.worker.set_exit_status(True) + self.worker.is_sleeping = True + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + temp = self.worker.set_exit_status + self.worker.set_exit_status = MagicMock(side_effect=temp) + self.assertEqual(self.worker.check_services_and_start(), "All services have become available, starting all workers") + self.worker.set_exit_status.assert_called_with(False) + self.assertEqual(self.worker.set_exit_status.call_count, 1) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + + def test_check_and_start_does_not_start_with_failed_api_then_starts(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + self.api_server.stop() + self.worker.set_exit_status(True) + self.worker.is_sleeping = True + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") + self.api_server.start() + self.assertEqual(self.worker.check_services_and_start(), "All services have become available, starting all workers") + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + + def test_check_and_start_starts_after_stop(self): + """ + 1. Configure worker, set up mocks of all 4 workers with exit parameters + 2. Start all jobs + 3. Simulate that /health now returns an error + 4. Check that check_services_and_stop stops all workers and sets is_sleeping to True + 5. Simulate return of /health to normal + 6. Check that check_services_and_start starts all workers and sets is_sleeping to False + """ + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') + self.worker.check_services_and_stop() + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + setup_routing_multiple(self.proxy_server_bottle, proxy_response, + paths=['/api/1.0/health', '/api/1.0/details/{}'.format(test_x_edr_internal_id)]) + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.worker.check_services_and_start() + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + + def test_check_and_start_does_not_start_after_stop(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') + self.worker.check_services_and_stop() + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + self.worker.check_services_and_start() + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.assertEqual(self.worker.is_sleeping, True) + + # endregion + + def test_combine_check_stop_and_start_api(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.assertEqual(self.worker.is_sleeping, False) + self.api_server.stop() + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, True) + self.assertEqual(self.worker.is_sleeping, True) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.api_server.start() + has_started = self.worker.check_services_and_start() + self.assertEqual(has_started, "All services have become available, starting all workers") + has_stopped = self.worker.check_services_and_stop() + self.assertEqual(has_stopped, False) + + def test_set_exit(self): + self.worker = EdrDataBridge(config) + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + scanner.return_value.exit = False + filter_tender.return_value.exit = False + edr_handler.return_value.exit = False + upload_file.return_value.exit = False + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + self.worker._start_jobs() + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + self.worker.set_exit_status(True) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, True) + self.worker.set_exit_status(False) + for i in self.worker.jobs.values(): + self.assertEqual(i.exit, False) + + @patch('gevent.sleep') + def test_run_mock_exception(self, sleep): + self.worker = EdrDataBridge(config) + # create mocks + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + temp = self.worker.check_services_and_stop + self.worker.check_services_and_stop = MagicMock(side_effect=Exception) + with patch('__builtin__.True', AlmostAlwaysTrue(100)): + self.worker.run() + self.assertEqual(scanner.call_count, 1) + self.assertEqual(filter_tender.call_count, 1) + self.assertEqual(edr_handler.call_count, 1) + self.assertEqual(upload_file.call_count, 1) + + # # @patch('gevent.sleep') + # def test_run_mock_check_services_and_start(self): + # self.worker = EdrDataBridge(config) + # self.assertEqual(self.worker.delay, 0.1) + # # create mocks + # scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + # self.worker.scanner = scanner + # self.worker.filter_tender = filter_tender + # self.worker.edr_handler = edr_handler + # self.worker.upload_file = upload_file + # temp = self.worker.check_services_and_stop + # self.worker.check_services_and_start = MagicMock() + # self.worker.is_sleeping = True + # class Mock_Sleep() + # with + # self.worker.run() + # self.assertEqual(self.worker.check_services_and_start.call_count, 1) + # self.assertEqual(scanner.call_count, 1) + # self.assertEqual(filter_tender.call_count, 1) + # self.assertEqual(edr_handler.call_count, 1) + # self.assertEqual(upload_file.call_count, 1) + + + + # @patch('gevent.sleep') + # def test_run_mock_keyboard_interrupt(self, sleep): + # self.worker = EdrDataBridge(config) + # # create mocks + # scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + # self.worker.scanner = scanner + # self.worker.filter_tender = filter_tender + # self.worker.edr_handler = edr_handler + # self.worker.upload_file = upload_file + # temp = self.worker.check_services_and_stop + # self.worker.check_services_and_stop = MagicMock(side_effect=KeyboardInterrupt) + # with patch("openprocurement.bot.identification.databridge.bridge.gevent.killall"): + # self.worker.run() + # self.assertEqual(scanner.call_count, 1) + # self.assertEqual(filter_tender.call_count, 1) + # self.assertEqual(edr_handler.call_count, 1) + # self.assertEqual(upload_file.call_count, 1) + + diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index 3bbc96d..60a8896 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -53,6 +53,7 @@ def test_proxy_client(self, mrequest, gevent_sleep): local_edr_ids = get_random_edr_ids(2) edr_req_ids = [generate_request_id(), generate_request_id()] edr_details_req_ids = [generate_request_id(), generate_request_id()] + mrequest.get("{uri}".format(uri=proxy_client.health_url)) mrequest.get("{uri}".format(uri=proxy_client.verify_url), [{'json': {'data': [{'x_edrInternalId': local_edr_ids[0]}], "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'json': {'data': [{'x_edrInternalId': local_edr_ids[1]}], "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}}]) diff --git a/openprocurement/bot/identification/tests/utils.py b/openprocurement/bot/identification/tests/utils.py index 59d578f..c810076 100644 --- a/openprocurement/bot/identification/tests/utils.py +++ b/openprocurement/bot/identification/tests/utils.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import logging from gevent import sleep as gsleep from uuid import uuid4 from json import dumps @@ -34,3 +35,31 @@ def body_string(self): def next(self): pass + +class MockLoggingHandler(logging.Handler): + """Mock logging handler to check for expected logs. + + Messages are available from an instance's ``messages`` dict, in order, indexed by + a lowercase log level string (e.g., 'debug', 'info', etc.). + """ + + def __init__(self, *args, **kwargs): + self.messages = {'debug': [], 'info': [], 'warning': [], 'error': [], + 'critical': []} + super(MockLoggingHandler, self).__init__(*args, **kwargs) + + def emit(self, record): + "Store a message from ``record`` in the instance's ``messages`` dict." + self.acquire() + try: + self.messages[record.levelname.lower()].append(record.getMessage()) + finally: + self.release() + + def reset(self): + self.acquire() + try: + for message_list in self.messages.values(): + del message_list[:] + finally: + self.release() From 9c29f3bc4b7fda7609c5e6207b48a79ae2760b00 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Thu, 1 Jun 2017 10:59:43 +0300 Subject: [PATCH 02/10] Added tests for the edr_handler.py's behaviour in case of 402 --- .../identification/databridge/edr_handler.py | 47 ++---- .../bot/identification/tests/bridge.py | 82 +++++---- .../bot/identification/tests/edr_handler.py | 158 +++++++++++++++++- 3 files changed, 208 insertions(+), 79 deletions(-) diff --git a/openprocurement/bot/identification/databridge/edr_handler.py b/openprocurement/bot/identification/databridge/edr_handler.py index 44f39d4..12c8941 100644 --- a/openprocurement/bot/identification/databridge/edr_handler.py +++ b/openprocurement/bot/identification/databridge/edr_handler.py @@ -113,6 +113,7 @@ def retry_get_edr_id(self): """Get data from retry_edrpou_codes_queue; Put data into edr_ids_queue if request is successful, otherwise put data back to retry_edrpou_codes_queue.""" while not self.exit: + # logger.info("retry iteration ") try: tender_data = self.retry_edrpou_codes_queue.get() except LoopExit: @@ -202,9 +203,7 @@ def get_edr_details(self): document_id = check_add_suffix(tender_data.edr_ids, meta_id, tender_data.edr_ids.index(edr_id) + 1) tender_data.file_content['meta']['id'] = document_id response = self.proxyClient.details(id=edr_id, headers={'X-Client-Request-ID': document_id}) - logger.info("Status: {}".format(response.status_code)) if response.status_code == 200: - logger.info("response.json(): {}".format(response.json())) if not isinstance(response.json(), dict): file_content = tender_data.file_content file_content['meta']['sourceRequests'].append(response.headers['X-Request-ID']) @@ -263,12 +262,16 @@ def retry_get_edr_details(self): if response.headers.get('X-Request-ID'): tender_data.file_content['meta']['sourceRequests'].append(response.headers['X-Request-ID']) except RetryException as re: - self.retry_edr_ids_queue.put((Data(tender_data.tender_id, tender_data.item_id, tender_data.code, - tender_data.item_name, [edr_id], tender_data.file_content))) - logger.info('Put tender {} with {} id {} {} to retry_edr_ids_queue. Error response {}'.format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.args[1].json().get('errors')), - extra=journal_context(params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) - gevent.sleep(0) + if re.args[1].status_code == 402: + logger.info("NOT ENOUGH MONEY") + self.exit = True + else: + self.retry_edr_ids_queue.put((Data(tender_data.tender_id, tender_data.item_id, tender_data.code, + tender_data.item_name, [edr_id], tender_data.file_content))) + logger.info('Put tender {} with {} id {} {} to retry_edr_ids_queue. Error response {}'.format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.args[1].json().get('errors')), + extra=journal_context(params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) + gevent.sleep(0) else: if not isinstance(response.json(), dict): logger.info('Error data type {} {} {} {}. Message {}.'.format( @@ -312,25 +315,6 @@ def handle_status_response(self, response, tender_id): 'Description: {err}'.format(err=response.text), extra=journal_context(params={"TENDER_ID": tender_id})) - def check_proxy(self): - """Makes request to the EDR proxy, returns True if it's up, raises RequestError otherwise - Separated to allow for possible granular checks - """ - try: - self.proxyClient.health() - except RequestException as e: - logger.info('Proxy server connection error, message {}'.format(e), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_PROXY_SERVER_CONN_ERROR}, {})) - raise e - else: - return True - - def check_edr_api(self): - try: - self.proxyClient.details(999186) - except RequestException as e: - logger.info("") - def _run(self): logger.info('Start EDR Handler', extra=journal_context({"MESSAGE_ID": DATABRIDGE_START_EDR_HANDLER}, {})) self.immortal_jobs = {'get_edr_id': spawn(self.get_edr_id), @@ -340,15 +324,8 @@ def _run(self): try: while not self.exit: - # if not(self.check_proxy() and self.check_edr_api()): - # logger.warning("Waiting because one of the services is unavailable: Proxy: {}; EDR API: {}".format( - # self.check_proxy(), self.check_edr_api() - # )) - # for name, job in self.immortal_jobs.items(): - # if not job.dead: - # self.immortal_jobs[name].wait(self.delay) gevent.sleep(self.delay) - logger.info("iteration of run") + # logger.info("iteration of run") for name, job in self.immortal_jobs.items(): if job.dead: logger.warning("EDR handler worker {} dead try restart".format(name), diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index f07d106..b174279 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -42,7 +42,7 @@ 'empty_stack_sync_delay': 101, 'on_error_sleep_delay': 5, 'api_token': "api_token", - # 'delay': 0.1 + 'delay': 1 } } @@ -759,46 +759,44 @@ def test_run_mock_exception(self, sleep): self.assertEqual(edr_handler.call_count, 1) self.assertEqual(upload_file.call_count, 1) - # # @patch('gevent.sleep') - # def test_run_mock_check_services_and_start(self): - # self.worker = EdrDataBridge(config) - # self.assertEqual(self.worker.delay, 0.1) - # # create mocks - # scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - # self.worker.scanner = scanner - # self.worker.filter_tender = filter_tender - # self.worker.edr_handler = edr_handler - # self.worker.upload_file = upload_file - # temp = self.worker.check_services_and_stop - # self.worker.check_services_and_start = MagicMock() - # self.worker.is_sleeping = True - # class Mock_Sleep() - # with - # self.worker.run() - # self.assertEqual(self.worker.check_services_and_start.call_count, 1) - # self.assertEqual(scanner.call_count, 1) - # self.assertEqual(filter_tender.call_count, 1) - # self.assertEqual(edr_handler.call_count, 1) - # self.assertEqual(upload_file.call_count, 1) - - - - # @patch('gevent.sleep') - # def test_run_mock_keyboard_interrupt(self, sleep): - # self.worker = EdrDataBridge(config) - # # create mocks - # scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - # self.worker.scanner = scanner - # self.worker.filter_tender = filter_tender - # self.worker.edr_handler = edr_handler - # self.worker.upload_file = upload_file - # temp = self.worker.check_services_and_stop - # self.worker.check_services_and_stop = MagicMock(side_effect=KeyboardInterrupt) - # with patch("openprocurement.bot.identification.databridge.bridge.gevent.killall"): - # self.worker.run() - # self.assertEqual(scanner.call_count, 1) - # self.assertEqual(filter_tender.call_count, 1) - # self.assertEqual(edr_handler.call_count, 1) - # self.assertEqual(upload_file.call_count, 1) + @patch('gevent.sleep') + def test_run_mock_check_services_and_start(self, sleep): + self.worker = EdrDataBridge(config) + self.assertEqual(self.worker.delay, 1) + # create mocks + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + temp = self.worker.check_services_and_start + self.worker.check_services_and_start = MagicMock() + self.worker.check_services_and_stop = MagicMock() + self.worker.is_sleeping = True + time.sleep(20) + self.worker.run() + self.assertEqual(self.worker.check_services_and_start.call_count, 1) + self.assertEqual(scanner.call_count, 1) + self.assertEqual(filter_tender.call_count, 1) + self.assertEqual(edr_handler.call_count, 1) + self.assertEqual(upload_file.call_count, 1) + + @patch('gevent.sleep') + def test_run_mock_keyboard_interrupt(self, sleep): + self.worker = EdrDataBridge(config) + # create mocks + scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] + self.worker.scanner = scanner + self.worker.filter_tender = filter_tender + self.worker.edr_handler = edr_handler + self.worker.upload_file = upload_file + temp = self.worker.check_services_and_stop + self.worker.check_services_and_stop = MagicMock(side_effect=KeyboardInterrupt) + with patch("openprocurement.bot.identification.databridge.bridge.gevent.killall"): + self.worker.run() + self.assertEqual(scanner.call_count, 1) + self.assertEqual(filter_tender.call_count, 1) + self.assertEqual(edr_handler.call_count, 1) + self.assertEqual(upload_file.call_count, 1) diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index 60a8896..edcaaa7 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -8,7 +8,7 @@ import datetime import requests_mock import random - +from time import sleep from gevent.queue import Queue from gevent.hub import LoopExit from mock import patch, MagicMock @@ -16,7 +16,7 @@ from openprocurement.bot.identification.databridge.edr_handler import EdrHandler from openprocurement.bot.identification.databridge.filter_tender import FilterTenders -from openprocurement.bot.identification.databridge.utils import Data, generate_doc_id +from openprocurement.bot.identification.databridge.utils import Data, generate_doc_id, RetryException from openprocurement.bot.identification.tests.utils import custom_sleep, generate_answers, generate_request_id, ResponseMock from openprocurement.bot.identification.client import ProxyClient from openprocurement.bot.identification.databridge.constants import version, author @@ -283,6 +283,99 @@ def test_get_edr_id_empty_response(self, mrequest, gevent_sleep): self.assertEqual(mrequest.call_count, 1) # Requests must call proxy once self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_retry_get_edr_id_mock_retry_exception(self, mrequest, gevent_sleep): + """Change get_edr_id_request to Mock which raises RetryException""" + gevent_sleep.side_effect = custom_sleep + tender_id = uuid.uuid4().hex + document_id = generate_doc_id() + award_id = uuid.uuid4().hex + edr_req_id = generate_request_id() + proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + mrequest.get("{uri}".format(uri=proxy_client.verify_url), + [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}]) + edrpou_codes_queue = Queue(10) + edrpou_ids_queue = Queue(10) + upload_to_doc_service_queue = Queue(10) + edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, + {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) + mock_response = MagicMock() + mock_response.status_code = 500 + worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edrpou_ids_queue, upload_to_doc_service_queue, MagicMock()) + worker.get_edr_id_request = MagicMock(side_effect=RetryException("Test exception", mock_response)) + sleep(0.0000000001) + self.assertEqual((worker.get_edr_id_request.call_count > 1), True) + mock_response.status_code = 404 + mock_response.json = MagicMock() + mock_response.json.return_value = {'errors': [ + {'description':[ {'error': {"errorDetails": "Couldn't find this code in EDR.",'code': "notFound"}, 'meta': {"sourceDate": "2017-04-25T11:56:36+00:00"}} ]} + ]} + sleep(0.1) + self.assertEquals(upload_to_doc_service_queue.get(), + Data(tender_id=tender_id, item_id=award_id, + code='123', item_name='awards', + edr_ids=[], + file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", 'id': document_id, "version": version, 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', + edr_req_id]}})) + worker.shutdown() + + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_retry_get_edr_id_mock_retry_exception_404(self, mrequest, gevent_sleep): + """Change get_edr_id_request to Mock which raises RetryException""" + gevent_sleep.side_effect = custom_sleep + tender_id = uuid.uuid4().hex + document_id = generate_doc_id() + award_id = uuid.uuid4().hex + edr_req_id = generate_request_id() + proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + mrequest.get("{uri}".format(uri=proxy_client.verify_url), + [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, + {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, + {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, + {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, + {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, + {'json': {'errors': [{'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}]}]}, + 'status_code': 404, 'headers': {'X-Request-ID': edr_req_id}}]) + + edrpou_codes_queue = Queue(10) + edrpou_ids_queue = Queue(10) + upload_to_doc_service_queue = Queue(10) + edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, + {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) + mock_response = MagicMock() + mock_response.status_code = 404 + mock_response.json = MagicMock() + mock_response.json.return_value = {'errors': [ + {'description': [{'error': {"errorDetails": "Couldn't find this code in EDR.", 'code': "notFound"}, + 'meta': {"sourceDate": "2017-04-25T11:56:36+00:00"}}]} + ]} + worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edrpou_ids_queue, upload_to_doc_service_queue, MagicMock()) + worker.get_edr_id_request = MagicMock(side_effect=RetryException("Test exception", mock_response)) + sleep(0.1) + self.assertEqual((worker.get_edr_id_request.call_count >= 1), True) + mock_response.status_code = 404 + mock_response.json = MagicMock() + mock_response.json.return_value = {'errors': [ + {'description':[ {'error': {"errorDetails": "Couldn't find this code in EDR.",'code': "notFound"}, 'meta': {"sourceDate": "2017-04-25T11:56:36+00:00"}} ]} + ]} + self.assertEquals(upload_to_doc_service_queue.get(), + Data(tender_id=tender_id, item_id=award_id, + code='123', item_name='awards', + edr_ids=[], + file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", 'id': document_id, "version": version, 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', + edr_req_id]}})) + worker.shutdown() + @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_id_empty_response(self, mrequest, gevent_sleep): @@ -381,6 +474,67 @@ def test_get_edr_details_two_ids(self, mrequest, gevent_sleep): self.assertEqual(mrequest.request_history[2].url, u'127.0.0.1:80/api/1.0/details/322') self.assertIsNotNone(mrequest.request_history[2].headers['X-Client-Request-ID']) + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): + """Accept two ids in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" + gevent_sleep.side_effect = custom_sleep + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + edr_req_id = generate_request_id() + edr_details_req_id = [generate_request_id(), generate_request_id(), generate_request_id()] + proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + mrequest.get("{url}".format(url=proxy_client.verify_url), + json={'data': [{'x_edrInternalId': '321'}, {'x_edrInternalId': '322'}], + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, status_code=200, headers={'X-Request-ID': edr_req_id}) + mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=321), + json={'errors':{'error': {'description': 'Not enough money'}, + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}}, + status_code=402, headers={'X-Request-ID': edr_details_req_id[0]}) + mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=322), + json={'errors':{'error': {'description': 'Not enough money'}, + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}}, + status_code=402, headers={'X-Request-ID': edr_details_req_id[0]}) + edrpou_codes_queue = Queue(10) + edr_ids_queue = Queue(10) + upload_to_doc_service_queue = Queue(10) + edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, + {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) + worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, MagicMock()) + sleep(1) + self.assertEqual(worker.exit, True) + worker.shutdown() + + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_retry_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): + """Accept two ids in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" + gevent_sleep.side_effect = custom_sleep + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + edr_req_id = generate_request_id() + edr_details_req_id = [generate_request_id(), generate_request_id(), generate_request_id()] + proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + mrequest.get("{url}".format(url=proxy_client.verify_url), + json={'data': [{'x_edrInternalId': '322'}], + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, status_code=200, headers={'X-Request-ID': edr_req_id}) + mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=322), + [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, + 'headers': {'X-Request-ID': edr_details_req_id[1]}}, + {'json': {'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, 'status_code': 402, + 'headers': {'X-Request-ID': edr_details_req_id[2]}}]) + edrpou_codes_queue = Queue(10) + edr_ids_queue = Queue(10) + upload_to_doc_service_queue = Queue(10) + edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, + {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) + worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, MagicMock()) + sleep(1) + self.assertEqual(worker.exit, True) + worker.shutdown() + @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_details_two_ids(self, mrequest, gevent_sleep): From 4bcc50f1039f61c35dccea34cc2267c0f692f764 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Thu, 1 Jun 2017 11:07:26 +0300 Subject: [PATCH 03/10] Fixed constant in bridge tests --- openprocurement/bot/identification/tests/bridge.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index b174279..65daf6a 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -42,7 +42,7 @@ 'empty_stack_sync_delay': 101, 'on_error_sleep_delay': 5, 'api_token': "api_token", - 'delay': 1 + 'delay': 15 } } @@ -762,7 +762,6 @@ def test_run_mock_exception(self, sleep): @patch('gevent.sleep') def test_run_mock_check_services_and_start(self, sleep): self.worker = EdrDataBridge(config) - self.assertEqual(self.worker.delay, 1) # create mocks scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] self.worker.scanner = scanner From 34f4009105e2be7a9926656fa3569e7dddefac45 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Thu, 1 Jun 2017 18:10:23 +0300 Subject: [PATCH 04/10] Removed extra lines and redundant tests --- .../bot/identification/databridge/bridge.py | 49 +- .../identification/databridge/edr_handler.py | 24 +- .../bot/identification/tests/bridge.py | 548 +++--------------- .../bot/identification/tests/edr_handler.py | 111 +--- .../bot/identification/tests/utils.py | 29 - 5 files changed, 124 insertions(+), 637 deletions(-) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index 1a28d08..a482098 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -119,7 +119,7 @@ def check_doc_service(self): Separated to allow for possible granular checks """ try: - r = request("{host}:{port}/".format(host=self.doc_service_host, port=self.doc_service_port)) + request("{host}:{port}/".format(host=self.doc_service_host, port=self.doc_service_port)) except RequestError as e: logger.info('DocService connection error, message {}'.format(e), extra=journal_context({"MESSAGE_ID": DATABRIDGE_DOC_SERVICE_CONN_ERROR}, {})) @@ -128,9 +128,7 @@ def check_doc_service(self): return True def check_openprocurement_api(self): - """Makes request to the TendersClient, returns True if it's up, raises RequestError otherwise - Separated to allow for possible granular checks - """ + """Makes request to the TendersClient, returns True if it's up, raises RequestError otherwise""" try: logger.debug("Checking status of openprocurement api") self.client.head('/api/{}/spore'.format(self.api_version)) @@ -142,9 +140,7 @@ def check_openprocurement_api(self): return True def check_proxy(self): - """Makes request to the EDR proxy, returns True if it's up, raises RequestError otherwise - Separated to allow for possible granular checks - """ + """Makes request to the EDR proxy, returns True if it's up, raises RequestError otherwise""" try: logger.debug("Checking status of proxy") self.proxyClient.health() @@ -173,34 +169,27 @@ def check_services_and_start(self): self.check_proxy() and self.check_paid_requests() and self.check_openprocurement_api() and self.check_doc_service() except Exception as e: logger.warning("Service is still unavailable") - return "Services are still unavailable" else: logger.warning("All services have become available, starting all workers") self.set_exit_status(False) self.is_sleeping = False - return "All services have become available, starting all workers" def check_services_and_stop(self): """ Check all services and workers; if at least one service is unavailable or at least one of the workers is sleeping (has self.exit=True), sleep all the workers and set self.is_sleeping to True - :return: True - Stopped, False = Not stopped """ - logger.info("Checking status of all services to stop bot if necessary") + logger.info("Checking status of all services to stop the bot if necessary") try: self.check_proxy() and self.check_doc_service() and self.check_openprocurement_api() except Exception as e: - logger.info("Service is down, stopping all bots") + logger.info("Service is down, stopping all workers") self.is_sleeping = True self.set_exit_status(True) - return True else: - for job in self.jobs.values(): - if job.exit: - self.is_sleeping = True - self.set_exit_status(True) - return True - return False + if any([job.exit for job in self.jobs.values()]): + self.is_sleeping = True + self.set_exit_status(True) def _start_jobs(self): self.jobs = {'scanner': self.scanner(), @@ -216,22 +205,22 @@ def run(self): while True: gevent.sleep(self.delay) self.check_services_and_stop() - if counter == 1: + if counter == 20: if self.is_sleeping: self.check_services_and_start() logger.info('Current state: Filtered tenders {}; Edrpou codes queue {}; Retry edrpou codes queue {}; ' 'Edr ids queue {}; Retry edr ids queue {}; Upload to doc service {}; Retry upload to doc service {}; ' 'Upload to tender {}; Retry upload to tender {}'.format( - self.filtered_tender_ids_queue.qsize(), - self.edrpou_codes_queue.qsize(), - self.jobs['edr_handler'].retry_edrpou_codes_queue.qsize() if self.jobs['edr_handler'] else 0, - self.edr_ids_queue.qsize(), - self.jobs['edr_handler'].retry_edr_ids_queue.qsize() if self.jobs['edr_handler'] else 0, - self.upload_to_doc_service_queue.qsize(), - self.jobs['upload_file'].retry_upload_to_doc_service_queue.qsize() if self.jobs['upload_file'] else 0, - self.upload_to_tender_queue.qsize(), - self.jobs['upload_file'].retry_upload_to_tender_queue.qsize() if self.jobs['upload_file'] else 0 - )) + self.filtered_tender_ids_queue.qsize(), + self.edrpou_codes_queue.qsize(), + self.jobs['edr_handler'].retry_edrpou_codes_queue.qsize() if self.jobs['edr_handler'] else 0, + self.edr_ids_queue.qsize(), + self.jobs['edr_handler'].retry_edr_ids_queue.qsize() if self.jobs['edr_handler'] else 0, + self.upload_to_doc_service_queue.qsize(), + self.jobs['upload_file'].retry_upload_to_doc_service_queue.qsize() if self.jobs['upload_file'] else 0, + self.upload_to_tender_queue.qsize(), + self.jobs['upload_file'].retry_upload_to_tender_queue.qsize() if self.jobs['upload_file'] else 0 + )) counter = 0 counter += 1 for name, job in self.jobs.items(): diff --git a/openprocurement/bot/identification/databridge/edr_handler.py b/openprocurement/bot/identification/databridge/edr_handler.py index 12c8941..0b52f48 100644 --- a/openprocurement/bot/identification/databridge/edr_handler.py +++ b/openprocurement/bot/identification/databridge/edr_handler.py @@ -18,7 +18,6 @@ Data, journal_context, validate_param, RetryException, check_add_suffix ) from openprocurement.bot.identification.databridge.constants import version -from requests import RequestException logger = logging.getLogger(__name__) @@ -113,7 +112,6 @@ def retry_get_edr_id(self): """Get data from retry_edrpou_codes_queue; Put data into edr_ids_queue if request is successful, otherwise put data back to retry_edrpou_codes_queue.""" while not self.exit: - # logger.info("retry iteration ") try: tender_data = self.retry_edrpou_codes_queue.get() except LoopExit: @@ -172,7 +170,7 @@ def retry_get_edr_id(self): tender_data.item_id)) gevent.sleep(0) - @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1) + @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_edr_id_request(self, param, code, document_id): """Execute request to EDR Api for retry queue objects.""" response = self.proxyClient.verify(param, code, headers={'X-Client-Request-ID': document_id}) @@ -186,7 +184,6 @@ def get_edr_details(self): """Get data from edr_ids_queue; make request to EDR Api for detailed info; Required fields is put to Data.file_content variable, Data object is put to upload_to_doc_service_queue.""" while not self.exit: - logger.info("get_edr_details iteration") try: tender_data = self.edr_ids_queue.peek() except LoopExit: @@ -198,7 +195,6 @@ def get_edr_details(self): self.until_too_many_requests_event.wait() meta_id = tender_data.file_content['meta']['id'] for edr_id in tender_data.edr_ids: - logger.info("edr_id: {}".format(edr_id)) # if more then 1 instance add amount and number of document to document_id document_id = check_add_suffix(tender_data.edr_ids, meta_id, tender_data.edr_ids.index(edr_id) + 1) tender_data.file_content['meta']['id'] = document_id @@ -225,8 +221,12 @@ def get_edr_details(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) elif response.status_code == 402: - logger.info("NOT ENOUGH MONEY") + logger.info("Payment required; received for tender {} {} {} {}.".format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, + params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.exit = True + break else: file_content = tender_data.file_content @@ -263,8 +263,16 @@ def retry_get_edr_details(self): tender_data.file_content['meta']['sourceRequests'].append(response.headers['X-Request-ID']) except RetryException as re: if re.args[1].status_code == 402: - logger.info("NOT ENOUGH MONEY") + logger.info("Payment required; received for tender {} {} {} {}.".format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, + params={"TENDER_ID": tender_data.tender_id, + "DOCUMENT_ID": document_id})) self.exit = True + file_content = tender_data.file_content + self.retry_edr_ids_queue.put(Data(tender_data.tender_id, tender_data.item_id, tender_data.code, + tender_data.item_name, [edr_id], file_content)) + break else: self.retry_edr_ids_queue.put((Data(tender_data.tender_id, tender_data.item_id, tender_data.code, tender_data.item_name, [edr_id], tender_data.file_content))) @@ -291,7 +299,7 @@ def retry_get_edr_details(self): params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) gevent.sleep(0) - @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1) + @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_edr_details_request(self, edr_id, document_id): """Execute request to EDR Api to get detailed info for retry queue objects.""" response = self.proxyClient.details(id=edr_id, headers={'X-Client-Request-ID': document_id}) diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index 65daf6a..35fbe38 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -2,8 +2,6 @@ import unittest import os -import logging -import time from requests import RequestException @@ -17,7 +15,6 @@ from openprocurement_client.client import TendersClientSync, TendersClient from openprocurement.bot.identification.client import DocServiceClient, ProxyClient from openprocurement.bot.identification.databridge.constants import test_x_edr_internal_id -from openprocurement.bot.identification.tests.utils import MockLoggingHandler @@ -41,8 +38,7 @@ 'full_stack_sync_delay': 15, 'empty_stack_sync_delay': 101, 'on_error_sleep_delay': 5, - 'api_token': "api_token", - 'delay': 15 + 'api_token': "api_token" } } @@ -76,7 +72,8 @@ def setUpClass(cls): cls.doc_server = WSGIServer(('127.0.0.1', 20606), cls.doc_server_bottle, log=None) setup_routing(cls.doc_server_bottle, doc_response, path='/') cls.proxy_server = WSGIServer(('127.0.0.1', 20607), cls.proxy_server_bottle, log=None) - setup_routing(cls.proxy_server_bottle, proxy_response, path='/api/1.0/health') + setup_routing(cls.proxy_server_bottle, proxy_response, + path=['/api/1.0/health', '/api/1.0/details/{}'.format(test_x_edr_internal_id)]) # start servers cls.api_server.start() @@ -84,12 +81,6 @@ def setUpClass(cls): cls.doc_server.start() cls.proxy_server.start() - # mock logging - logger = logging.getLogger("openprocurement.bot.identification.databridge.bridge") - cls.log_handler = MockLoggingHandler(level='DEBUG') - logger.addHandler(cls.log_handler) - cls.log_messages = cls.log_handler.messages - @classmethod def tearDownClass(cls): cls.api_server.close() @@ -99,24 +90,22 @@ def tearDownClass(cls): def setUp(self): super(BaseServersTest, self).setUp() - self.log_handler.reset() + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/health') + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.worker = EdrDataBridge(config) + workers = {'scanner': MagicMock(), 'filter_tender':MagicMock(),'edr_handler': MagicMock(), 'upload_file': MagicMock()} + for name, value in workers.items(): + value.return_value.exit = False + setattr(self.worker, name, value) + self.worker._start_jobs() def tearDown(self): setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/health') del self.worker - def setup_routing(app, func, path='/api/2.3/spore', method='GET'): - app.routes = [] app.route(path, method, func) -def setup_routing_multiple(app, func, paths=None, method='GET'): - if paths is None: - paths = ['/api/2.3/spore'] - app.routes = [] - for path in paths: - app.route(path, method, func) - def response_spore(): response.set_cookie("SERVER_ID", ("a7afc9b1fc79e640f2487ba48243ca071c07a823d27" "8cf9b7adf0fae467a524747e3c6c6973262130fac2b" @@ -139,7 +128,6 @@ def proxy_response_402(): class TestBridgeWorker(BaseServersTest): def test_init(self): - # setup_routing(self.api_server_bottle, response_spore) self.worker = EdrDataBridge(config) self.assertEqual(self.worker.delay, 15) self.assertEqual(self.worker.increment_step, 1) @@ -190,7 +178,7 @@ def test_tender_sync_clients(self, sync_client, client, doc_service_client, prox 'version': config['main']['proxy_version']}) def test_start_jobs(self): - # setup_routing(self.api_server_bottle, response_spore) + setup_routing(self.api_server_bottle, response_spore) self.worker = EdrDataBridge(config) scanner, filter_tender, edr_handler, upload_file = [MagicMock(return_value=i) for i in range(4)] @@ -202,9 +190,9 @@ def test_start_jobs(self): self.worker._start_jobs() # check that all jobs were started self.assertTrue(scanner.called) - self.assertTrue(scanner.called) - self.assertTrue(scanner.called) - self.assertTrue(scanner.called) + self.assertTrue(filter_tender.called) + self.assertTrue(edr_handler.called) + self.assertTrue(upload_file.called) self.assertEqual(self.worker.jobs['scanner'], 0) self.assertEqual(self.worker.jobs['filter_tender'], 1) @@ -277,7 +265,6 @@ def test_paid_requests_failure(self): self.assertEqual(self.worker.check_paid_requests(), True) def test_paid_requests_success(self): - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) self.worker = EdrDataBridge(config) self.assertEqual(self.worker.check_paid_requests(), True) @@ -286,516 +273,139 @@ def test_paid_requests_success(self): # region check_and_stop def test_check_and_stop_did_not_stop(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker.check_proxy = MagicMock() - self.worker.check_proxy.return_value = True - self.worker.check_doc_service = MagicMock() - self.worker.check_doc_service.return_value = True - self.worker.check_openprocurement_api = MagicMock() - self.worker.check_openprocurement_api.return_value = True - self.worker._start_jobs() + """Test that check_and_stop, when called with all services working, does call them and does not shut down bot""" + functions = {'check_proxy': MagicMock(), 'check_doc_service': MagicMock(), 'check_openprocurement_api': MagicMock()} + for name, value in functions.items(): + value.return_value = True + setattr(self.worker, name, value) self.assertEqual(self.worker.is_sleeping, False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) - has_stopped = self.worker.check_services_and_stop() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) + self.worker.check_services_and_stop() self.assertEqual(self.worker.check_proxy.call_count, 1) self.assertEqual(self.worker.check_doc_service.call_count, 1) self.assertEqual(self.worker.check_openprocurement_api.call_count, 1) - self.assertEqual(has_stopped, False) self.assertEqual(self.worker.is_sleeping, False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) def test_check_and_stop_stops_after_exit(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - temp = self.worker.set_exit_status - self.worker.set_exit_status = MagicMock(side_effect=temp) - self.assertEqual(self.worker.set_exit_status.call_count, 0) + """ + 1. Stop scanner + 2. Check that calling check_and_stop stops the bot + """ + self.worker.set_exit_status = MagicMock(side_effect=self.worker.set_exit_status) self.worker._start_jobs() - self.assertEqual(self.worker.is_sleeping, False) - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(self.worker.set_exit_status.call_count, 0) - self.assertEqual(has_stopped, False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) - scanner.return_value.exit = True - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(self.worker.set_exit_status.call_count, 1) - self.assertEqual(has_stopped, True) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.worker.set_exit_status.assert_called_with(True) - self.assertEqual(self.worker.set_exit_status.call_count, 1) - - def test_check_and_stop_not_stops_mock_jobs(self): - self.worker = EdrDataBridge(config) - temp = self.worker.set_exit_status - self.worker.set_exit_status = MagicMock(side_effect=temp) - self.assertEqual(self.worker.set_exit_status.call_count, 0) - self.worker.is_sleeping = False - mock_job = MagicMock() - self.worker.jobs = {'test': mock_job} - - mock_job.exit = False - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, False) - self.assertEqual(self.worker.is_sleeping, False) - self.assertEqual(self.worker.set_exit_status.call_count, 0) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) - - def test_check_and_stop_stops_after_exit_mock_jobs(self): - self.worker = EdrDataBridge(config) - temp = self.worker.set_exit_status - self.worker.set_exit_status = MagicMock(side_effect=temp) - self.assertEqual(self.worker.set_exit_status.call_count, 0) - self.worker.is_sleeping = False - mock_job = MagicMock() - self.worker.jobs = {'test': mock_job} - - mock_job.exit = True - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, True) - self.assertEqual(self.worker.is_sleeping, True) + self.worker.scanner.return_value.exit = True + self.worker.check_services_and_stop() self.assertEqual(self.worker.set_exit_status.call_count, 1) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) self.worker.set_exit_status.assert_called_with(True) - def test_check_and_stop_stops_with_failed_proxy(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - self.assertEqual(self.worker.check_services_and_stop(), False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) - self.assertEqual(self.worker.is_sleeping, False) - setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') - self.assertEqual(self.worker.check_services_and_stop(), True) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/health') - def test_check_and_stop_stops_with_failed_doc_service(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) + """ + 1. Stop doc_service + 2. Check that check_and_stop stops the bot + """ + self.worker.check_services_and_stop() self.assertEqual(self.worker.is_sleeping, False) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) self.doc_server.stop() - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, True) + self.worker.check_services_and_stop() self.assertEqual(self.worker.is_sleeping, True) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.doc_server.start() - def test_check_and_stop_stops_with_failed_api(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) - self.assertEqual(self.worker.is_sleeping, False) - self.api_server.stop() - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, True) - self.assertEqual(self.worker.is_sleeping, True) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.api_server.start() - # endregion # region check_and_start def test_check_and_start_call(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() + """ + Check that check_and_start runs and changes values accordingly + """ self.worker.set_exit_status(True) - setup_routing_multiple(self.proxy_server_bottle, proxy_response, - paths=['/api/1.0/health', '/api/1.0/details/{}'.format(test_x_edr_internal_id)]) - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) self.worker.check_services_and_start() - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) self.assertEqual(self.worker.is_sleeping, False) def test_check_and_start_does_not_start_with_failed_proxy(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - # setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') self.proxy_server.stop() self.worker.set_exit_status(True) self.worker.is_sleeping = True - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) + self.worker.check_services_and_start() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.assertEqual(self.worker.is_sleeping, True) self.proxy_server.start() def test_check_and_start_does_not_start_without_paid_requests(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() + """ + 1. Imitate running out of paid requests + 2. Check that check_and_start does not start the bot + """ setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) self.worker.set_exit_status(True) self.worker.is_sleeping = True - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - - def test_check_and_start_does_not_start_with_failed_doc_service(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - self.doc_server.stop() - self.worker.set_exit_status(True) - self.worker.is_sleeping = True - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.doc_server.start() - - def test_check_and_start_does_not_start_with_failed_api(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - self.api_server.stop() - self.worker.set_exit_status(True) - self.worker.is_sleeping = True - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) + self.worker.check_services_and_start() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.assertEqual(self.worker.is_sleeping, True) - self.api_server.start() def test_check_and_start_starts_with_all_up(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker.check_proxy = MagicMock() - self.worker.check_proxy.return_value = True - self.worker.check_paid_requests = MagicMock() - self.worker.check_paid_requests.return_value = True - self.worker.check_doc_service = MagicMock() - self.worker.check_doc_service.return_value = True - self.worker.check_openprocurement_api = MagicMock() - self.worker.check_openprocurement_api.return_value = True - self.worker._start_jobs() + """ + 1. Set all workers to sleep and is_sleeping to True + 2. Check that check_and_start starts workers and sets is_sleeping to True + """ + functions = {'check_proxy': MagicMock(), 'check_paid_requests': MagicMock(), 'check_doc_service': MagicMock(), 'check_openprocurement_api': MagicMock()} + for name, value in functions.items(): + value.return_value = True + setattr(self.worker, name, value) self.worker.set_exit_status(True) self.worker.is_sleeping = True - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - temp = self.worker.set_exit_status - self.worker.set_exit_status = MagicMock(side_effect=temp) - self.assertEqual(self.worker.check_services_and_start(), "All services have become available, starting all workers") + self.worker.set_exit_status = MagicMock(side_effect=self.worker.set_exit_status) + self.worker.check_services_and_start() self.worker.set_exit_status.assert_called_with(False) self.assertEqual(self.worker.set_exit_status.call_count, 1) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) - self.assertEqual(self.worker.is_sleeping, False) - - def test_check_and_start_does_not_start_with_failed_api_then_starts(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - self.api_server.stop() - self.worker.set_exit_status(True) - self.worker.is_sleeping = True - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.assertEqual(self.worker.check_services_and_start(), "Services are still unavailable") - self.api_server.start() - self.assertEqual(self.worker.check_services_and_start(), "All services have become available, starting all workers") - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) - self.assertEqual(self.worker.is_sleeping, False) - - def test_check_and_start_starts_after_stop(self): - """ - 1. Configure worker, set up mocks of all 4 workers with exit parameters - 2. Start all jobs - 3. Simulate that /health now returns an error - 4. Check that check_services_and_stop stops all workers and sets is_sleeping to True - 5. Simulate return of /health to normal - 6. Check that check_services_and_start starts all workers and sets is_sleeping to False - """ - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') - self.worker.check_services_and_stop() - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - setup_routing_multiple(self.proxy_server_bottle, proxy_response, - paths=['/api/1.0/health', '/api/1.0/details/{}'.format(test_x_edr_internal_id)]) - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) - self.worker.check_services_and_start() - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) self.assertEqual(self.worker.is_sleeping, False) - def test_check_and_start_does_not_start_after_stop(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/health') - self.worker.check_services_and_stop() - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - self.worker.check_services_and_start() - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) - self.assertEqual(self.worker.is_sleeping, True) - # endregion def test_combine_check_stop_and_start_api(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) + self.worker.check_services_and_stop() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) self.assertEqual(self.worker.is_sleeping, False) self.api_server.stop() - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, True) + self.worker.check_services_and_stop() self.assertEqual(self.worker.is_sleeping, True) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.api_server.start() - has_started = self.worker.check_services_and_start() - self.assertEqual(has_started, "All services have become available, starting all workers") - has_stopped = self.worker.check_services_and_stop() - self.assertEqual(has_stopped, False) + self.worker.check_services_and_start() + self.worker.check_services_and_stop() def test_set_exit(self): - self.worker = EdrDataBridge(config) - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - scanner.return_value.exit = False - filter_tender.return_value.exit = False - edr_handler.return_value.exit = False - upload_file.return_value.exit = False - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - self.worker._start_jobs() - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) self.worker.set_exit_status(True) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, True) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.worker.set_exit_status(False) - for i in self.worker.jobs.values(): - self.assertEqual(i.exit, False) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) + @patch('gevent.sleep') def test_run_mock_exception(self, sleep): - self.worker = EdrDataBridge(config) - # create mocks - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - temp = self.worker.check_services_and_stop self.worker.check_services_and_stop = MagicMock(side_effect=Exception) with patch('__builtin__.True', AlmostAlwaysTrue(100)): self.worker.run() - self.assertEqual(scanner.call_count, 1) - self.assertEqual(filter_tender.call_count, 1) - self.assertEqual(edr_handler.call_count, 1) - self.assertEqual(upload_file.call_count, 1) + self.assertEqual(self.worker.scanner.call_count, 2) + self.assertEqual(self.worker.filter_tender.call_count, 2) + self.assertEqual(self.worker.edr_handler.call_count, 2) + self.assertEqual(self.worker.upload_file.call_count, 2) @patch('gevent.sleep') def test_run_mock_check_services_and_start(self, sleep): - self.worker = EdrDataBridge(config) - # create mocks - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - temp = self.worker.check_services_and_start self.worker.check_services_and_start = MagicMock() self.worker.check_services_and_stop = MagicMock() self.worker.is_sleeping = True - time.sleep(20) self.worker.run() self.assertEqual(self.worker.check_services_and_start.call_count, 1) - self.assertEqual(scanner.call_count, 1) - self.assertEqual(filter_tender.call_count, 1) - self.assertEqual(edr_handler.call_count, 1) - self.assertEqual(upload_file.call_count, 1) - - @patch('gevent.sleep') - def test_run_mock_keyboard_interrupt(self, sleep): - self.worker = EdrDataBridge(config) - # create mocks - scanner, filter_tender, edr_handler, upload_file = [MagicMock() for i in range(4)] - self.worker.scanner = scanner - self.worker.filter_tender = filter_tender - self.worker.edr_handler = edr_handler - self.worker.upload_file = upload_file - temp = self.worker.check_services_and_stop - self.worker.check_services_and_stop = MagicMock(side_effect=KeyboardInterrupt) - with patch("openprocurement.bot.identification.databridge.bridge.gevent.killall"): - self.worker.run() - self.assertEqual(scanner.call_count, 1) - self.assertEqual(filter_tender.call_count, 1) - self.assertEqual(edr_handler.call_count, 1) - self.assertEqual(upload_file.call_count, 1) - + self.assertEqual(self.worker.check_services_and_stop.call_count, 21) + self.assertEqual(self.worker.scanner.call_count, 2) + self.assertEqual(self.worker.filter_tender.call_count, 2) + self.assertEqual(self.worker.edr_handler.call_count, 2) + self.assertEqual(self.worker.upload_file.call_count, 2) diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index edcaaa7..273ffef 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -53,7 +53,6 @@ def test_proxy_client(self, mrequest, gevent_sleep): local_edr_ids = get_random_edr_ids(2) edr_req_ids = [generate_request_id(), generate_request_id()] edr_details_req_ids = [generate_request_id(), generate_request_id()] - mrequest.get("{uri}".format(uri=proxy_client.health_url)) mrequest.get("{uri}".format(uri=proxy_client.verify_url), [{'json': {'data': [{'x_edrInternalId': local_edr_ids[0]}], "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'json': {'data': [{'x_edrInternalId': local_edr_ids[1]}], "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}}]) @@ -283,99 +282,6 @@ def test_get_edr_id_empty_response(self, mrequest, gevent_sleep): self.assertEqual(mrequest.call_count, 1) # Requests must call proxy once self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_retry_get_edr_id_mock_retry_exception(self, mrequest, gevent_sleep): - """Change get_edr_id_request to Mock which raises RetryException""" - gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - document_id = generate_doc_id() - award_id = uuid.uuid4().hex - edr_req_id = generate_request_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}]) - edrpou_codes_queue = Queue(10) - edrpou_ids_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - mock_response = MagicMock() - mock_response.status_code = 500 - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edrpou_ids_queue, upload_to_doc_service_queue, MagicMock()) - worker.get_edr_id_request = MagicMock(side_effect=RetryException("Test exception", mock_response)) - sleep(0.0000000001) - self.assertEqual((worker.get_edr_id_request.call_count > 1), True) - mock_response.status_code = 404 - mock_response.json = MagicMock() - mock_response.json.return_value = {'errors': [ - {'description':[ {'error': {"errorDetails": "Couldn't find this code in EDR.",'code': "notFound"}, 'meta': {"sourceDate": "2017-04-25T11:56:36+00:00"}} ]} - ]} - sleep(0.1) - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - edr_ids=[], - file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", 'id': document_id, "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id]}})) - worker.shutdown() - - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_retry_get_edr_id_mock_retry_exception_404(self, mrequest, gevent_sleep): - """Change get_edr_id_request to Mock which raises RetryException""" - gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - document_id = generate_doc_id() - award_id = uuid.uuid4().hex - edr_req_id = generate_request_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}]}]}, - 'status_code': 404, 'headers': {'X-Request-ID': edr_req_id}}]) - - edrpou_codes_queue = Queue(10) - edrpou_ids_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - mock_response = MagicMock() - mock_response.status_code = 404 - mock_response.json = MagicMock() - mock_response.json.return_value = {'errors': [ - {'description': [{'error': {"errorDetails": "Couldn't find this code in EDR.", 'code': "notFound"}, - 'meta': {"sourceDate": "2017-04-25T11:56:36+00:00"}}]} - ]} - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edrpou_ids_queue, upload_to_doc_service_queue, MagicMock()) - worker.get_edr_id_request = MagicMock(side_effect=RetryException("Test exception", mock_response)) - sleep(0.1) - self.assertEqual((worker.get_edr_id_request.call_count >= 1), True) - mock_response.status_code = 404 - mock_response.json = MagicMock() - mock_response.json.return_value = {'errors': [ - {'description':[ {'error': {"errorDetails": "Couldn't find this code in EDR.",'code': "notFound"}, 'meta': {"sourceDate": "2017-04-25T11:56:36+00:00"}} ]} - ]} - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - edr_ids=[], - file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", 'id': document_id, "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id]}})) - worker.shutdown() - @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_id_empty_response(self, mrequest, gevent_sleep): @@ -486,23 +392,25 @@ def test_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): edr_details_req_id = [generate_request_id(), generate_request_id(), generate_request_id()] proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') mrequest.get("{url}".format(url=proxy_client.verify_url), - json={'data': [{'x_edrInternalId': '321'}, {'x_edrInternalId': '322'}], + json={'data': [{'x_edrInternalId': '321'}], "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, status_code=200, headers={'X-Request-ID': edr_req_id}) mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=321), json={'errors':{'error': {'description': 'Not enough money'}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}}, status_code=402, headers={'X-Request-ID': edr_details_req_id[0]}) - mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=322), - json={'errors':{'error': {'description': 'Not enough money'}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}}, - status_code=402, headers={'X-Request-ID': edr_details_req_id[0]}) edrpou_codes_queue = Queue(10) edr_ids_queue = Queue(10) upload_to_doc_service_queue = Queue(10) edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, MagicMock()) - sleep(1) + while not worker.exit: + sleep(1) + self.assertEqual(edrpou_codes_queue.qsize(), 0) + self.assertEqual(edr_ids_queue.qsize(), 0) # This means that queue empties every outer loop, regardless of inner one + self.assertEqual(mrequest.call_count, 2) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') + self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/details/321') self.assertEqual(worker.exit, True) worker.shutdown() @@ -531,7 +439,8 @@ def test_retry_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep) edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, MagicMock()) - sleep(1) + while not worker.exit: + sleep(1) self.assertEqual(worker.exit, True) worker.shutdown() diff --git a/openprocurement/bot/identification/tests/utils.py b/openprocurement/bot/identification/tests/utils.py index c810076..59d578f 100644 --- a/openprocurement/bot/identification/tests/utils.py +++ b/openprocurement/bot/identification/tests/utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import logging from gevent import sleep as gsleep from uuid import uuid4 from json import dumps @@ -35,31 +34,3 @@ def body_string(self): def next(self): pass - -class MockLoggingHandler(logging.Handler): - """Mock logging handler to check for expected logs. - - Messages are available from an instance's ``messages`` dict, in order, indexed by - a lowercase log level string (e.g., 'debug', 'info', etc.). - """ - - def __init__(self, *args, **kwargs): - self.messages = {'debug': [], 'info': [], 'warning': [], 'error': [], - 'critical': []} - super(MockLoggingHandler, self).__init__(*args, **kwargs) - - def emit(self, record): - "Store a message from ``record`` in the instance's ``messages`` dict." - self.acquire() - try: - self.messages[record.levelname.lower()].append(record.getMessage()) - finally: - self.release() - - def reset(self): - self.acquire() - try: - for message_list in self.messages.values(): - del message_list[:] - finally: - self.release() From 9d4a917e4516f28e146bb0f573c2294e27eda4ad Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Fri, 2 Jun 2017 11:29:18 +0300 Subject: [PATCH 05/10] Removed unnecessary parts --- .../bot/identification/databridge/bridge.py | 34 +-- .../identification/databridge/edr_handler.py | 8 +- .../bot/identification/tests/bridge.py | 217 +++++++----------- .../bot/identification/tests/edr_handler.py | 8 +- 4 files changed, 96 insertions(+), 171 deletions(-) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index a482098..58efe35 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -115,9 +115,6 @@ def config_get(self, name): return self.config.get('main').get(name) def check_doc_service(self): - """Makes request to the doc_service, returns True if it's up, raises RequestError otherwise - Separated to allow for possible granular checks - """ try: request("{host}:{port}/".format(host=self.doc_service_host, port=self.doc_service_port)) except RequestError as e: @@ -128,9 +125,7 @@ def check_doc_service(self): return True def check_openprocurement_api(self): - """Makes request to the TendersClient, returns True if it's up, raises RequestError otherwise""" try: - logger.debug("Checking status of openprocurement api") self.client.head('/api/{}/spore'.format(self.api_version)) except RequestError as e: logger.info('TendersServer connection error, message {}'.format(e), @@ -140,9 +135,7 @@ def check_openprocurement_api(self): return True def check_proxy(self): - """Makes request to the EDR proxy, returns True if it's up, raises RequestError otherwise""" try: - logger.debug("Checking status of proxy") self.proxyClient.health() except RequestException as e: logger.info('Proxy server connection error, message {}'.format(e), @@ -152,44 +145,35 @@ def check_proxy(self): return True def check_paid_requests(self): - """Makes request to the EDR proxy, returns True if we have paid requests, otherwise return False""" response = self.proxyClient.details(id=test_x_edr_internal_id) - if response.status_code == 402: # status of not having paid requests - logger.info("Checking for paid requests failed") + if response.status_code == 402: raise RequestException("Not enough money") return True - def set_exit_status(self, new_status): + def set_sleep(self, new_status): + self.is_sleeping= new_status for job in self.jobs.values(): job.exit = new_status def check_services_and_start(self): - logger.info("Checking status of all services to try restarting") try: self.check_proxy() and self.check_paid_requests() and self.check_openprocurement_api() and self.check_doc_service() - except Exception as e: + except Exception: logger.warning("Service is still unavailable") else: logger.warning("All services have become available, starting all workers") - self.set_exit_status(False) - self.is_sleeping = False + self.set_sleep(False) def check_services_and_stop(self): - """ - Check all services and workers; if at least one service is unavailable or at least one of the workers is - sleeping (has self.exit=True), sleep all the workers and set self.is_sleeping to True - """ - logger.info("Checking status of all services to stop the bot if necessary") + """Check all services and workers; sleep all the workers and set self.is_sleeping to True if needed""" try: self.check_proxy() and self.check_doc_service() and self.check_openprocurement_api() - except Exception as e: + except Exception: logger.info("Service is down, stopping all workers") - self.is_sleeping = True - self.set_exit_status(True) + self.set_sleep(True) else: if any([job.exit for job in self.jobs.values()]): - self.is_sleeping = True - self.set_exit_status(True) + self.set_sleep(True) def _start_jobs(self): self.jobs = {'scanner': self.scanner(), diff --git a/openprocurement/bot/identification/databridge/edr_handler.py b/openprocurement/bot/identification/databridge/edr_handler.py index 0b52f48..ee1f87f 100644 --- a/openprocurement/bot/identification/databridge/edr_handler.py +++ b/openprocurement/bot/identification/databridge/edr_handler.py @@ -30,7 +30,6 @@ class EdrHandler(Greenlet): def __init__(self, proxyClient, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, processing_items, delay=15): super(EdrHandler, self).__init__() self.exit = False - self.has_paid_requests = True self.start_time = datetime.now() # init clients @@ -221,7 +220,7 @@ def get_edr_details(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) elif response.status_code == 402: - logger.info("Payment required; received for tender {} {} {} {}.".format( + logger.info("Payment required; received for tender id {} {} {} document_id {}.".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) @@ -235,7 +234,7 @@ def get_edr_details(self): self.retry_edr_ids_queue.put(Data(tender_data.tender_id, tender_data.item_id, tender_data.code, tender_data.item_name, [edr_id], file_content)) self.handle_status_response(response, tender_data.tender_id) - logger.info('Put tender {} with {} id {} {} to retry_edr_ids_queue'.format( + logger.info('Put tender {} with {} id {} document_id {} to retry_edr_ids_queue'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context(params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.edr_ids_queue.get() @@ -263,7 +262,7 @@ def retry_get_edr_details(self): tender_data.file_content['meta']['sourceRequests'].append(response.headers['X-Request-ID']) except RetryException as re: if re.args[1].status_code == 402: - logger.info("Payment required; received for tender {} {} {} {}.".format( + logger.info("Payment required; received for tender id {} {} {} document_id {}.".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, params={"TENDER_ID": tender_data.tender_id, @@ -333,7 +332,6 @@ def _run(self): try: while not self.exit: gevent.sleep(self.delay) - # logger.info("iteration of run") for name, job in self.immortal_jobs.items(): if job.dead: logger.warning("EDR handler worker {} dead try restart".format(name), diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index 35fbe38..7c5318e 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -23,7 +23,7 @@ 'main': { 'tenders_api_server': 'http://127.0.0.1:20604', - 'tenders_api_version': '2.3', + 'tenders_api_version': '0', 'public_tenders_api_server': 'http://127.0.0.1:20605', 'doc_service_server': 'http://127.0.0.1', 'doc_service_port': 20606, @@ -72,8 +72,7 @@ def setUpClass(cls): cls.doc_server = WSGIServer(('127.0.0.1', 20606), cls.doc_server_bottle, log=None) setup_routing(cls.doc_server_bottle, doc_response, path='/') cls.proxy_server = WSGIServer(('127.0.0.1', 20607), cls.proxy_server_bottle, log=None) - setup_routing(cls.proxy_server_bottle, proxy_response, - path=['/api/1.0/health', '/api/1.0/details/{}'.format(test_x_edr_internal_id)]) + setup_routing(cls.proxy_server_bottle, proxy_response, path='/api/1.0/health') # start servers cls.api_server.start() @@ -89,21 +88,19 @@ def tearDownClass(cls): cls.proxy_server.close() def setUp(self): - super(BaseServersTest, self).setUp() - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/health') setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) self.worker = EdrDataBridge(config) - workers = {'scanner': MagicMock(), 'filter_tender':MagicMock(),'edr_handler': MagicMock(), 'upload_file': MagicMock()} + workers = {'scanner': MagicMock(return_value=MagicMock(exit=False)), + 'filter_tender': MagicMock(return_value=MagicMock(exit=False)), + 'edr_handler': MagicMock(return_value=MagicMock(exit=False)), + 'upload_file': MagicMock(return_value=MagicMock(exit=False))} for name, value in workers.items(): - value.return_value.exit = False setattr(self.worker, name, value) - self.worker._start_jobs() def tearDown(self): - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/health') del self.worker -def setup_routing(app, func, path='/api/2.3/spore', method='GET'): +def setup_routing(app, func, path='/api/0/spore', method='GET'): app.route(path, method, func) def response_spore(): @@ -120,9 +117,8 @@ def proxy_response(): return response def proxy_response_402(): - res = response - res.status = "402 Payment required" - return res + response.status = "402 Payment required" + return response class TestBridgeWorker(BaseServersTest): @@ -211,191 +207,139 @@ def test_run(self, sleep): with patch('__builtin__.True', AlmostAlwaysTrue(100)): self.worker.run() - self.assertEqual(scanner.call_count, 1) - self.assertEqual(filter_tender.call_count, 1) - self.assertEqual(edr_handler.call_count, 1) - self.assertEqual(upload_file.call_count, 1) - - # region testing checks of separate services + self.assertEqual(self.worker.scanner.call_count, 1) + self.assertEqual(self.worker.filter_tender.call_count, 1) + self.assertEqual(self.worker.edr_handler.call_count, 1) + self.assertEqual(self.worker.upload_file.call_count, 1) - def test_proxy_server_failure(self): + def test_proxy_server(self): self.proxy_server.stop() - self.worker = EdrDataBridge(config) with self.assertRaises(RequestException): self.worker.check_proxy() self.proxy_server.start() self.assertEqual(self.worker.check_proxy(), True) - def test_proxy_server_success(self): - self.worker = EdrDataBridge(config) - self.assertEqual(self.worker.check_proxy(), True) - - def test_doc_service_failure(self): + def test_doc_service(self): self.doc_server.stop() - self.worker = EdrDataBridge(config) with self.assertRaises(RequestError): self.worker.check_doc_service() self.doc_server.start() self.assertEqual(self.worker.check_doc_service(), True) - def test_doc_service_success(self): - self.worker = EdrDataBridge(config) - self.assertEqual(self.worker.check_doc_service(), True) - - def test_api_server_failure(self): - self.worker = EdrDataBridge(config) + def test_api_server(self): self.api_server.stop() - with self.assertRaises(RequestError) as err_context: + with self.assertRaises(RequestError): self.worker.check_openprocurement_api() - self.assertEqual(err_context.exception.message, "socket.error: [Errno 111] Connection refused") self.api_server.start() self.assertEqual(self.worker.check_openprocurement_api(), True) - def test_api_server_success(self): - self.worker = EdrDataBridge(config) - self.assertEqual(self.worker.check_openprocurement_api(), True) - - def test_paid_requests_failure(self): + def test_paid_requests(self): setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) - self.worker = EdrDataBridge(config) - with self.assertRaises(RequestException) as err: + with self.assertRaises(RequestException): self.assertEqual(self.worker.check_paid_requests(), True) - self.assertEqual(err.exception.message, "Not enough money") setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) self.assertEqual(self.worker.check_paid_requests(), True) - def test_paid_requests_success(self): - self.worker = EdrDataBridge(config) - self.assertEqual(self.worker.check_paid_requests(), True) - - # endregion - - # region check_and_stop - def test_check_and_stop_did_not_stop(self): - """Test that check_and_stop, when called with all services working, does call them and does not shut down bot""" - functions = {'check_proxy': MagicMock(), 'check_doc_service': MagicMock(), 'check_openprocurement_api': MagicMock()} + self.worker._start_jobs() + functions = {'check_proxy': MagicMock(return_value = True), + 'check_doc_service': MagicMock(return_value = True), + 'check_openprocurement_api': MagicMock(return_value = True)} for name, value in functions.items(): - value.return_value = True setattr(self.worker, name, value) - self.assertEqual(self.worker.is_sleeping, False) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) self.worker.check_services_and_stop() - self.assertEqual(self.worker.check_proxy.call_count, 1) - self.assertEqual(self.worker.check_doc_service.call_count, 1) - self.assertEqual(self.worker.check_openprocurement_api.call_count, 1) + self.assertEqual(all([i.call_count == 1 for i in functions.values()]), True) self.assertEqual(self.worker.is_sleeping, False) self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - def test_check_and_stop_stops_after_exit(self): - """ - 1. Stop scanner - 2. Check that calling check_and_stop stops the bot - """ - self.worker.set_exit_status = MagicMock(side_effect=self.worker.set_exit_status) + def test_check_and_stop(self): self.worker._start_jobs() - self.worker.scanner.return_value.exit = True - self.worker.check_services_and_stop() - self.assertEqual(self.worker.set_exit_status.call_count, 1) - self.worker.set_exit_status.assert_called_with(True) - - def test_check_and_stop_stops_with_failed_doc_service(self): - """ - 1. Stop doc_service - 2. Check that check_and_stop stops the bot - """ + self.proxy_server.stop() self.worker.check_services_and_stop() - self.assertEqual(self.worker.is_sleeping, False) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) + self.proxy_server.start() + self.worker.set_sleep(False) + self.doc_server.stop() self.worker.check_services_and_stop() self.assertEqual(self.worker.is_sleeping, True) self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.doc_server.start() + self.worker.set_sleep(False) - # endregion + self.api_server.stop() + self.worker.check_services_and_stop() + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) + self.api_server.start() + self.worker.set_sleep(False) - # region check_and_start + self.worker.scanner.return_value.exit = True + self.worker.check_services_and_stop() + self.assertEqual(self.worker.is_sleeping, True) + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - def test_check_and_start_call(self): - """ - Check that check_and_start runs and changes values accordingly - """ - self.worker.set_exit_status(True) + def test_check_and_start_does_not_start(self): + self.worker._start_jobs() + self.proxy_server.stop() + self.worker.set_sleep(True) + self.worker.check_services_and_start() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) + self.assertEqual(self.worker.is_sleeping, True) + self.proxy_server.start() self.worker.check_services_and_start() self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - self.assertEqual(self.worker.is_sleeping, False) - def test_check_and_start_does_not_start_with_failed_proxy(self): - self.proxy_server.stop() - self.worker.set_exit_status(True) - self.worker.is_sleeping = True + self.doc_server.stop() + self.worker.set_sleep(True) self.worker.check_services_and_start() self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.assertEqual(self.worker.is_sleeping, True) - self.proxy_server.start() + self.doc_server.start() + self.worker.check_services_and_start() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) + + self.api_server.stop() + self.worker.set_sleep(True) + self.worker.check_services_and_start() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) + self.assertEqual(self.worker.is_sleeping, True) + self.api_server.start() + self.worker.check_services_and_start() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - def test_check_and_start_does_not_start_without_paid_requests(self): - """ - 1. Imitate running out of paid requests - 2. Check that check_and_start does not start the bot - """ setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) - self.worker.set_exit_status(True) - self.worker.is_sleeping = True + self.worker.set_sleep(True) self.worker.check_services_and_start() self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.assertEqual(self.worker.is_sleeping, True) + setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) + self.worker.check_services_and_start() + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - def test_check_and_start_starts_with_all_up(self): - """ - 1. Set all workers to sleep and is_sleeping to True - 2. Check that check_and_start starts workers and sets is_sleeping to True - """ - functions = {'check_proxy': MagicMock(), 'check_paid_requests': MagicMock(), 'check_doc_service': MagicMock(), 'check_openprocurement_api': MagicMock()} + def test_check_and_start(self): + self.worker._start_jobs() + functions = {'check_proxy': MagicMock(return_value = True), 'check_paid_requests': MagicMock(return_value = True), + 'check_doc_service': MagicMock(return_value = True), 'check_openprocurement_api': MagicMock(return_value = True)} for name, value in functions.items(): - value.return_value = True setattr(self.worker, name, value) - self.worker.set_exit_status(True) - self.worker.is_sleeping = True - self.worker.set_exit_status = MagicMock(side_effect=self.worker.set_exit_status) + self.worker.set_sleep(True) self.worker.check_services_and_start() - self.worker.set_exit_status.assert_called_with(False) - self.assertEqual(self.worker.set_exit_status.call_count, 1) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) self.assertEqual(self.worker.is_sleeping, False) - - # endregion + self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) def test_combine_check_stop_and_start_api(self): - self.worker.check_services_and_stop() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - self.assertEqual(self.worker.is_sleeping, False) + self.worker._start_jobs() self.api_server.stop() self.worker.check_services_and_stop() self.assertEqual(self.worker.is_sleeping, True) self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.api_server.start() self.worker.check_services_and_start() - self.worker.check_services_and_stop() - - def test_set_exit(self): - self.worker.set_exit_status(True) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - self.worker.set_exit_status(False) + self.assertEqual(self.worker.is_sleeping, False) self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - - @patch('gevent.sleep') - def test_run_mock_exception(self, sleep): - self.worker.check_services_and_stop = MagicMock(side_effect=Exception) - with patch('__builtin__.True', AlmostAlwaysTrue(100)): - self.worker.run() - self.assertEqual(self.worker.scanner.call_count, 2) - self.assertEqual(self.worker.filter_tender.call_count, 2) - self.assertEqual(self.worker.edr_handler.call_count, 2) - self.assertEqual(self.worker.upload_file.call_count, 2) - @patch('gevent.sleep') def test_run_mock_check_services_and_start(self, sleep): self.worker.check_services_and_start = MagicMock() @@ -404,8 +348,7 @@ def test_run_mock_check_services_and_start(self, sleep): self.worker.run() self.assertEqual(self.worker.check_services_and_start.call_count, 1) self.assertEqual(self.worker.check_services_and_stop.call_count, 21) - self.assertEqual(self.worker.scanner.call_count, 2) - self.assertEqual(self.worker.filter_tender.call_count, 2) - self.assertEqual(self.worker.edr_handler.call_count, 2) - self.assertEqual(self.worker.upload_file.call_count, 2) - + self.assertEqual(self.worker.scanner.call_count, 1) + self.assertEqual(self.worker.filter_tender.call_count, 1) + self.assertEqual(self.worker.edr_handler.call_count, 1) + self.assertEqual(self.worker.upload_file.call_count, 1) diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index 273ffef..2004863 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -383,7 +383,7 @@ def test_get_edr_details_two_ids(self, mrequest, gevent_sleep): @requests_mock.Mocker() @patch('gevent.sleep') def test_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): - """Accept two ids in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" + """Accept one id in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" gevent_sleep.side_effect = custom_sleep tender_id = uuid.uuid4().hex award_id = uuid.uuid4().hex @@ -417,7 +417,7 @@ def test_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): - """Accept two ids in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" + """Accept one id in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" gevent_sleep.side_effect = custom_sleep tender_id = uuid.uuid4().hex award_id = uuid.uuid4().hex @@ -426,9 +426,9 @@ def test_retry_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep) edr_details_req_id = [generate_request_id(), generate_request_id(), generate_request_id()] proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') mrequest.get("{url}".format(url=proxy_client.verify_url), - json={'data': [{'x_edrInternalId': '322'}], + json={'data': [{'x_edrInternalId': '321'}], "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, status_code=200, headers={'X-Request-ID': edr_req_id}) - mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=322), + mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=321), [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_details_req_id[1]}}, {'json': {'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, 'status_code': 402, From 765c417e9175f53d3110c512402802ad7bf7c2b9 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Fri, 2 Jun 2017 12:07:22 +0300 Subject: [PATCH 06/10] Removed the paid_requests check-ups because of the new information --- .../bot/identification/databridge/bridge.py | 25 +++---- .../identification/databridge/constants.py | 1 - .../identification/databridge/edr_handler.py | 34 +++------- .../bot/identification/tests/bridge.py | 25 ++----- .../bot/identification/tests/edr_handler.py | 68 +------------------ 5 files changed, 22 insertions(+), 131 deletions(-) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index 58efe35..7c37f1a 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- from gevent import monkey -from openprocurement.bot.identification.databridge.constants import test_x_edr_internal_id - monkey.patch_all() import logging @@ -13,7 +11,7 @@ from functools import partial from yaml import load from gevent.queue import Queue -from restkit import request, RequestError, ResourceError +from restkit import request, RequestError from requests import RequestException from openprocurement_client.client import TendersClientSync, TendersClient @@ -144,12 +142,6 @@ def check_proxy(self): else: return True - def check_paid_requests(self): - response = self.proxyClient.details(id=test_x_edr_internal_id) - if response.status_code == 402: - raise RequestException("Not enough money") - return True - def set_sleep(self, new_status): self.is_sleeping= new_status for job in self.jobs.values(): @@ -157,19 +149,18 @@ def set_sleep(self, new_status): def check_services_and_start(self): try: - self.check_proxy() and self.check_paid_requests() and self.check_openprocurement_api() and self.check_doc_service() - except Exception: - logger.warning("Service is still unavailable") + self.check_proxy() and self.check_openprocurement_api() and self.check_doc_service() + except Exception as e: + logger.info("Service is still unavailable, message {}".format(e)) else: - logger.warning("All services have become available, starting all workers") + logger.info("All services have become available, starting all workers") self.set_sleep(False) def check_services_and_stop(self): - """Check all services and workers; sleep all the workers and set self.is_sleeping to True if needed""" try: self.check_proxy() and self.check_doc_service() and self.check_openprocurement_api() - except Exception: - logger.info("Service is down, stopping all workers") + except Exception as e: + logger.info("Service is down, stopping all workers, message {}".format(e)) self.set_sleep(True) else: if any([job.exit for job in self.jobs.values()]): @@ -229,7 +220,7 @@ def main(): config = load(config_file_obj.read()) logging.config.dictConfig(config) bridge = EdrDataBridge(config) - bridge.check_proxy() and bridge.check_doc_service() and bridge.check_paid_requests() and bridge.check_openprocurement_api() + bridge.check_proxy() and bridge.check_doc_service() and bridge.check_openprocurement_api() bridge.run() else: logger.info('Invalid configuration file. Exiting...') diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index 4439569..3d04f0c 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,4 +4,3 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -test_x_edr_internal_id = 1050844 \ No newline at end of file diff --git a/openprocurement/bot/identification/databridge/edr_handler.py b/openprocurement/bot/identification/databridge/edr_handler.py index ee1f87f..e9e7d6b 100644 --- a/openprocurement/bot/identification/databridge/edr_handler.py +++ b/openprocurement/bot/identification/databridge/edr_handler.py @@ -13,7 +13,8 @@ from openprocurement.bot.identification.databridge.journal_msg_ids import ( DATABRIDGE_GET_TENDER_FROM_QUEUE, DATABRIDGE_START_EDR_HANDLER, DATABRIDGE_SUCCESS_CREATE_FILE, - DATABRIDGE_EMPTY_RESPONSE, DATABRIDGE_PROXY_SERVER_CONN_ERROR) + DATABRIDGE_EMPTY_RESPONSE +) from openprocurement.bot.identification.databridge.utils import ( Data, journal_context, validate_param, RetryException, check_add_suffix ) @@ -219,13 +220,6 @@ def get_edr_details(self): tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) - elif response.status_code == 402: - logger.info("Payment required; received for tender id {} {} {} document_id {}.".format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, - params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) - self.exit = True - break else: file_content = tender_data.file_content @@ -261,24 +255,12 @@ def retry_get_edr_details(self): if response.headers.get('X-Request-ID'): tender_data.file_content['meta']['sourceRequests'].append(response.headers['X-Request-ID']) except RetryException as re: - if re.args[1].status_code == 402: - logger.info("Payment required; received for tender id {} {} {} document_id {}.".format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_CREATE_FILE}, - params={"TENDER_ID": tender_data.tender_id, - "DOCUMENT_ID": document_id})) - self.exit = True - file_content = tender_data.file_content - self.retry_edr_ids_queue.put(Data(tender_data.tender_id, tender_data.item_id, tender_data.code, - tender_data.item_name, [edr_id], file_content)) - break - else: - self.retry_edr_ids_queue.put((Data(tender_data.tender_id, tender_data.item_id, tender_data.code, - tender_data.item_name, [edr_id], tender_data.file_content))) - logger.info('Put tender {} with {} id {} {} to retry_edr_ids_queue. Error response {}'.format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.args[1].json().get('errors')), - extra=journal_context(params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) - gevent.sleep(0) + self.retry_edr_ids_queue.put((Data(tender_data.tender_id, tender_data.item_id, tender_data.code, + tender_data.item_name, [edr_id], tender_data.file_content))) + logger.info('Put tender {} with {} id {} {} to retry_edr_ids_queue. Error response {}'.format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.args[1].json().get('errors')), + extra=journal_context(params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) + gevent.sleep(0) else: if not isinstance(response.json(), dict): logger.info('Error data type {} {} {} {}. Message {}.'.format( diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index 7c5318e..d9ebb17 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -14,7 +14,6 @@ from openprocurement.bot.identification.databridge.bridge import EdrDataBridge from openprocurement_client.client import TendersClientSync, TendersClient from openprocurement.bot.identification.client import DocServiceClient, ProxyClient -from openprocurement.bot.identification.databridge.constants import test_x_edr_internal_id @@ -88,7 +87,6 @@ def tearDownClass(cls): cls.proxy_server.close() def setUp(self): - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) self.worker = EdrDataBridge(config) workers = {'scanner': MagicMock(return_value=MagicMock(exit=False)), 'filter_tender': MagicMock(return_value=MagicMock(exit=False)), @@ -226,20 +224,13 @@ def test_doc_service(self): self.doc_server.start() self.assertEqual(self.worker.check_doc_service(), True) - def test_api_server(self): + def test_api(self): self.api_server.stop() with self.assertRaises(RequestError): self.worker.check_openprocurement_api() self.api_server.start() self.assertEqual(self.worker.check_openprocurement_api(), True) - def test_paid_requests(self): - setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) - with self.assertRaises(RequestException): - self.assertEqual(self.worker.check_paid_requests(), True) - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) - self.assertEqual(self.worker.check_paid_requests(), True) - def test_check_and_stop_did_not_stop(self): self.worker._start_jobs() functions = {'check_proxy': MagicMock(return_value = True), @@ -309,19 +300,11 @@ def test_check_and_start_does_not_start(self): self.worker.check_services_and_start() self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - setup_routing(self.proxy_server_bottle, proxy_response_402, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) - self.worker.set_sleep(True) - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - self.assertEqual(self.worker.is_sleeping, True) - setup_routing(self.proxy_server_bottle, proxy_response, path='/api/1.0/details/{}'.format(test_x_edr_internal_id)) - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - def test_check_and_start(self): self.worker._start_jobs() - functions = {'check_proxy': MagicMock(return_value = True), 'check_paid_requests': MagicMock(return_value = True), - 'check_doc_service': MagicMock(return_value = True), 'check_openprocurement_api': MagicMock(return_value = True)} + functions = {'check_proxy': MagicMock(return_value = True), + 'check_doc_service': MagicMock(return_value = True), + 'check_openprocurement_api': MagicMock(return_value = True)} for name, value in functions.items(): setattr(self.worker, name, value) self.worker.set_sleep(True) diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index 2004863..3bbc96d 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -8,7 +8,7 @@ import datetime import requests_mock import random -from time import sleep + from gevent.queue import Queue from gevent.hub import LoopExit from mock import patch, MagicMock @@ -16,7 +16,7 @@ from openprocurement.bot.identification.databridge.edr_handler import EdrHandler from openprocurement.bot.identification.databridge.filter_tender import FilterTenders -from openprocurement.bot.identification.databridge.utils import Data, generate_doc_id, RetryException +from openprocurement.bot.identification.databridge.utils import Data, generate_doc_id from openprocurement.bot.identification.tests.utils import custom_sleep, generate_answers, generate_request_id, ResponseMock from openprocurement.bot.identification.client import ProxyClient from openprocurement.bot.identification.databridge.constants import version, author @@ -380,70 +380,6 @@ def test_get_edr_details_two_ids(self, mrequest, gevent_sleep): self.assertEqual(mrequest.request_history[2].url, u'127.0.0.1:80/api/1.0/details/322') self.assertIsNotNone(mrequest.request_history[2].headers['X-Client-Request-ID']) - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): - """Accept one id in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" - gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = generate_request_id() - edr_details_req_id = [generate_request_id(), generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - json={'data': [{'x_edrInternalId': '321'}], - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, status_code=200, headers={'X-Request-ID': edr_req_id}) - mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=321), - json={'errors':{'error': {'description': 'Not enough money'}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}}, - status_code=402, headers={'X-Request-ID': edr_details_req_id[0]}) - edrpou_codes_queue = Queue(10) - edr_ids_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, MagicMock()) - while not worker.exit: - sleep(1) - self.assertEqual(edrpou_codes_queue.qsize(), 0) - self.assertEqual(edr_ids_queue.qsize(), 0) # This means that queue empties every outer loop, regardless of inner one - self.assertEqual(mrequest.call_count, 2) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/details/321') - self.assertEqual(worker.exit, True) - worker.shutdown() - - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_retry_get_edr_details_exits_with_no_money(self, mrequest, gevent_sleep): - """Accept one id in /verify request. Then send 402 to /details requests and check that bot goes to sleep""" - gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = generate_request_id() - edr_details_req_id = [generate_request_id(), generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - json={'data': [{'x_edrInternalId': '321'}], - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, status_code=200, headers={'X-Request-ID': edr_req_id}) - mrequest.get("{url}/{id}".format(url=proxy_client.details_url, id=321), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, - 'headers': {'X-Request-ID': edr_details_req_id[1]}}, - {'json': {'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}, 'status_code': 402, - 'headers': {'X-Request-ID': edr_details_req_id[2]}}]) - edrpou_codes_queue = Queue(10) - edr_ids_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", None, - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, edr_ids_queue, upload_to_doc_service_queue, MagicMock()) - while not worker.exit: - sleep(1) - self.assertEqual(worker.exit, True) - worker.shutdown() - @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_details_two_ids(self, mrequest, gevent_sleep): From c21f4522ad8daa835d388c957f87e5af2305946a Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Fri, 2 Jun 2017 12:52:01 +0300 Subject: [PATCH 07/10] Removed is_sleeping, unified two united checks --- .../bot/identification/databridge/bridge.py | 19 +---- .../bot/identification/tests/bridge.py | 80 ++----------------- 2 files changed, 11 insertions(+), 88 deletions(-) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index 7c37f1a..e964ac0 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -107,7 +107,6 @@ def __init__(self, config): processing_items=self.processing_items, doc_service_client=self.doc_service_client, delay=self.delay) - self.is_sleeping = False def config_get(self, name): return self.config.get('main').get(name) @@ -143,29 +142,19 @@ def check_proxy(self): return True def set_sleep(self, new_status): - self.is_sleeping= new_status for job in self.jobs.values(): job.exit = new_status - def check_services_and_start(self): + def check_services(self): try: self.check_proxy() and self.check_openprocurement_api() and self.check_doc_service() except Exception as e: logger.info("Service is still unavailable, message {}".format(e)) + self.set_sleep(True) else: logger.info("All services have become available, starting all workers") self.set_sleep(False) - def check_services_and_stop(self): - try: - self.check_proxy() and self.check_doc_service() and self.check_openprocurement_api() - except Exception as e: - logger.info("Service is down, stopping all workers, message {}".format(e)) - self.set_sleep(True) - else: - if any([job.exit for job in self.jobs.values()]): - self.set_sleep(True) - def _start_jobs(self): self.jobs = {'scanner': self.scanner(), 'filter_tender': self.filter_tender(), @@ -179,10 +168,8 @@ def run(self): try: while True: gevent.sleep(self.delay) - self.check_services_and_stop() + self.check_services() if counter == 20: - if self.is_sleeping: - self.check_services_and_start() logger.info('Current state: Filtered tenders {}; Edrpou codes queue {}; Retry edrpou codes queue {}; ' 'Edr ids queue {}; Retry edr ids queue {}; Upload to doc service {}; Retry upload to doc service {}; ' 'Upload to tender {}; Retry upload to tender {}'.format( diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index d9ebb17..8308be4 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -231,106 +231,42 @@ def test_api(self): self.api_server.start() self.assertEqual(self.worker.check_openprocurement_api(), True) - def test_check_and_stop_did_not_stop(self): + def test_check_services_did_not_stop(self): self.worker._start_jobs() functions = {'check_proxy': MagicMock(return_value = True), 'check_doc_service': MagicMock(return_value = True), 'check_openprocurement_api': MagicMock(return_value = True)} for name, value in functions.items(): setattr(self.worker, name, value) - self.worker.check_services_and_stop() + self.worker.check_services() self.assertEqual(all([i.call_count == 1 for i in functions.values()]), True) - self.assertEqual(self.worker.is_sleeping, False) self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - def test_check_and_stop(self): + def test_check_services(self): self.worker._start_jobs() self.proxy_server.stop() - self.worker.check_services_and_stop() - self.assertEqual(self.worker.is_sleeping, True) + self.worker.check_services() self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.proxy_server.start() self.worker.set_sleep(False) self.doc_server.stop() - self.worker.check_services_and_stop() - self.assertEqual(self.worker.is_sleeping, True) + self.worker.check_services() self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.doc_server.start() self.worker.set_sleep(False) self.api_server.stop() - self.worker.check_services_and_stop() - self.assertEqual(self.worker.is_sleeping, True) + self.worker.check_services() self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) self.api_server.start() self.worker.set_sleep(False) - self.worker.scanner.return_value.exit = True - self.worker.check_services_and_stop() - self.assertEqual(self.worker.is_sleeping, True) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - - def test_check_and_start_does_not_start(self): - self.worker._start_jobs() - self.proxy_server.stop() - self.worker.set_sleep(True) - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - self.assertEqual(self.worker.is_sleeping, True) - self.proxy_server.start() - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - - self.doc_server.stop() - self.worker.set_sleep(True) - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - self.assertEqual(self.worker.is_sleeping, True) - self.doc_server.start() - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - - self.api_server.stop() - self.worker.set_sleep(True) - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - self.assertEqual(self.worker.is_sleeping, True) - self.api_server.start() - self.worker.check_services_and_start() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - - def test_check_and_start(self): - self.worker._start_jobs() - functions = {'check_proxy': MagicMock(return_value = True), - 'check_doc_service': MagicMock(return_value = True), - 'check_openprocurement_api': MagicMock(return_value = True)} - for name, value in functions.items(): - setattr(self.worker, name, value) - self.worker.set_sleep(True) - self.worker.check_services_and_start() - self.assertEqual(self.worker.is_sleeping, False) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - - def test_combine_check_stop_and_start_api(self): - self.worker._start_jobs() - self.api_server.stop() - self.worker.check_services_and_stop() - self.assertEqual(self.worker.is_sleeping, True) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) - self.api_server.start() - self.worker.check_services_and_start() - self.assertEqual(self.worker.is_sleeping, False) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) - @patch('gevent.sleep') def test_run_mock_check_services_and_start(self, sleep): - self.worker.check_services_and_start = MagicMock() - self.worker.check_services_and_stop = MagicMock() - self.worker.is_sleeping = True + self.worker.check_services = MagicMock() self.worker.run() - self.assertEqual(self.worker.check_services_and_start.call_count, 1) - self.assertEqual(self.worker.check_services_and_stop.call_count, 21) + self.assertEqual(self.worker.check_services.call_count, 21) self.assertEqual(self.worker.scanner.call_count, 1) self.assertEqual(self.worker.filter_tender.call_count, 1) self.assertEqual(self.worker.edr_handler.call_count, 1) From 93872e087a21b268b3fd0017aa009570d93f4a12 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Fri, 2 Jun 2017 15:47:30 +0300 Subject: [PATCH 08/10] Added proper sleep in the scanner --- .../bot/identification/databridge/bridge.py | 4 +- .../bot/identification/databridge/scanner.py | 58 ++++++++++--------- .../bot/identification/tests/bridge.py | 38 ++++++++---- 3 files changed, 60 insertions(+), 40 deletions(-) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index e964ac0..d4b4178 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -149,7 +149,7 @@ def check_services(self): try: self.check_proxy() and self.check_openprocurement_api() and self.check_doc_service() except Exception as e: - logger.info("Service is still unavailable, message {}".format(e)) + logger.info("Service is unavailable, message {}".format(e)) self.set_sleep(True) else: logger.info("All services have become available, starting all workers") @@ -186,7 +186,7 @@ def run(self): counter = 0 counter += 1 for name, job in self.jobs.items(): - if job.dead: + if job.dead and not job.exit: logger.warning('Restarting {} worker'.format(name), extra=journal_context({"MESSAGE_ID": DATABRIDGE_RESTART_WORKER})) self.jobs[name] = gevent.spawn(getattr(self, name)) diff --git a/openprocurement/bot/identification/databridge/scanner.py b/openprocurement/bot/identification/databridge/scanner.py index 5185dc7..1d9138b 100644 --- a/openprocurement/bot/identification/databridge/scanner.py +++ b/openprocurement/bot/identification/databridge/scanner.py @@ -92,36 +92,38 @@ def get_tenders(self, params={}, direction=""): raise re def get_tenders_forward(self): - logger.info('Start forward data sync worker...') - params = {'opt_fields': 'status,procurementMethodType', 'mode': '_all_'} - try: - for tender in self.get_tenders(params=params, direction="forward"): - logger.info('Forward sync: Put tender {} to process...'.format(tender['id']), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS}, - {"TENDER_ID": tender['id']})) - self.filtered_tender_ids_queue.put(tender['id']) - except Exception as e: - logger.warning('Forward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {})) - logger.exception(e) - else: - logger.warning('Forward data sync finished!') + if not self.exit: + logger.info('Start forward data sync worker...') + params = {'opt_fields': 'status,procurementMethodType', 'mode': '_all_'} + try: + for tender in self.get_tenders(params=params, direction="forward"): + logger.info('Forward sync: Put tender {} to process...'.format(tender['id']), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS}, + {"TENDER_ID": tender['id']})) + self.filtered_tender_ids_queue.put(tender['id']) + except Exception as e: + logger.warning('Forward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {})) + logger.exception(e) + else: + logger.warning('Forward data sync finished!') def get_tenders_backward(self): - logger.info('Start backward data sync worker...') - params = {'opt_fields': 'status,procurementMethodType', 'descending': 1, 'mode': '_all_'} - try: - for tender in self.get_tenders(params=params, direction="backward"): - logger.info('Backward sync: Put tender {} to process...'.format(tender['id']), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS}, - {"TENDER_ID": tender['id']})) - self.filtered_tender_ids_queue.put(tender['id']) - except Exception as e: - logger.warning('Backward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {})) - logger.exception(e) - return False - else: - logger.info('Backward data sync finished.') - return True + if not self.exit: + logger.info('Start backward data sync worker...') + params = {'opt_fields': 'status,procurementMethodType', 'descending': 1, 'mode': '_all_'} + try: + for tender in self.get_tenders(params=params, direction="backward"): + logger.info('Backward sync: Put tender {} to process...'.format(tender['id']), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS}, + {"TENDER_ID": tender['id']})) + self.filtered_tender_ids_queue.put(tender['id']) + except Exception as e: + logger.warning('Backward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {})) + logger.exception(e) + return False + else: + logger.info('Backward data sync finished.') + return True def _start_synchronization_workers(self): logger.info('Scanner starting forward and backward sync workers') diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index 8308be4..c70c888 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -16,8 +16,6 @@ from openprocurement.bot.identification.client import DocServiceClient, ProxyClient - - config = { 'main': { @@ -215,21 +213,21 @@ def test_proxy_server(self): with self.assertRaises(RequestException): self.worker.check_proxy() self.proxy_server.start() - self.assertEqual(self.worker.check_proxy(), True) + self.assertTrue(self.worker.check_proxy()) def test_doc_service(self): self.doc_server.stop() with self.assertRaises(RequestError): self.worker.check_doc_service() self.doc_server.start() - self.assertEqual(self.worker.check_doc_service(), True) + self.assertTrue(self.worker.check_doc_service()) def test_api(self): self.api_server.stop() with self.assertRaises(RequestError): self.worker.check_openprocurement_api() self.api_server.start() - self.assertEqual(self.worker.check_openprocurement_api(), True) + self.assertTrue(self.worker.check_openprocurement_api()) def test_check_services_did_not_stop(self): self.worker._start_jobs() @@ -239,29 +237,49 @@ def test_check_services_did_not_stop(self): for name, value in functions.items(): setattr(self.worker, name, value) self.worker.check_services() - self.assertEqual(all([i.call_count == 1 for i in functions.values()]), True) - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), False) + self.assertTrue(all([i.call_count == 1 for i in functions.values()])) + self.assertFalse(all([i.exit for i in self.worker.jobs.values()])) def test_check_services(self): self.worker._start_jobs() self.proxy_server.stop() self.worker.check_services() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) + self.assertTrue(all([i.exit for i in self.worker.jobs.values()])) self.proxy_server.start() self.worker.set_sleep(False) self.doc_server.stop() self.worker.check_services() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) + self.assertTrue(all([i.exit for i in self.worker.jobs.values()])) self.doc_server.start() self.worker.set_sleep(False) self.api_server.stop() self.worker.check_services() - self.assertEqual(all([i.exit for i in self.worker.jobs.values()]), True) + self.assertTrue(all([i.exit for i in self.worker.jobs.values()])) self.api_server.start() self.worker.set_sleep(False) + def test_check_services_needs_all(self): + self.worker._start_jobs() + self.worker.set_sleep(True) + self.proxy_server.stop() + self.doc_server.stop() + self.api_server.stop() + + self.proxy_server.start() + self.worker.check_services() + self.assertTrue(all([i.exit for i in self.worker.jobs.values()])) + + self.doc_server.start() + self.worker.check_services() + self.assertTrue(all([i.exit for i in self.worker.jobs.values()])) + self.worker.set_sleep(False) + + self.api_server.start() + self.worker.check_services() + self.assertFalse(all([i.exit for i in self.worker.jobs.values()])) + @patch('gevent.sleep') def test_run_mock_check_services_and_start(self, sleep): self.worker.check_services = MagicMock() From 942638c8af6f6ae0da27cd4b5e1d805d8c9d91a8 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Fri, 2 Jun 2017 17:46:52 +0300 Subject: [PATCH 09/10] Fixed the test --- openprocurement/bot/identification/tests/bridge.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index c70c888..386a09e 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -52,6 +52,14 @@ def __nonzero__(self): return bool(1) return bool(0) +number1 = 0 +class AlmostKeyboardInterrupt(KeyboardInterrupt): + + def __init__(self): + global number1 + if number1 < 100: + number1 += 1 + super(KeyboardInterrupt, self).__init__() class BaseServersTest(unittest.TestCase): """Api server to test openprocurement.integrations.edr.databridge.bridge """ @@ -279,12 +287,12 @@ def test_check_services_needs_all(self): self.api_server.start() self.worker.check_services() self.assertFalse(all([i.exit for i in self.worker.jobs.values()])) - + # @patch('gevent.sleep') - def test_run_mock_check_services_and_start(self, sleep): + def test_run_with_mock_check_services(self, gevent_sleep): + """Basic test to ensure run() goes into the while (and inside that for) loops and that jobs are called only once""" self.worker.check_services = MagicMock() self.worker.run() - self.assertEqual(self.worker.check_services.call_count, 21) self.assertEqual(self.worker.scanner.call_count, 1) self.assertEqual(self.worker.filter_tender.call_count, 1) self.assertEqual(self.worker.edr_handler.call_count, 1) From 84c6d87fa83d278a800f93d112cd52a317ef4c4f Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Fri, 2 Jun 2017 18:07:45 +0300 Subject: [PATCH 10/10] Fixed tabulation and removed leftover stuff --- .../bot/identification/databridge/bridge.py | 20 +++++++++---------- .../bot/identification/tests/bridge.py | 13 ++---------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index d4b4178..a947dda 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -173,16 +173,16 @@ def run(self): logger.info('Current state: Filtered tenders {}; Edrpou codes queue {}; Retry edrpou codes queue {}; ' 'Edr ids queue {}; Retry edr ids queue {}; Upload to doc service {}; Retry upload to doc service {}; ' 'Upload to tender {}; Retry upload to tender {}'.format( - self.filtered_tender_ids_queue.qsize(), - self.edrpou_codes_queue.qsize(), - self.jobs['edr_handler'].retry_edrpou_codes_queue.qsize() if self.jobs['edr_handler'] else 0, - self.edr_ids_queue.qsize(), - self.jobs['edr_handler'].retry_edr_ids_queue.qsize() if self.jobs['edr_handler'] else 0, - self.upload_to_doc_service_queue.qsize(), - self.jobs['upload_file'].retry_upload_to_doc_service_queue.qsize() if self.jobs['upload_file'] else 0, - self.upload_to_tender_queue.qsize(), - self.jobs['upload_file'].retry_upload_to_tender_queue.qsize() if self.jobs['upload_file'] else 0 - )) + self.filtered_tender_ids_queue.qsize(), + self.edrpou_codes_queue.qsize(), + self.jobs['edr_handler'].retry_edrpou_codes_queue.qsize() if self.jobs['edr_handler'] else 0, + self.edr_ids_queue.qsize(), + self.jobs['edr_handler'].retry_edr_ids_queue.qsize() if self.jobs['edr_handler'] else 0, + self.upload_to_doc_service_queue.qsize(), + self.jobs['upload_file'].retry_upload_to_doc_service_queue.qsize() if self.jobs['upload_file'] else 0, + self.upload_to_tender_queue.qsize(), + self.jobs['upload_file'].retry_upload_to_tender_queue.qsize() if self.jobs['upload_file'] else 0 + )) counter = 0 counter += 1 for name, job in self.jobs.items(): diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index 386a09e..3a80b81 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -52,14 +52,6 @@ def __nonzero__(self): return bool(1) return bool(0) -number1 = 0 -class AlmostKeyboardInterrupt(KeyboardInterrupt): - - def __init__(self): - global number1 - if number1 < 100: - number1 += 1 - super(KeyboardInterrupt, self).__init__() class BaseServersTest(unittest.TestCase): """Api server to test openprocurement.integrations.edr.databridge.bridge """ @@ -178,7 +170,6 @@ def test_tender_sync_clients(self, sync_client, client, doc_service_client, prox 'version': config['main']['proxy_version']}) def test_start_jobs(self): - setup_routing(self.api_server_bottle, response_spore) self.worker = EdrDataBridge(config) scanner, filter_tender, edr_handler, upload_file = [MagicMock(return_value=i) for i in range(4)] @@ -287,9 +278,9 @@ def test_check_services_needs_all(self): self.api_server.start() self.worker.check_services() self.assertFalse(all([i.exit for i in self.worker.jobs.values()])) - # + @patch('gevent.sleep') - def test_run_with_mock_check_services(self, gevent_sleep): + def test_run_with_mock_check_services(self, sleep): """Basic test to ensure run() goes into the while (and inside that for) loops and that jobs are called only once""" self.worker.check_services = MagicMock() self.worker.run()