From 560eb21c5cd49c0b15a4759da152009ea222e18c Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Tue, 23 May 2017 15:49:32 +0300 Subject: [PATCH 01/14] Added processing of the case where item status was changed while bot was processing it --- .../databridge/journal_msg_ids.py | 1 + .../identification/databridge/upload_file.py | 18 ++++- .../bot/identification/tests/upload_file.py | 74 +++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/openprocurement/bot/identification/databridge/journal_msg_ids.py b/openprocurement/bot/identification/databridge/journal_msg_ids.py index d5e155d..8930eb6 100644 --- a/openprocurement/bot/identification/databridge/journal_msg_ids.py +++ b/openprocurement/bot/identification/databridge/journal_msg_ids.py @@ -39,3 +39,4 @@ DATABRIDGE_DOC_SERVICE_CONN_ERROR = 'edr_databridge_doc_service_conn_error' DATABRIDGE_PROXY_SERVER_CONN_ERROR = 'edr_databridge_proxy_server_conn_error' DATABRIDGE_422_UPLOAD_TO_TENDER = 'edr_databridge_422_upload_to_tender' +DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING = 'edr_databridge_item_status_changed_while_processing' diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index fef95cc..2acccd3 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -15,8 +15,7 @@ DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER, DATABRIDGE_START_UPLOAD, - DATABRIDGE_422_UPLOAD_TO_TENDER -) + DATABRIDGE_422_UPLOAD_TO_TENDER, DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING) from openprocurement.bot.identification.databridge.constants import file_name logger = logging.getLogger(__name__) @@ -163,6 +162,13 @@ def upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) continue + elif re.status_int == 403 and "Can't add document in current qualification status" in re.message: + logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, + {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id}) + ) + self.update_processing_items(tender_data.tender_id, tender_data.item_id) else: self.retry_upload_to_tender_queue.put(tender_data) except Exception as e: @@ -201,6 +207,14 @@ def retry_upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) continue + elif re.status_int == 403 and "Can't add document in current qualification status" in re.message: + logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, + {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id}) + ) + self.update_processing_items(tender_data.tender_id, tender_data.item_id) + continue else: logger.info('Exception while retry uploading file to tender {} {} {} {}. Message: {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.message), diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index da0e668..67cc164 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -194,6 +194,39 @@ def test_request_failed(self, mrequest, gevent_sleep): self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_request_failed_item_status_change(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}' + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + mrequest.post('{url}'.format(url=doc_service_client.url), + json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + status_code=200) + client = MagicMock() + client._create_tender_resource_item.side_effect = ResourceError(http_code=403, msg=err_message) + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) + upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) + sleep(10) + worker.shutdown() + self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + self.assertEqual(processing_items, {}) + self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request + @requests_mock.Mocker() @patch('gevent.sleep') def test_request_failed_in_retry(self, mrequest, gevent_sleep): @@ -231,6 +264,47 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_request_failed_in_retry_item_status_change(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}' + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + mrequest.post('{url}'.format(url=doc_service_client.url), + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + status_code=200) + client = MagicMock() + client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message)] + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) + upload_to_doc_service_queue.put( + Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, + doc_service_client) + sleep(60) + worker.shutdown() + self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + self.assertEqual(processing_items, {}) + self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + @requests_mock.Mocker() @patch('gevent.sleep') def test_processing_items(self, mrequest, gevent_sleep): From 18696026746ad2eb395238fb73df82b6beb6129c Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Tue, 23 May 2017 18:10:55 +0300 Subject: [PATCH 02/14] Added checking for awards --- .../identification/databridge/constants.py | 6 ++ .../identification/databridge/upload_file.py | 12 +-- .../bot/identification/tests/upload_file.py | 79 ++++++++++++++++++- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index 033116d..74479e2 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,3 +4,9 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" +upload_file_error_messages = ["Can\'t add document in current qualification status", + "Can\'t add document in current cancelled qualification status", + "Can\'t add document in current (unsuccessful) award status", + "Can\'t add document in current (active) award status", + "Can\'t add document in current (cancelled) award status", + ] diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index 2acccd3..d320b44 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -9,14 +9,15 @@ from gevent import Greenlet, spawn from gevent.hub import LoopExit from restkit import ResourceError +from simplejson import loads -from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file +from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, RetryException from openprocurement.bot.identification.databridge.journal_msg_ids import ( DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER, DATABRIDGE_START_UPLOAD, DATABRIDGE_422_UPLOAD_TO_TENDER, DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING) -from openprocurement.bot.identification.databridge.constants import file_name +from openprocurement.bot.identification.databridge.constants import file_name, upload_file_error_messages logger = logging.getLogger(__name__) @@ -141,6 +142,7 @@ def upload_to_tender(self): while not self.exit: try: tender_data = self.upload_to_tender_queue.get() + gevent.sleep(20) except LoopExit: gevent.sleep(0) continue @@ -162,7 +164,7 @@ def upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) continue - elif re.status_int == 403 and "Can't add document in current qualification status" in re.message: + elif re.status_int == 403 and re.msg and loads(re.msg).get('errors')[0].get('description') in upload_file_error_messages: logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, @@ -207,7 +209,8 @@ def retry_upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) continue - elif re.status_int == 403 and "Can't add document in current qualification status" in re.message: + elif re.status_int == 403 and loads(re.msg).get('errors')[0].get('description') in upload_file_error_messages: + logger.exception(re) logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, @@ -220,7 +223,6 @@ def retry_upload_to_tender(self): tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) - logger.exception(re) self.retry_upload_to_tender_queue.put(tender_data) except Exception as e: logger.info('Exception while retry uploading file to tender {} {} {} {}. Message: {}'.format( diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 67cc164..1d99e59 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -196,7 +196,7 @@ def test_request_failed(self, mrequest, gevent_sleep): @requests_mock.Mocker() @patch('gevent.sleep') - def test_request_failed_item_status_change(self, mrequest, gevent_sleep): + def test_request_failed_qualification_status_change(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}' doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') @@ -227,6 +227,42 @@ def test_request_failed_item_status_change(self, mrequest, gevent_sleep): self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_request_failed_award_status_change(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (unsuccessful) award status"}]}' + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + mrequest.post('{url}'.format(url=doc_service_client.url), + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + status_code=200) + client = MagicMock() + client._create_tender_resource_item.side_effect = ResourceError(http_code=403, msg=err_message) + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) + upload_to_doc_service_queue.put( + Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, + doc_service_client) + sleep(10) + worker.shutdown() + self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + self.assertEqual(processing_items, {}) + self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request + @requests_mock.Mocker() @patch('gevent.sleep') def test_request_failed_in_retry(self, mrequest, gevent_sleep): @@ -305,6 +341,47 @@ def test_request_failed_in_retry_item_status_change(self, mrequest, gevent_sleep self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_request_failed_in_retry_item_status_change(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}' + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + mrequest.post('{url}'.format(url=doc_service_client.url), + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + status_code=200) + client = MagicMock() + client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message)] + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) + upload_to_doc_service_queue.put( + Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, + doc_service_client) + sleep(60) + worker.shutdown() + self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + self.assertEqual(processing_items, {}) + self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + @requests_mock.Mocker() @patch('gevent.sleep') def test_processing_items(self, mrequest, gevent_sleep): From 89e535deee49a112d49e89a6878825aa7c0e4b2f Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 08:47:17 +0300 Subject: [PATCH 03/14] Removal of extra checks --- .../identification/databridge/constants.py | 7 +- .../identification/databridge/upload_file.py | 4 +- .../bot/identification/tests/upload_file.py | 118 ------------------ 3 files changed, 3 insertions(+), 126 deletions(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index 74479e2..d27a3bf 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,9 +4,4 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -upload_file_error_messages = ["Can\'t add document in current qualification status", - "Can\'t add document in current cancelled qualification status", - "Can\'t add document in current (unsuccessful) award status", - "Can\'t add document in current (active) award status", - "Can\'t add document in current (cancelled) award status", - ] +upload_file_error_messages = ["Can\'t add document in current qualification status"] diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index d320b44..068c116 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -164,7 +164,7 @@ def upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) continue - elif re.status_int == 403 and re.msg and loads(re.msg).get('errors')[0].get('description') in upload_file_error_messages: + elif re.status_int == 403 and re.msg and any([i in re.msg for i in upload_file_error_messages]): logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, @@ -209,7 +209,7 @@ def retry_upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) continue - elif re.status_int == 403 and loads(re.msg).get('errors')[0].get('description') in upload_file_error_messages: + elif re.status_int == 403 and any([i in re.msg for i in upload_file_error_messages]): logger.exception(re) logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 1d99e59..1fd81b2 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -227,42 +227,6 @@ def test_request_failed_qualification_status_change(self, mrequest, gevent_sleep self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_request_failed_award_status_change(self, mrequest, gevent_sleep): - gevent_sleep.side_effect = custom_sleep - err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (unsuccessful) award status"}]}' - doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - mrequest.post('{url}'.format(url=doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) - client = MagicMock() - client._create_tender_resource_item.side_effect = ResourceError(http_code=403, msg=err_message) - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - key = '{}_{}'.format(tender_id, award_id) - processing_items = {key: 1} - upload_to_doc_service_queue = Queue(10) - upload_to_tender_queue = Queue(10) - upload_to_doc_service_queue.put( - Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) - worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, - doc_service_client) - sleep(10) - worker.shutdown() - self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(mrequest.call_count, 1) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(processing_items, {}) - self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request - @requests_mock.Mocker() @patch('gevent.sleep') def test_request_failed_in_retry(self, mrequest, gevent_sleep): @@ -300,88 +264,6 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_request_failed_in_retry_item_status_change(self, mrequest, gevent_sleep): - gevent_sleep.side_effect = custom_sleep - err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}' - doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - mrequest.post('{url}'.format(url=doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) - client = MagicMock() - client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message)] - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - key = '{}_{}'.format(tender_id, award_id) - processing_items = {key: 1} - upload_to_doc_service_queue = Queue(10) - upload_to_tender_queue = Queue(10) - upload_to_doc_service_queue.put( - Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) - worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, - doc_service_client) - sleep(60) - worker.shutdown() - self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(mrequest.call_count, 1) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(processing_items, {}) - self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request - - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_request_failed_in_retry_item_status_change(self, mrequest, gevent_sleep): - gevent_sleep.side_effect = custom_sleep - err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}' - doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - mrequest.post('{url}'.format(url=doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) - client = MagicMock() - client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message)] - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - key = '{}_{}'.format(tender_id, award_id) - processing_items = {key: 1} - upload_to_doc_service_queue = Queue(10) - upload_to_tender_queue = Queue(10) - upload_to_doc_service_queue.put( - Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) - worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, - doc_service_client) - sleep(60) - worker.shutdown() - self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(mrequest.call_count, 1) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(processing_items, {}) - self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request - @requests_mock.Mocker() @patch('gevent.sleep') def test_processing_items(self, mrequest, gevent_sleep): From 2df5dd8969fb833f26320ddfee3b48572c2448c9 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 09:02:58 +0300 Subject: [PATCH 04/14] Added tests for retry --- .../identification/databridge/upload_file.py | 1 - .../bot/identification/tests/upload_file.py | 49 +++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index 068c116..d09ceff 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -210,7 +210,6 @@ def retry_upload_to_tender(self): self.update_processing_items(tender_data.tender_id, tender_data.item_id) continue elif re.status_int == 403 and any([i in re.msg for i in upload_file_error_messages]): - logger.exception(re) logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 1fd81b2..b7807fb 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -233,10 +233,11 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post('{url}'.format(url=doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, status_code=200) client = MagicMock() client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), @@ -252,6 +253,46 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): processing_items = {key: 1} upload_to_doc_service_queue = Queue(10) upload_to_tender_queue = Queue(10) + upload_to_doc_service_queue.put( + Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, + doc_service_client) + sleep(60) + worker.shutdown() + self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + self.assertEqual(processing_items, {}) + self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_request_failed_in_retry_status(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}' + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + mrequest.post('{url}'.format(url=doc_service_client.url), + json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + status_code=200) + client = MagicMock() + client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message), + ResourceError(http_code=403, msg=err_message)] + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) sleep(60) From c5d70d845cc6bcc8aa20367068252c502927e250 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 09:45:31 +0300 Subject: [PATCH 05/14] Adding active awards processing --- openprocurement/bot/identification/databridge/constants.py | 3 ++- openprocurement/bot/identification/tests/upload_file.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index d27a3bf..4f0eae9 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,4 +4,5 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -upload_file_error_messages = ["Can\'t add document in current qualification status"] +upload_file_error_messages = ["Can\'t add document in current qualification status", + "Can\'t add document in current (active) award status"] diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index b7807fb..d01313e 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -271,7 +271,7 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_request_failed_in_retry_status(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}' + err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}' doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post('{url}'.format(url=doc_service_client.url), json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', From 75b6ae0b0db94b8ad2b0c3aa0c85db01501e09dc Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 10:40:51 +0300 Subject: [PATCH 06/14] Check whether travis kills on time --- .../bot/identification/tests/upload_file.py | 162 +++++++++--------- 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index d01313e..c8a007f 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -119,48 +119,48 @@ def test_retry_doc_service(self, mrequest, gevent_sleep): self.assertItemsEqual(processing_items.keys(), []) # test that item removed from processing_items self.assertEqual(client._create_tender_resource_item.call_count, 1) # check upload to tender - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_retry_upload_to_tender(self, mrequest, gevent_sleep): - gevent_sleep.side_effect = custom_sleep - doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - mrequest.post('{url}'.format(url=doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) - client = MagicMock() - client._create_tender_resource_item.side_effect = [Unauthorized(http_code=403), - Unauthorized(http_code=403), - Unauthorized(http_code=403), - Unauthorized(http_code=403), - Unauthorized(http_code=403), - Exception(), - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - key = '{}_{}'.format(tender_id, award_id) - processing_items = {key: 1} - upload_to_doc_service_queue = Queue(10) - upload_to_tender_queue = Queue(10) - upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) - self.assertItemsEqual(processing_items.keys(), [key]) - self.assertEqual(upload_to_doc_service_queue.qsize(), 1) - worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(60) - worker.shutdown() - self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(mrequest.call_count, 1) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(processing_items, {}) # test that item removed from processing_items - self.assertEqual(client._create_tender_resource_item.call_count, 7) # check upload to tender + # @requests_mock.Mocker() + # @patch('gevent.sleep') + # def test_retry_upload_to_tender(self, mrequest, gevent_sleep): + # gevent_sleep.side_effect = custom_sleep + # doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + # mrequest.post('{url}'.format(url=doc_service_client.url), + # json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + # 'format': 'application/yaml', + # 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + # 'title': file_name}}, + # status_code=200) + # client = MagicMock() + # client._create_tender_resource_item.side_effect = [Unauthorized(http_code=403), + # Unauthorized(http_code=403), + # Unauthorized(http_code=403), + # Unauthorized(http_code=403), + # Unauthorized(http_code=403), + # Exception(), + # {'data': {'id': uuid.uuid4().hex, + # 'documentOf': 'tender', + # 'documentType': 'registerExtract', + # 'url': 'url'}}] + # tender_id = uuid.uuid4().hex + # award_id = uuid.uuid4().hex + # document_id = generate_doc_id() + # key = '{}_{}'.format(tender_id, award_id) + # processing_items = {key: 1} + # upload_to_doc_service_queue = Queue(10) + # upload_to_tender_queue = Queue(10) + # upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + # self.assertItemsEqual(processing_items.keys(), [key]) + # self.assertEqual(upload_to_doc_service_queue.qsize(), 1) + # worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) + # sleep(60) + # worker.shutdown() + # self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + # self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + # self.assertEqual(mrequest.call_count, 1) + # self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + # self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + # self.assertEqual(processing_items, {}) # test that item removed from processing_items + # self.assertEqual(client._create_tender_resource_item.call_count, 7) # check upload to tender @requests_mock.Mocker() @patch('gevent.sleep') @@ -227,45 +227,45 @@ def test_request_failed_qualification_status_change(self, mrequest, gevent_sleep self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request - @requests_mock.Mocker() - @patch('gevent.sleep') - def test_request_failed_in_retry(self, mrequest, gevent_sleep): - gevent_sleep.side_effect = custom_sleep - doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - mrequest.post('{url}'.format(url=doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) - client = MagicMock() - client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), - ResourceError(http_code=422), - ResourceError(http_code=422), - ResourceError(http_code=422), - ResourceError(http_code=422), - ResourceError(http_code=422)] - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - key = '{}_{}'.format(tender_id, award_id) - processing_items = {key: 1} - upload_to_doc_service_queue = Queue(10) - upload_to_tender_queue = Queue(10) - upload_to_doc_service_queue.put( - Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) - worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, - doc_service_client) - sleep(60) - worker.shutdown() - self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(mrequest.call_count, 1) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(processing_items, {}) - self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + # @requests_mock.Mocker() + # @patch('gevent.sleep') + # def test_request_failed_in_retry(self, mrequest, gevent_sleep): + # gevent_sleep.side_effect = custom_sleep + # doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + # mrequest.post('{url}'.format(url=doc_service_client.url), + # json={'data': { + # 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + # 'format': 'application/yaml', + # 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + # 'title': file_name}}, + # status_code=200) + # client = MagicMock() + # client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), + # ResourceError(http_code=422), + # ResourceError(http_code=422), + # ResourceError(http_code=422), + # ResourceError(http_code=422), + # ResourceError(http_code=422)] + # tender_id = uuid.uuid4().hex + # award_id = uuid.uuid4().hex + # document_id = generate_doc_id() + # key = '{}_{}'.format(tender_id, award_id) + # processing_items = {key: 1} + # upload_to_doc_service_queue = Queue(10) + # upload_to_tender_queue = Queue(10) + # upload_to_doc_service_queue.put( + # Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + # worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, + # doc_service_client) + # sleep(60) + # worker.shutdown() + # self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + # self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + # self.assertEqual(mrequest.call_count, 1) + # self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + # self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + # self.assertEqual(processing_items, {}) + # self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request @requests_mock.Mocker() @patch('gevent.sleep') From 709252e509329709069e20a5c0656151beb13e78 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 11:00:08 +0300 Subject: [PATCH 07/14] Modify how travis runs tests --- .travis.yml | 6 +- .../bot/identification/tests/upload_file.py | 162 +++++++++--------- 2 files changed, 86 insertions(+), 82 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a6b5c8..87dd21a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,11 @@ before_install: install: - bin/buildout -N script: - - bin/nosetests + - bin/nosetests openprocurement/bot/identification/tests/bridge.py + - bin/nosetests openprocurement/bot/identification/tests/scanner.py + - bin/nosetests openprocurement/bot/identification/tests/filter_tender.py + - bin/nosetests openprocurement/bot/identification/tests/edr_handler.py + - bin/nosetests openprocurement/bot/identification/tests/upload_file.py after_success: - bin/coveralls diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index c8a007f..d01313e 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -119,48 +119,48 @@ def test_retry_doc_service(self, mrequest, gevent_sleep): self.assertItemsEqual(processing_items.keys(), []) # test that item removed from processing_items self.assertEqual(client._create_tender_resource_item.call_count, 1) # check upload to tender - # @requests_mock.Mocker() - # @patch('gevent.sleep') - # def test_retry_upload_to_tender(self, mrequest, gevent_sleep): - # gevent_sleep.side_effect = custom_sleep - # doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - # mrequest.post('{url}'.format(url=doc_service_client.url), - # json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - # 'format': 'application/yaml', - # 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - # 'title': file_name}}, - # status_code=200) - # client = MagicMock() - # client._create_tender_resource_item.side_effect = [Unauthorized(http_code=403), - # Unauthorized(http_code=403), - # Unauthorized(http_code=403), - # Unauthorized(http_code=403), - # Unauthorized(http_code=403), - # Exception(), - # {'data': {'id': uuid.uuid4().hex, - # 'documentOf': 'tender', - # 'documentType': 'registerExtract', - # 'url': 'url'}}] - # tender_id = uuid.uuid4().hex - # award_id = uuid.uuid4().hex - # document_id = generate_doc_id() - # key = '{}_{}'.format(tender_id, award_id) - # processing_items = {key: 1} - # upload_to_doc_service_queue = Queue(10) - # upload_to_tender_queue = Queue(10) - # upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) - # self.assertItemsEqual(processing_items.keys(), [key]) - # self.assertEqual(upload_to_doc_service_queue.qsize(), 1) - # worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - # sleep(60) - # worker.shutdown() - # self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') - # self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - # self.assertEqual(mrequest.call_count, 1) - # self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') - # self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - # self.assertEqual(processing_items, {}) # test that item removed from processing_items - # self.assertEqual(client._create_tender_resource_item.call_count, 7) # check upload to tender + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_retry_upload_to_tender(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + mrequest.post('{url}'.format(url=doc_service_client.url), + json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + status_code=200) + client = MagicMock() + client._create_tender_resource_item.side_effect = [Unauthorized(http_code=403), + Unauthorized(http_code=403), + Unauthorized(http_code=403), + Unauthorized(http_code=403), + Unauthorized(http_code=403), + Exception(), + {'data': {'id': uuid.uuid4().hex, + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}}] + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) + upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + self.assertItemsEqual(processing_items.keys(), [key]) + self.assertEqual(upload_to_doc_service_queue.qsize(), 1) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) + sleep(60) + worker.shutdown() + self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + self.assertEqual(processing_items, {}) # test that item removed from processing_items + self.assertEqual(client._create_tender_resource_item.call_count, 7) # check upload to tender @requests_mock.Mocker() @patch('gevent.sleep') @@ -227,45 +227,45 @@ def test_request_failed_qualification_status_change(self, mrequest, gevent_sleep self.assertEqual(processing_items, {}) self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request - # @requests_mock.Mocker() - # @patch('gevent.sleep') - # def test_request_failed_in_retry(self, mrequest, gevent_sleep): - # gevent_sleep.side_effect = custom_sleep - # doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - # mrequest.post('{url}'.format(url=doc_service_client.url), - # json={'data': { - # 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - # 'format': 'application/yaml', - # 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - # 'title': file_name}}, - # status_code=200) - # client = MagicMock() - # client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), - # ResourceError(http_code=422), - # ResourceError(http_code=422), - # ResourceError(http_code=422), - # ResourceError(http_code=422), - # ResourceError(http_code=422)] - # tender_id = uuid.uuid4().hex - # award_id = uuid.uuid4().hex - # document_id = generate_doc_id() - # key = '{}_{}'.format(tender_id, award_id) - # processing_items = {key: 1} - # upload_to_doc_service_queue = Queue(10) - # upload_to_tender_queue = Queue(10) - # upload_to_doc_service_queue.put( - # Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) - # worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, - # doc_service_client) - # sleep(60) - # worker.shutdown() - # self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') - # self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - # self.assertEqual(mrequest.call_count, 1) - # self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') - # self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - # self.assertEqual(processing_items, {}) - # self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_request_failed_in_retry(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + mrequest.post('{url}'.format(url=doc_service_client.url), + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + status_code=200) + client = MagicMock() + client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), + ResourceError(http_code=422), + ResourceError(http_code=422), + ResourceError(http_code=422), + ResourceError(http_code=422), + ResourceError(http_code=422)] + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) + upload_to_doc_service_queue.put( + Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, + doc_service_client) + sleep(60) + worker.shutdown() + self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') + self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) + self.assertEqual(processing_items, {}) + self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request @requests_mock.Mocker() @patch('gevent.sleep') From 619803e06a9eda631255500c45a2c85d87aaf9f4 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 12:05:07 +0300 Subject: [PATCH 08/14] Changed generation of coverage --- .travis.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.travis.yml b/.travis.yml index 87dd21a..2773a42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,21 @@ install: - bin/buildout -N script: - bin/nosetests openprocurement/bot/identification/tests/bridge.py + - mv .coverage .coverage_bridge - bin/nosetests openprocurement/bot/identification/tests/scanner.py + - mv .coverage .coverage_scanner - bin/nosetests openprocurement/bot/identification/tests/filter_tender.py + - mv .coverage .coverage_filter - bin/nosetests openprocurement/bot/identification/tests/edr_handler.py + - mv .coverage .coverage_edr - bin/nosetests openprocurement/bot/identification/tests/upload_file.py + - mv .coverage .coverage_upload + - mv .coverage_bridge .coverage.bridge + - mv .coverage_scanner .coverage.scanner + - mv .coverage_filter .coverage.filter + - mv .coverage_edr .coverage.edr + - mv .coverage_upload .coverage.upload + - bin/coverage combine after_success: - bin/coveralls From 2ae1e7daad88152bb06a13f2358ecc4e6a80d649 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 12:15:39 +0300 Subject: [PATCH 09/14] Cleaned up travis script --- .travis.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2773a42..5724d95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,20 +14,15 @@ install: - bin/buildout -N script: - bin/nosetests openprocurement/bot/identification/tests/bridge.py - - mv .coverage .coverage_bridge + - mv .coverage .coverage.bridge - bin/nosetests openprocurement/bot/identification/tests/scanner.py - - mv .coverage .coverage_scanner + - mv .coverage .coverage.scanner - bin/nosetests openprocurement/bot/identification/tests/filter_tender.py - - mv .coverage .coverage_filter + - mv .coverage .coverage.filter - bin/nosetests openprocurement/bot/identification/tests/edr_handler.py - - mv .coverage .coverage_edr + - mv .coverage .coverage.edr - bin/nosetests openprocurement/bot/identification/tests/upload_file.py - - mv .coverage .coverage_upload - - mv .coverage_bridge .coverage.bridge - - mv .coverage_scanner .coverage.scanner - - mv .coverage_filter .coverage.filter - - mv .coverage_edr .coverage.edr - - mv .coverage_upload .coverage.upload + - mv .coverage .coverage.upload - bin/coverage combine after_success: - bin/coveralls From 3a665cafe325c16e78948244c974a1362ede7c6d Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 12:26:11 +0300 Subject: [PATCH 10/14] Fixing travis --- .travis.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5724d95..2773a42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,15 +14,20 @@ install: - bin/buildout -N script: - bin/nosetests openprocurement/bot/identification/tests/bridge.py - - mv .coverage .coverage.bridge + - mv .coverage .coverage_bridge - bin/nosetests openprocurement/bot/identification/tests/scanner.py - - mv .coverage .coverage.scanner + - mv .coverage .coverage_scanner - bin/nosetests openprocurement/bot/identification/tests/filter_tender.py - - mv .coverage .coverage.filter + - mv .coverage .coverage_filter - bin/nosetests openprocurement/bot/identification/tests/edr_handler.py - - mv .coverage .coverage.edr + - mv .coverage .coverage_edr - bin/nosetests openprocurement/bot/identification/tests/upload_file.py - - mv .coverage .coverage.upload + - mv .coverage .coverage_upload + - mv .coverage_bridge .coverage.bridge + - mv .coverage_scanner .coverage.scanner + - mv .coverage_filter .coverage.filter + - mv .coverage_edr .coverage.edr + - mv .coverage_upload .coverage.upload - bin/coverage combine after_success: - bin/coveralls From ff94ace7f1c180af70309900255e5e34107a4cdb Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 13:01:15 +0300 Subject: [PATCH 11/14] Removed unused import --- openprocurement/bot/identification/databridge/upload_file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index d09ceff..5e9ce10 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -9,7 +9,6 @@ from gevent import Greenlet, spawn from gevent.hub import LoopExit from restkit import ResourceError -from simplejson import loads from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, RetryException from openprocurement.bot.identification.databridge.journal_msg_ids import ( From 4e4fb3a5a964c13f084eb6f1ec859d1191901ba2 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Wed, 24 May 2017 18:26:51 +0300 Subject: [PATCH 12/14] Fixed tests --- .../identification/databridge/upload_file.py | 1 - .../bot/identification/tests/upload_file.py | 52 ++++++++++++------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index 5e9ce10..6c50d38 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -141,7 +141,6 @@ def upload_to_tender(self): while not self.exit: try: tender_data = self.upload_to_tender_queue.get() - gevent.sleep(20) except LoopExit: gevent.sleep(0) continue diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index d01313e..38a1b09 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -196,9 +196,10 @@ def test_request_failed(self, mrequest, gevent_sleep): @requests_mock.Mocker() @patch('gevent.sleep') - def test_request_failed_qualification_status_change(self, mrequest, gevent_sleep): + def test_request_failed_item_status_change(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}' + err_message = ['{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}', + '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}'] doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post('{url}'.format(url=doc_service_client.url), json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', @@ -207,25 +208,28 @@ def test_request_failed_qualification_status_change(self, mrequest, gevent_sleep 'title': file_name}}, status_code=200) client = MagicMock() - client._create_tender_resource_item.side_effect = ResourceError(http_code=403, msg=err_message) + client._create_tender_resource_item.side_effect = [ResourceError(http_code=403, msg=err_message[0]), + ResourceError(http_code=403, msg=err_message[1])] tender_id = uuid.uuid4().hex award_id = uuid.uuid4().hex + qualification_id = uuid.uuid4().hex document_id = generate_doc_id() - key = '{}_{}'.format(tender_id, award_id) - processing_items = {key: 1} + keys = ['{}_{}'.format(tender_id, award_id), '{}_{}'.format(tender_id, qualification_id)] + processing_items = {keys[0]: 1, keys[1]: 1} upload_to_doc_service_queue = Queue(10) upload_to_tender_queue = Queue(10) - upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'qualifications', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + upload_to_doc_service_queue.put(Data(tender_id, qualification_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) sleep(10) worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(processing_items, {}) - self.assertEqual(client._create_tender_resource_item.call_count, 1) # check that processed just 1 request + self.assertEqual(client._create_tender_resource_item.call_count, 2) # check that processed just 1 request @requests_mock.Mocker() @patch('gevent.sleep') @@ -269,9 +273,10 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): @requests_mock.Mocker() @patch('gevent.sleep') - def test_request_failed_in_retry_status(self, mrequest, gevent_sleep): + def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - err_message = '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}' + err_message = ['{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}', + '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}'] doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post('{url}'.format(url=doc_service_client.url), json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', @@ -281,29 +286,38 @@ def test_request_failed_in_retry_status(self, mrequest, gevent_sleep): status_code=200) client = MagicMock() client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message), - ResourceError(http_code=403, msg=err_message)] + ResourceError(http_code=403, msg=err_message[0]), + ResourceError(http_code=403, msg=err_message[0]), + ResourceError(http_code=403, msg=err_message[0]), + ResourceError(http_code=403, msg=err_message[0]), + ResourceError(http_code=403, msg=err_message[0]), + # ResourceError(http_code=425, msg='lol'), + ResourceError(http_code=403, msg=err_message[1]), + ResourceError(http_code=403, msg=err_message[1]), + ResourceError(http_code=403, msg=err_message[1]), + ResourceError(http_code=403, msg=err_message[1]), + ResourceError(http_code=403, msg=err_message[1]), + ] tender_id = uuid.uuid4().hex award_id = uuid.uuid4().hex + qualification_id = uuid.uuid4().hex document_id = generate_doc_id() - key = '{}_{}'.format(tender_id, award_id) - processing_items = {key: 1} + keys = ['{}_{}'.format(tender_id, award_id), '{}_{}'.format(tender_id, qualification_id)] + processing_items = {keys[0]: 1, keys[1]: 1} upload_to_doc_service_queue = Queue(10) upload_to_tender_queue = Queue(10) upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) + upload_to_doc_service_queue.put(Data(tender_id, qualification_id, '123', 'qualifications', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) sleep(60) worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') - self.assertEqual(mrequest.call_count, 1) + self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(processing_items, {}) - self.assertEqual(client._create_tender_resource_item.call_count, 6) # check that processed just 1 request + self.assertEqual(client._create_tender_resource_item.call_count, 7) # check that processed just 1 request @requests_mock.Mocker() @patch('gevent.sleep') From b449fa039e4b563a54160ec7cc412dbc455d2fbf Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Thu, 25 May 2017 11:47:24 +0300 Subject: [PATCH 13/14] Broadened the 403 case to include all possible messages --- .travis.yml | 17 +--- .../identification/databridge/constants.py | 2 - .../identification/databridge/upload_file.py | 39 ++++++--- .../bot/identification/tests/upload_file.py | 84 +++++++++++-------- 4 files changed, 78 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2773a42..19c6f7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,22 +13,7 @@ before_install: install: - bin/buildout -N script: - - bin/nosetests openprocurement/bot/identification/tests/bridge.py - - mv .coverage .coverage_bridge - - bin/nosetests openprocurement/bot/identification/tests/scanner.py - - mv .coverage .coverage_scanner - - bin/nosetests openprocurement/bot/identification/tests/filter_tender.py - - mv .coverage .coverage_filter - - bin/nosetests openprocurement/bot/identification/tests/edr_handler.py - - mv .coverage .coverage_edr - - bin/nosetests openprocurement/bot/identification/tests/upload_file.py - - mv .coverage .coverage_upload - - mv .coverage_bridge .coverage.bridge - - mv .coverage_scanner .coverage.scanner - - mv .coverage_filter .coverage.filter - - mv .coverage_edr .coverage.edr - - mv .coverage_upload .coverage.upload - - bin/coverage combine + - bin/nosetests after_success: - bin/coveralls diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index 4f0eae9..033116d 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,5 +4,3 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -upload_file_error_messages = ["Can\'t add document in current qualification status", - "Can\'t add document in current (active) award status"] diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index 6c50d38..cffdf8c 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -16,7 +16,7 @@ DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER, DATABRIDGE_START_UPLOAD, DATABRIDGE_422_UPLOAD_TO_TENDER, DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING) -from openprocurement.bot.identification.databridge.constants import file_name, upload_file_error_messages +from openprocurement.bot.identification.databridge.constants import file_name logger = logging.getLogger(__name__) @@ -52,7 +52,7 @@ def upload_to_doc_service(self): object to upload_file_to_tender, otherwise put Data to retry_upload_file_queue.""" while not self.exit: try: - tender_data = self.upload_to_doc_service_queue.get() + tender_data = self.upload_to_doc_service_queue.peek() document_id = tender_data.file_content.get('meta', {}).get('id') except LoopExit: gevent.sleep(0) @@ -69,11 +69,13 @@ def upload_to_doc_service(self): "ITEM_ID": tender_data.item_id, "DOCUMENT_ID": document_id})) logger.exception(e) self.retry_upload_to_doc_service_queue.put(tender_data) + self.upload_to_doc_service_queue.get() else: if response.status_code == 200: data = Data(tender_data.tender_id, tender_data.item_id, tender_data.code, tender_data.item_name, tender_data.edr_ids, dict(response.json(), **{'meta': {'id': document_id}})) self.upload_to_tender_queue.put(data) + self.upload_to_doc_service_queue.get() logger.info('Successfully uploaded file to doc service {} {} {} {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE}, @@ -85,6 +87,7 @@ def upload_to_doc_service(self): params={"TENDER_ID": tender_data.tender_id, "ITEM_ID": tender_data.item_id, "DOCUMENT_ID": document_id})) self.retry_upload_to_doc_service_queue.put(tender_data) + self.upload_to_doc_service_queue.get() gevent.sleep(0) def retry_upload_to_doc_service(self): @@ -92,7 +95,7 @@ def retry_upload_to_doc_service(self): upload_to_tender_queue, otherwise put Data obj back to retry_upload_file_queue""" while not self.exit: try: - tender_data = self.retry_upload_to_doc_service_queue.get() + tender_data = self.retry_upload_to_doc_service_queue.peek() document_id = tender_data.file_content.get('meta', {}).get('id') except LoopExit: gevent.sleep(0) @@ -109,12 +112,15 @@ def retry_upload_to_doc_service(self): params={"TENDER_ID": tender_data.tender_id, "ITEM_ID": tender_data.item_id, "DOCUMENT_ID": document_id})) logger.exception(e) + self.retry_upload_to_doc_service_queue.get() + self.update_processing_items(tender_data.tender_id, tender_data.item_id) raise e else: if response.status_code == 200: data = Data(tender_data.tender_id, tender_data.item_id, tender_data.code, tender_data.item_name, tender_data.edr_ids, dict(response.json(), **{'meta': {'id': document_id}})) self.upload_to_tender_queue.put(data) + self.retry_upload_to_doc_service_queue.get() logger.info('Successfully uploaded file to doc service {} {} {} {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE}, @@ -125,7 +131,7 @@ def retry_upload_to_doc_service(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE}, params={"TENDER_ID": tender_data.tender_id, "ITEM_ID": tender_data.item_id, "DOCUMENT_ID": document_id})) - self.retry_upload_to_doc_service_queue.put(tender_data) + # self.retry_upload_to_doc_service_queue.put(tender_data) gevent.sleep(0) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) @@ -140,7 +146,7 @@ def upload_to_tender(self): award/qualification from processing_items.""" while not self.exit: try: - tender_data = self.upload_to_tender_queue.get() + tender_data = self.upload_to_tender_queue.peek() except LoopExit: gevent.sleep(0) continue @@ -161,22 +167,26 @@ def upload_to_tender(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.upload_to_tender_queue.get() continue - elif re.status_int == 403 and re.msg and any([i in re.msg for i in upload_file_error_messages]): + elif re.status_int == 403: logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id}) ) self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.upload_to_tender_queue.get() else: self.retry_upload_to_tender_queue.put(tender_data) + self.upload_to_tender_queue.get() except Exception as e: logger.info('Exception while uploading file to tender {} {} {} {}. Message: {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, e.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.retry_upload_to_tender_queue.put(tender_data) + self.upload_to_tender_queue.get() else: logger.info('Successfully uploaded file to tender {} {} {} {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), @@ -184,14 +194,18 @@ def upload_to_tender(self): params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) # delete current tender after successful upload file (to avoid reloading file) self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.upload_to_tender_queue.get() gevent.sleep(0) def retry_upload_to_tender(self): """Get data from retry_upload_to_tender_queue; If upload was unsuccessful put Data obj back to retry_upload_to_tender_queue""" while not self.exit: + logger.info("Retry iteration to tender") + # import pdb; pdb.set_trace() try: - tender_data = self.retry_upload_to_tender_queue.get() + tender_data = self.retry_upload_to_tender_queue.peek() + # import pdb; pdb.set_trace() except LoopExit: gevent.sleep(0) continue @@ -199,6 +213,7 @@ def retry_upload_to_tender(self): document_id = tender_data.file_content.get('meta', {}).get('id') self.client.headers.update({'X-Client-Request-ID': document_id}) self.client_upload_to_tender(tender_data) + # logger.info() except ResourceError as re: if re.status_int == 422: # WARNING and don't retry logger.warning("Accept 422, skip tender {} {} {} {}.".format(tender_data.tender_id, tender_data.item_name, @@ -206,28 +221,30 @@ def retry_upload_to_tender(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.retry_upload_to_tender_queue.get() continue - elif re.status_int == 403 and any([i in re.msg for i in upload_file_error_messages]): + elif re.status_int == 403: logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id}) ) self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.retry_upload_to_tender_queue.get() continue else: logger.info('Exception while retry uploading file to tender {} {} {} {}. Message: {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) - self.retry_upload_to_tender_queue.put(tender_data) + # self.retry_upload_to_tender_queue.put(tender_data) except Exception as e: logger.info('Exception while retry uploading file to tender {} {} {} {}. Message: {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, e.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) logger.exception(e) - self.retry_upload_to_tender_queue.put(tender_data) + # self.retry_upload_to_tender_queue.put(tender_data) else: logger.info('Successfully uploaded file to tender {} {} {} {} in retry'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), @@ -235,12 +252,14 @@ def retry_upload_to_tender(self): params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) # delete current tender after successful upload file (to avoid reloading file) self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.retry_upload_to_tender_queue.get() gevent.sleep(0) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def client_upload_to_tender(self, tender_data): """Process upload to tender request for retry queue objects.""" document_data = tender_data.file_content.get('data', {}) + logger.info("Trying to upload to tender") document_data["documentType"] = "registerExtract" self.client.headers.update({'X-Client-Request-ID': tender_data.file_content.get('meta', {}).get('id')}) self.client._create_tender_resource_item(munchify({'data': {'id': tender_data.tender_id}}), diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 38a1b09..22ec3e8 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -66,7 +66,9 @@ def test_successful_upload(self, mrequest, gevent_sleep): self.assertItemsEqual(processing_items.keys(), [key]) self.assertEqual(upload_to_doc_service_queue.qsize(), 1) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(4) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -109,7 +111,9 @@ def test_retry_doc_service(self, mrequest, gevent_sleep): self.assertItemsEqual(processing_items.keys(), [key]) self.assertEqual(upload_to_doc_service_queue.qsize(), 1) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(7) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -131,11 +135,11 @@ def test_retry_upload_to_tender(self, mrequest, gevent_sleep): 'title': file_name}}, status_code=200) client = MagicMock() - client._create_tender_resource_item.side_effect = [Unauthorized(http_code=403), - Unauthorized(http_code=403), - Unauthorized(http_code=403), - Unauthorized(http_code=403), - Unauthorized(http_code=403), + client._create_tender_resource_item.side_effect = [Unauthorized(http_code=401), + Unauthorized(http_code=401), + Unauthorized(http_code=401), + Unauthorized(http_code=401), + Unauthorized(http_code=401), Exception(), {'data': {'id': uuid.uuid4().hex, 'documentOf': 'tender', @@ -152,7 +156,9 @@ def test_retry_upload_to_tender(self, mrequest, gevent_sleep): self.assertItemsEqual(processing_items.keys(), [key]) self.assertEqual(upload_to_doc_service_queue.qsize(), 1) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(60) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -184,7 +190,9 @@ def test_request_failed(self, mrequest, gevent_sleep): upload_to_tender_queue = Queue(10) upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(10) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -198,8 +206,6 @@ def test_request_failed(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_request_failed_item_status_change(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - err_message = ['{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}', - '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}'] doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post('{url}'.format(url=doc_service_client.url), json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', @@ -208,8 +214,8 @@ def test_request_failed_item_status_change(self, mrequest, gevent_sleep): 'title': file_name}}, status_code=200) client = MagicMock() - client._create_tender_resource_item.side_effect = [ResourceError(http_code=403, msg=err_message[0]), - ResourceError(http_code=403, msg=err_message[1])] + client._create_tender_resource_item.side_effect = [ResourceError(http_code=403), + ResourceError(http_code=403)] tender_id = uuid.uuid4().hex award_id = uuid.uuid4().hex qualification_id = uuid.uuid4().hex @@ -221,7 +227,9 @@ def test_request_failed_item_status_change(self, mrequest, gevent_sleep): upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'qualifications', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) upload_to_doc_service_queue.put(Data(tender_id, qualification_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(10) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -261,7 +269,9 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(60) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -275,8 +285,6 @@ def test_request_failed_in_retry(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - err_message = ['{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current qualification status"}]}', - '{"status": "error", "errors": [{"location": "body", "name": "data", "description": "Can\'t add document in current (active) award status"}]}'] doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post('{url}'.format(url=doc_service_client.url), json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', @@ -286,17 +294,16 @@ def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): status_code=200) client = MagicMock() client._create_tender_resource_item.side_effect = [ResourceError(http_code=425), - ResourceError(http_code=403, msg=err_message[0]), - ResourceError(http_code=403, msg=err_message[0]), - ResourceError(http_code=403, msg=err_message[0]), - ResourceError(http_code=403, msg=err_message[0]), - ResourceError(http_code=403, msg=err_message[0]), - # ResourceError(http_code=425, msg='lol'), - ResourceError(http_code=403, msg=err_message[1]), - ResourceError(http_code=403, msg=err_message[1]), - ResourceError(http_code=403, msg=err_message[1]), - ResourceError(http_code=403, msg=err_message[1]), - ResourceError(http_code=403, msg=err_message[1]), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), + ResourceError(http_code=403), ] tender_id = uuid.uuid4().hex award_id = uuid.uuid4().hex @@ -309,7 +316,9 @@ def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) upload_to_doc_service_queue.put(Data(tender_id, qualification_id, '123', 'qualifications', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(60) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -356,7 +365,9 @@ def test_processing_items(self, mrequest, gevent_sleep): upload_to_doc_service_queue.put(Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) - sleep(10) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty worker.shutdown() self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) @@ -400,8 +411,10 @@ def test_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_sleep): processing_items = {key: 2} upload_to_doc_service_queue = MagicMock() upload_to_tender_queue = Queue(10) - upload_to_doc_service_queue.get.side_effect = generate_answers( - answers=[LoopExit(), Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'}), Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})], + upload_to_doc_service_queue.peek.side_effect = generate_answers( + answers=[LoopExit(), + Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'}), + Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})], default=LoopExit()) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) sleep(10) @@ -435,7 +448,7 @@ def test_upload_to_tender_queue_loop_exit(self, mrequest, gevent_sleep): processing_items = {key: 2} upload_to_doc_service_queue = Queue(1) upload_to_tender_queue = MagicMock() - upload_to_tender_queue.get.side_effect = generate_answers( + upload_to_tender_queue.peek.side_effect = generate_answers( answers=[LoopExit(), Data(tender_id=tender_id, item_id=award_id, @@ -490,7 +503,7 @@ def test_retry_upload_to_tender_queue_loop_exit(self, mrequest, gevent_sleep): upload_to_tender_queue = Queue(1) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) worker.retry_upload_to_tender_queue = MagicMock() - worker.retry_upload_to_tender_queue.get.side_effect = generate_answers( + worker.retry_upload_to_tender_queue.peek.side_effect = generate_answers( answers=[LoopExit(), Data(tender_id=tender_id, item_id=award_id, @@ -511,7 +524,6 @@ def test_retry_upload_to_tender_queue_loop_exit(self, mrequest, gevent_sleep): u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555', u'title': file_name})], default=LoopExit()) - sleep(10) worker.shutdown() self.assertEqual(processing_items, {}) @@ -557,7 +569,7 @@ def test_retry_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_slee upload_to_doc_service_queue = Queue(1) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, doc_service_client) worker.retry_upload_to_doc_service_queue = MagicMock() - worker.retry_upload_to_doc_service_queue.get.side_effect = generate_answers( + worker.retry_upload_to_doc_service_queue.peek.side_effect = generate_answers( answers=[LoopExit(), Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'}), Data(tender_id, award_id, '123', 'awards', None, {'meta': {'id': document_id}, 'test_data': 'test_data'})], From 85a37382b2a6cb60a71dd39ac4beccafdd1aee10 Mon Sep 17 00:00:00 2001 From: dtrenkenshu Date: Thu, 25 May 2017 12:07:18 +0300 Subject: [PATCH 14/14] Fixing mistakes --- .travis.yml | 2 +- .../identification/databridge/upload_file.py | 30 ++++++++----------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 19c6f7d..3a6b5c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ before_install: install: - bin/buildout -N script: - - bin/nosetests + - bin/nosetests after_success: - bin/coveralls diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index cffdf8c..1be9f39 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -10,7 +10,7 @@ from gevent.hub import LoopExit from restkit import ResourceError -from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, RetryException +from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file from openprocurement.bot.identification.databridge.journal_msg_ids import ( DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, @@ -131,7 +131,6 @@ def retry_upload_to_doc_service(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE}, params={"TENDER_ID": tender_data.tender_id, "ITEM_ID": tender_data.item_id, "DOCUMENT_ID": document_id})) - # self.retry_upload_to_doc_service_queue.put(tender_data) gevent.sleep(0) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) @@ -161,23 +160,27 @@ def upload_to_tender(self): tender_data.item_id)) except ResourceError as re: if re.status_int == 422: # WARNING and don't retry - logger.warning("Accept 422, skip tender {} {} {} {}.".format(tender_data.tender_id, + logger.warning("Accept 422, skip tender {} {} {} {}. Message: {}".format(tender_data.tender_id, tender_data.item_name, - tender_data.item_id, document_id), + tender_data.item_id, document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) self.upload_to_tender_queue.get() continue elif re.status_int == 403: - logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), + logger.warning("Accept 403 while uploading to tender {} {} {} doc_id: {}. Message {}".format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id}) ) self.update_processing_items(tender_data.tender_id, tender_data.item_id) self.upload_to_tender_queue.get() else: + logger.warning('Exception while retry uploading file to tender {} {} {} {}. Message: {}'.format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.message), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, + params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.retry_upload_to_tender_queue.put(tender_data) self.upload_to_tender_queue.get() except Exception as e: @@ -201,11 +204,8 @@ def retry_upload_to_tender(self): """Get data from retry_upload_to_tender_queue; If upload was unsuccessful put Data obj back to retry_upload_to_tender_queue""" while not self.exit: - logger.info("Retry iteration to tender") - # import pdb; pdb.set_trace() try: tender_data = self.retry_upload_to_tender_queue.peek() - # import pdb; pdb.set_trace() except LoopExit: gevent.sleep(0) continue @@ -213,19 +213,18 @@ def retry_upload_to_tender(self): document_id = tender_data.file_content.get('meta', {}).get('id') self.client.headers.update({'X-Client-Request-ID': document_id}) self.client_upload_to_tender(tender_data) - # logger.info() except ResourceError as re: if re.status_int == 422: # WARNING and don't retry - logger.warning("Accept 422, skip tender {} {} {} {}.".format(tender_data.tender_id, tender_data.item_name, - tender_data.item_id, document_id), + logger.warning("Accept 422, skip tender {} {} {} {}. Message {}".format(tender_data.tender_id, tender_data.item_name, + tender_data.item_id, document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) self.update_processing_items(tender_data.tender_id, tender_data.item_id) self.retry_upload_to_tender_queue.get() continue elif re.status_int == 403: - logger.warning("Tender {} {} {} doc_id: {} changed status while processing and is no longer a valid target for the upload".format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), + logger.warning("Accept 403 while uploading to tender {} {} {} doc_id: {}. Message {}".format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id}) ) @@ -237,14 +236,12 @@ def retry_upload_to_tender(self): tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) - # self.retry_upload_to_tender_queue.put(tender_data) except Exception as e: logger.info('Exception while retry uploading file to tender {} {} {} {}. Message: {}'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, e.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, params={"TENDER_ID": tender_data.tender_id, "DOCUMENT_ID": document_id})) logger.exception(e) - # self.retry_upload_to_tender_queue.put(tender_data) else: logger.info('Successfully uploaded file to tender {} {} {} {} in retry'.format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id), @@ -259,7 +256,6 @@ def retry_upload_to_tender(self): def client_upload_to_tender(self, tender_data): """Process upload to tender request for retry queue objects.""" document_data = tender_data.file_content.get('data', {}) - logger.info("Trying to upload to tender") document_data["documentType"] = "registerExtract" self.client.headers.update({'X-Client-Request-ID': tender_data.file_content.get('meta', {}).get('id')}) self.client._create_tender_resource_item(munchify({'data': {'id': tender_data.tender_id}}),