From 6175000e4958577db76e32c12db585b935b2f764 Mon Sep 17 00:00:00 2001 From: Thiago Salles Date: Thu, 2 Jun 2022 17:26:30 -0300 Subject: [PATCH 1/5] Include ASDF .tool-versions file to support the tool usage --- .tool-versions | 1 + 1 file changed, 1 insertion(+) create mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..605f473 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +python latest:3.7 From 8c0d3c687180056a0815dd444c0e9a7e5c57e04a Mon Sep 17 00:00:00 2001 From: Thiago Salles Date: Thu, 2 Jun 2022 17:39:27 -0300 Subject: [PATCH 2/5] Add the ability to check extra modules on healthcheck --- README.md | 46 ++++++++ barterdude/hooks/healthcheck.py | 33 +++++- tests_unit/test_hooks/test_healthcheck.py | 138 ++++++++++++++++++++-- 3 files changed, 202 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 341cbf0..d784b0a 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,52 @@ async def consumer_access_storage(msg): data = baterdude["my_variable"] ``` +### Monitoring extra modules on Healthcheck + +If you run extra modules on your application, like workers or services, you can include them in the healthcheck. + +First, you need to update your module to implement the interface `HealthcheckMonitored`: +```python +from barterdude.hooks.healthcheck import HealthcheckMonitored + +class ExtraService(HealthcheckMonitored): +``` + +Implementing that interface will require the definition of the method `healthcheck` in your module. It should return a boolean value indicating if your module is healhty or not: +```python + def healthcheck(self): + return self._thread.is_alive() +``` + +Finally, you need to make the BarterDude and Healthcheck module be aware of your module. To do so, you'll use the Data Sharing feature: +```python +from barterdude import BarterDude +from app.extra_service import ExtraService + +barterdude = BarterDude() +barterdude["extra_service"] = ExtraService() +``` + +If you are already running your extra modules on BartedDude startup using the data shareing model, it's all done: +```python +@app.run_on_startup +async def startup(app): + app["client_session"] = ClientSession() + app["extra_service"] = ExtraService() +``` + +The healthcheck module will identify all shared modules that implement the interface `HealthcheckMonitored` and run its healthcheck method automatically. +The result of all monitored modules will be included in the result body of the healthcheck endpoint and if any of the modules fail, the healthcheck endpoint will indicate that: +```json +{ + "extra_service": "ok", + "message": "Success rate: 1.0 (expected: 0.9)", + "fail": 0, + "success": 1, + "status": "ok" +} +``` + ### Schema Validation Consumed messages can be validated by json schema: diff --git a/barterdude/hooks/healthcheck.py b/barterdude/hooks/healthcheck.py index 172cee2..4b12bdb 100644 --- a/barterdude/hooks/healthcheck.py +++ b/barterdude/hooks/healthcheck.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod import json from barterdude import BarterDude @@ -20,6 +21,10 @@ def _response(status, body): body["status"] = "ok" if status == 200 else "fail" return web.Response(status=status, body=json.dumps(body)) +class HealthcheckMonitored(ABC): + @abstractmethod + def healthcheck(self): + pass class Healthcheck(HttpHook): def __init__( @@ -30,6 +35,7 @@ def __init__( health_window: float = 60.0, # seconds max_connection_fails: int = 3 ): + self.__barterdude = barterdude self.__success_rate = success_rate self.__health_window = health_window self.__success = deque() @@ -68,18 +74,37 @@ async def __call__(self, req: web.Request): ) }) + response = {} + status = 200 + + all_monitored_modules_passed = True + for module in self.__barterdude: + if isinstance(self.__barterdude[module], HealthcheckMonitored): + passed = self.__barterdude[module].healthcheck() + response[module] = "ok" if passed else "fail" + all_monitored_modules_passed &= passed + + if not all_monitored_modules_passed: + status = 500 + old_timestamp = time() - self.__health_window success = _remove_old(self.__success, old_timestamp) fail = _remove_old(self.__fail, old_timestamp) + if success == 0 and fail == 0: - return _response(200, { - "message": f"No messages in last {self.__health_window}s" - }) + response["message"] = f"No messages in last {self.__health_window}s" + return _response(status, response) rate = success / (success + fail) - return _response(200 if rate >= self.__success_rate else 500, { + + if rate < self.__success_rate: + status = 500 + + response.update({ "message": f"Success rate: {rate} (expected: {self.__success_rate})", "fail": fail, "success": success }) + + return _response(status, response) diff --git a/tests_unit/test_hooks/test_healthcheck.py b/tests_unit/test_hooks/test_healthcheck.py index b8114df..1710062 100644 --- a/tests_unit/test_hooks/test_healthcheck.py +++ b/tests_unit/test_hooks/test_healthcheck.py @@ -1,6 +1,13 @@ -from asynctest import TestCase, Mock +from asynctest import TestCase, MagicMock from freezegun import freeze_time -from barterdude.hooks.healthcheck import Healthcheck +from barterdude.hooks.healthcheck import Healthcheck, HealthcheckMonitored + + +class HealthcheckMonitoredMock(HealthcheckMonitored): + def __init__(self, healthy=True): + self.healthy = healthy + def healthcheck(self): + return self.healthy @freeze_time() @@ -10,8 +17,12 @@ class TestHealthcheck(TestCase): def setUp(self): self.success_rate = 0.9 self.health_window = 60.0 + self.app = MagicMock() + self.monitoredModules = {} + self.app.__iter__.side_effect = lambda: iter(self.monitoredModules) + self.app.__getitem__.side_effect = lambda module: self.monitoredModules[module] self.healthcheck = Healthcheck( - Mock(), + self.app, "/healthcheck", self.success_rate, self.health_window @@ -21,7 +32,7 @@ async def test_should_call_before_consume(self): await self.healthcheck.before_consume(None) async def test_should_pass_healthcheck_when_no_messages(self): - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual(response.status, 200) self.assertEqual(response.content_type, "text/plain") self.assertEqual( @@ -31,7 +42,7 @@ async def test_should_pass_healthcheck_when_no_messages(self): async def test_should_pass_healthcheck_when_only_sucess(self): await self.healthcheck.on_success(None) - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual(response.status, 200) self.assertEqual(response.content_type, "text/plain") self.assertEqual( @@ -44,7 +55,7 @@ async def test_should_pass_healthcheck_when_success_rate_is_high(self): await self.healthcheck.on_fail(None, None) for i in range(0, 9): await self.healthcheck.on_success(None) - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual(response.status, 200) self.assertEqual(response.content_type, "text/plain") self.assertEqual( @@ -55,7 +66,7 @@ async def test_should_pass_healthcheck_when_success_rate_is_high(self): async def test_should_fail_healthcheck_when_only_fail(self): await self.healthcheck.on_fail(None, None) - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual(response.status, 500) self.assertEqual(response.content_type, "text/plain") self.assertEqual( @@ -67,7 +78,7 @@ async def test_should_fail_healthcheck_when_only_fail(self): async def test_should_fail_healthcheck_when_success_rate_is_low(self): await self.healthcheck.on_success(None) await self.healthcheck.on_fail(None, None) - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual(response.status, 500) self.assertEqual(response.content_type, "text/plain") self.assertEqual( @@ -79,7 +90,7 @@ async def test_should_fail_healthcheck_when_success_rate_is_low(self): async def test_should_fail_when_force_fail_is_called(self): self.healthcheck.force_fail() await self.healthcheck.on_success(None) - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual(response.status, 500) self.assertEqual(response.content_type, "text/plain") self.assertEqual( @@ -93,7 +104,7 @@ async def test_should_erase_old_messages(self): for i in range(0, 10): await self.healthcheck.on_fail(None, None) await self.healthcheck.on_success(None) - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual( response.body._value.decode('utf-8'), '{"message": "Success rate: 0.125 (expected: 0.9)", ' @@ -102,8 +113,113 @@ async def test_should_erase_old_messages(self): async def test_should_fail_healthcheck_when_fail_to_connect(self): await self.healthcheck.on_connection_fail(None, 3) - response = await self.healthcheck(Mock()) + response = await self.healthcheck(self.app) self.assertEqual( response.body._value.decode('utf-8'), '{"message": "Reached max connection fails (3)", "status": "fail"}' ) + + async def test_should_pass_healthcheck_when_has_one_healthy_monitored_module(self): + self.monitoredModules = { + "testModule": HealthcheckMonitoredMock(True) + } + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 200) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"testModule": "ok", "message": "No messages in last 60.0s", "status": "ok"}' + ) + + async def test_should_pass_healthcheck_when_has_two_healthy_monitored_module(self): + self.monitoredModules = { + "testModule1": HealthcheckMonitoredMock(True), + "testModule2": HealthcheckMonitoredMock(True) + } + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 200) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"testModule1": "ok", "testModule2": "ok", "message": "No messages in last 60.0s", "status": "ok"}' + ) + + async def test_should_fail_healthcheck_when_has_one_failing_monitored_module(self): + self.monitoredModules = { + "testModule": HealthcheckMonitoredMock(False) + } + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 500) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"testModule": "fail", "message": "No messages in last 60.0s", "status": "fail"}' + ) + + async def test_should_fail_healthcheck_when_has_two_failing_monitored_module(self): + self.monitoredModules = { + "testModule1": HealthcheckMonitoredMock(False), + "testModule2": HealthcheckMonitoredMock(False) + } + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 500) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"testModule1": "fail", "testModule2": "fail", "message": "No messages in last 60.0s", "status": "fail"}' + ) + + async def test_should_fail_healthcheck_when_has_failing_and_healthy_monitored_modules(self): + self.monitoredModules = { + "testModule1": HealthcheckMonitoredMock(False), + "testModule2": HealthcheckMonitoredMock(True) + } + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 500) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"testModule1": "fail", "testModule2": "ok", "message": "No messages in last 60.0s", "status": "fail"}' + ) + + async def test_should_pass_healthcheck_when_has_one_healthy_monitored_module_and_messages(self): + self.monitoredModules = { + "testModule": HealthcheckMonitoredMock(True) + } + await self.healthcheck.on_success(None) + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 200) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"testModule": "ok", "message": "Success rate: 1.0 (expected: 0.9)", ' + '"fail": 0, "success": 1, "status": "ok"}' + ) + + async def test_should_pass_healthcheck_when_has_one_failing_monitored_module_and_messages(self): + self.monitoredModules = { + "testModule": HealthcheckMonitoredMock(False) + } + await self.healthcheck.on_success(None) + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 500) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"testModule": "fail", "message": "Success rate: 1.0 (expected: 0.9)", ' + '"fail": 0, "success": 1, "status": "fail"}' + ) + + async def test_should_pass_healthcheck_when_has_simple_module(self): + self.monitoredModules = { + "testModule": MagicMock() + } + await self.healthcheck.on_success(None) + response = await self.healthcheck(self.app) + self.assertEqual(response.status, 200) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual( + response.body._value.decode('utf-8'), + '{"message": "Success rate: 1.0 (expected: 0.9)", ' + '"fail": 0, "success": 1, "status": "ok"}' + ) From 1edd1ccc9072a875bad78a7a2657cd6703c956fa Mon Sep 17 00:00:00 2001 From: Thiago Salles Date: Thu, 2 Jun 2022 18:17:39 -0300 Subject: [PATCH 3/5] Fix linter problems --- barterdude/hooks/healthcheck.py | 6 +++- tests_unit/test_hooks/test_healthcheck.py | 40 ++++++++++++++--------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/barterdude/hooks/healthcheck.py b/barterdude/hooks/healthcheck.py index 4b12bdb..843f834 100644 --- a/barterdude/hooks/healthcheck.py +++ b/barterdude/hooks/healthcheck.py @@ -21,11 +21,13 @@ def _response(status, body): body["status"] = "ok" if status == 200 else "fail" return web.Response(status=status, body=json.dumps(body)) + class HealthcheckMonitored(ABC): @abstractmethod def healthcheck(self): pass + class Healthcheck(HttpHook): def __init__( self, @@ -92,7 +94,9 @@ async def __call__(self, req: web.Request): fail = _remove_old(self.__fail, old_timestamp) if success == 0 and fail == 0: - response["message"] = f"No messages in last {self.__health_window}s" + response["message"] = ( + f"No messages in last {self.__health_window}s" + ) return _response(status, response) rate = success / (success + fail) diff --git a/tests_unit/test_hooks/test_healthcheck.py b/tests_unit/test_hooks/test_healthcheck.py index 1710062..eb8d595 100644 --- a/tests_unit/test_hooks/test_healthcheck.py +++ b/tests_unit/test_hooks/test_healthcheck.py @@ -6,6 +6,7 @@ class HealthcheckMonitoredMock(HealthcheckMonitored): def __init__(self, healthy=True): self.healthy = healthy + def healthcheck(self): return self.healthy @@ -20,7 +21,9 @@ def setUp(self): self.app = MagicMock() self.monitoredModules = {} self.app.__iter__.side_effect = lambda: iter(self.monitoredModules) - self.app.__getitem__.side_effect = lambda module: self.monitoredModules[module] + self.app.__getitem__.side_effect = ( + lambda module: self.monitoredModules[module] + ) self.healthcheck = Healthcheck( self.app, "/healthcheck", @@ -119,7 +122,7 @@ async def test_should_fail_healthcheck_when_fail_to_connect(self): '{"message": "Reached max connection fails (3)", "status": "fail"}' ) - async def test_should_pass_healthcheck_when_has_one_healthy_monitored_module(self): + async def test_should_pass_when_has_one_healthy_monitored_module(self): self.monitoredModules = { "testModule": HealthcheckMonitoredMock(True) } @@ -128,10 +131,11 @@ async def test_should_pass_healthcheck_when_has_one_healthy_monitored_module(sel self.assertEqual(response.content_type, "text/plain") self.assertEqual( response.body._value.decode('utf-8'), - '{"testModule": "ok", "message": "No messages in last 60.0s", "status": "ok"}' + '{"testModule": "ok", ' + '"message": "No messages in last 60.0s", "status": "ok"}' ) - async def test_should_pass_healthcheck_when_has_two_healthy_monitored_module(self): + async def test_should_pass_when_has_two_healthy_monitored_module(self): self.monitoredModules = { "testModule1": HealthcheckMonitoredMock(True), "testModule2": HealthcheckMonitoredMock(True) @@ -141,10 +145,11 @@ async def test_should_pass_healthcheck_when_has_two_healthy_monitored_module(sel self.assertEqual(response.content_type, "text/plain") self.assertEqual( response.body._value.decode('utf-8'), - '{"testModule1": "ok", "testModule2": "ok", "message": "No messages in last 60.0s", "status": "ok"}' + '{"testModule1": "ok", "testModule2": "ok", ' + '"message": "No messages in last 60.0s", "status": "ok"}' ) - async def test_should_fail_healthcheck_when_has_one_failing_monitored_module(self): + async def test_should_fail_when_has_one_failing_monitored_module(self): self.monitoredModules = { "testModule": HealthcheckMonitoredMock(False) } @@ -153,10 +158,11 @@ async def test_should_fail_healthcheck_when_has_one_failing_monitored_module(sel self.assertEqual(response.content_type, "text/plain") self.assertEqual( response.body._value.decode('utf-8'), - '{"testModule": "fail", "message": "No messages in last 60.0s", "status": "fail"}' + '{"testModule": "fail", ' + '"message": "No messages in last 60.0s", "status": "fail"}' ) - async def test_should_fail_healthcheck_when_has_two_failing_monitored_module(self): + async def test_should_fail_when_has_two_failing_monitored_module(self): self.monitoredModules = { "testModule1": HealthcheckMonitoredMock(False), "testModule2": HealthcheckMonitoredMock(False) @@ -166,10 +172,11 @@ async def test_should_fail_healthcheck_when_has_two_failing_monitored_module(sel self.assertEqual(response.content_type, "text/plain") self.assertEqual( response.body._value.decode('utf-8'), - '{"testModule1": "fail", "testModule2": "fail", "message": "No messages in last 60.0s", "status": "fail"}' + '{"testModule1": "fail", "testModule2": "fail", ' + '"message": "No messages in last 60.0s", "status": "fail"}' ) - async def test_should_fail_healthcheck_when_has_failing_and_healthy_monitored_modules(self): + async def test_should_fail_when_has_failing_and_healthy_modules(self): self.monitoredModules = { "testModule1": HealthcheckMonitoredMock(False), "testModule2": HealthcheckMonitoredMock(True) @@ -179,10 +186,11 @@ async def test_should_fail_healthcheck_when_has_failing_and_healthy_monitored_mo self.assertEqual(response.content_type, "text/plain") self.assertEqual( response.body._value.decode('utf-8'), - '{"testModule1": "fail", "testModule2": "ok", "message": "No messages in last 60.0s", "status": "fail"}' + '{"testModule1": "fail", "testModule2": "ok", ' + '"message": "No messages in last 60.0s", "status": "fail"}' ) - async def test_should_pass_healthcheck_when_has_one_healthy_monitored_module_and_messages(self): + async def test_should_pass_when_has_one_healthy_module_and_messages(self): self.monitoredModules = { "testModule": HealthcheckMonitoredMock(True) } @@ -192,11 +200,12 @@ async def test_should_pass_healthcheck_when_has_one_healthy_monitored_module_and self.assertEqual(response.content_type, "text/plain") self.assertEqual( response.body._value.decode('utf-8'), - '{"testModule": "ok", "message": "Success rate: 1.0 (expected: 0.9)", ' + '{"testModule": "ok", ' + '"message": "Success rate: 1.0 (expected: 0.9)", ' '"fail": 0, "success": 1, "status": "ok"}' ) - async def test_should_pass_healthcheck_when_has_one_failing_monitored_module_and_messages(self): + async def test_should_pass_when_has_one_failing_module_and_messages(self): self.monitoredModules = { "testModule": HealthcheckMonitoredMock(False) } @@ -206,7 +215,8 @@ async def test_should_pass_healthcheck_when_has_one_failing_monitored_module_and self.assertEqual(response.content_type, "text/plain") self.assertEqual( response.body._value.decode('utf-8'), - '{"testModule": "fail", "message": "Success rate: 1.0 (expected: 0.9)", ' + '{"testModule": "fail", ' + '"message": "Success rate: 1.0 (expected: 0.9)", ' '"fail": 0, "success": 1, "status": "fail"}' ) From 4068c4677a1183f9c2ae117934a2f1eee3999e26 Mon Sep 17 00:00:00 2001 From: Thiago Salles Date: Thu, 2 Jun 2022 18:22:13 -0300 Subject: [PATCH 4/5] Fix the python version in .tool-versions to avoid ASDF problems --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 605f473..4896bb9 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -python latest:3.7 +python 3.7.13 From 70a4f3d2f97e52ae169e67575f883083ef1abfec Mon Sep 17 00:00:00 2001 From: Thiago Salles Date: Fri, 3 Jun 2022 15:17:06 -0300 Subject: [PATCH 5/5] Fix typo on README.md Co-authored-by: Felipe Ribeiro --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d784b0a..bf0d8ab 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ barterdude = BarterDude() barterdude["extra_service"] = ExtraService() ``` -If you are already running your extra modules on BartedDude startup using the data shareing model, it's all done: +If you are already running your extra modules on BartedDude startup using the data sharing model, it's all done: ```python @app.run_on_startup async def startup(app):