From 60c68adba305b5ae00a17a9aaa83d5b113631b48 Mon Sep 17 00:00:00 2001 From: Bart Veraart Date: Thu, 31 Dec 2020 16:15:52 +0100 Subject: [PATCH 1/3] OPT: update readme with some comments about starting elasticsearch --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index ecd5b90..8bcec74 100644 --- a/README.rst +++ b/README.rst @@ -354,6 +354,13 @@ There is a Makefile with a few targets that we use often: ``make test`` command will run tests for the python versions specified in ``tox.ini`` spinning up all necessary services via docker. +In some cases (on Ubuntu 18.04) the Elasticsearch Docker image might not be able to start and will exit with the following error:: + + max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144] + +This can be solved by adding the following line to `/etc/sysctl.conf`:: + + vm.max_map_count=262144 .. _Grafana: http://grafana.org/ .. _Kibana: https://www.elastic.co/products/kibana From e92ebeffb090f7055a8a420c799b58fa0fce4a81 Mon Sep 17 00:00:00 2001 From: Bart Veraart Date: Mon, 4 Jan 2021 13:02:31 +0100 Subject: [PATCH 2/3] OPT: Speed up error_resilience test by decreasing max_retries --- tests/test_elasticsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elasticsearch.py b/tests/test_elasticsearch.py index c588f59..8de99d9 100644 --- a/tests/test_elasticsearch.py +++ b/tests/test_elasticsearch.py @@ -17,7 +17,7 @@ class TestConnectionErrors(TestBaseBackend): @mock.patch("time_execution.backends.elasticsearch.logger") def test_error_resilience(self, mocked_logger): - backend = ElasticsearchBackend(hosts=["non-existant-domain"]) + backend = ElasticsearchBackend(hosts=["non-existent-domain"], max_retries=1) # ensure index and mapping setup failures are caught and logged self.assertEqual(2, len(mocked_logger.error.call_args_list)) # ensure write failure is caught and logged From 6faa1fa09c11ba9e99cf892ba94ad87eb2e5a2bd Mon Sep 17 00:00:00 2001 From: Bart Veraart Date: Mon, 4 Jan 2021 13:30:17 +0100 Subject: [PATCH 3/3] NEW: flag to skip index creation on class instantiation --- tests/test_elasticsearch.py | 7 ++++++ time_execution/backends/elasticsearch.py | 27 +++++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/test_elasticsearch.py b/tests/test_elasticsearch.py index 8de99d9..c2d3c29 100644 --- a/tests/test_elasticsearch.py +++ b/tests/test_elasticsearch.py @@ -145,3 +145,10 @@ def test_bulk_write_error(self, mocked_logger): with es_index_error_ctx: self.backend.bulk_write(metrics) mocked_logger.warning.assert_called_once_with("bulk_write metrics %r failure %r", metrics, transport_error) + + @mock.patch("time_execution.backends.elasticsearch.ElasticsearchBackend._setup_mapping") + @mock.patch("time_execution.backends.elasticsearch.ElasticsearchBackend._setup_index") + def test_do_not_create_index(self, setup_index, setup_mapping): + ElasticsearchBackend(ELASTICSEARCH_HOST, index="unittest", create_index=False) + setup_index.assert_not_called() + setup_mapping.assert_not_called() diff --git a/time_execution/backends/elasticsearch.py b/time_execution/backends/elasticsearch.py index 5eb4997..17c14f6 100644 --- a/time_execution/backends/elasticsearch.py +++ b/time_execution/backends/elasticsearch.py @@ -11,7 +11,14 @@ class ElasticsearchBackend(BaseMetricsBackend): def __init__( - self, hosts=None, index="metrics", doc_type="metric", index_pattern="{index}-{date:%Y.%m.%d}", *args, **kwargs + self, + hosts=None, + index="metrics", + doc_type="metric", + index_pattern="{index}-{date:%Y.%m.%d}", + create_index=True, + *args, + **kwargs, ): # Assign these in the backend as they are needed when writing metrics # to elasticsearch @@ -22,15 +29,15 @@ def __init__( # setup the client self.client = Elasticsearch(hosts=hosts, *args, **kwargs) - # ensure the index is created - try: - self._setup_index() - except TransportError as exc: - logger.error("index setup error %r", exc) - try: - self._setup_mapping() - except TransportError as exc: - logger.error("mapping setup error %r", exc) + if create_index: + try: + self._setup_index() + except TransportError as exc: + logger.error("index setup error %r", exc) + try: + self._setup_mapping() + except TransportError as exc: + logger.error("mapping setup error %r", exc) def get_index(self): return self.index_pattern.format(index=self.index, date=datetime.now())