From fd95d6788746c0f46ef64dd9b56735c59faecb8b Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Wed, 24 Jul 2024 08:31:51 +0300 Subject: [PATCH] feature(elasticsearch): move to using api key switching form user/password to using api keys (tokens) also switching from using urls to using the elastic cloud_id which is what recommanded by elasticsearch cloud, and enable compression. Ref: https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/connecting.html#connect-ec Ref: https://github.com/scylladb/qa-tasks/issues/1747 (cherry picked from commit 767d199f95cde77b9e76053eef56705ce35348c2) # Conflicts: # utils/update_field.py --- pylintrc | 1 + sdcm/es.py | 11 +++++------ sdcm/keystore.py | 4 ++-- sdcm/nemesis_publisher.py | 5 ++--- sdcm/results_analyze/__init__.py | 2 +- sdcm/results_analyze/test.py | 2 +- utils/fix_es_mapping.py | 9 +++++---- utils/migrate_nemesis_data.py | 5 ++--- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/pylintrc b/pylintrc index 5961faa531..036da4bb3c 100644 --- a/pylintrc +++ b/pylintrc @@ -370,6 +370,7 @@ good-names=i, ks, cf, pytestmark, + es, # Include a hint for the correct naming format with invalid-name. include-naming-hint=no diff --git a/sdcm/es.py b/sdcm/es.py index 7a031cd4c5..e392f73520 100644 --- a/sdcm/es.py +++ b/sdcm/es.py @@ -1,4 +1,5 @@ import logging +from functools import cached_property import elasticsearch @@ -13,13 +14,11 @@ class ES(elasticsearch.Elasticsearch): """ def __init__(self): - self._conf = self.get_conf() - super().__init__(hosts=[self._conf["es_url"]], verify_certs=False, - http_auth=(self._conf["es_user"], self._conf["es_password"])) + super().__init__(**self.conf) - def get_conf(self): - self.key_store = KeyStore() - return self.key_store.get_elasticsearch_credentials() + @cached_property + def conf(self): + return KeyStore().get_elasticsearch_token() def _create_index(self, index): self.indices.create(index=index, ignore=400) # pylint: disable=unexpected-keyword-arg diff --git a/sdcm/keystore.py b/sdcm/keystore.py index 538227395c..e3c105b7fd 100644 --- a/sdcm/keystore.py +++ b/sdcm/keystore.py @@ -43,8 +43,8 @@ def download_file(self, filename, dest_filename): def get_email_credentials(self): return self.get_json("email_config.json") - def get_elasticsearch_credentials(self): - return self.get_json("es.json") + def get_elasticsearch_token(self): + return self.get_json("es_token.json") def get_gcp_credentials(self): project = os.environ.get('SCT_GCE_PROJECT') or 'gcp-sct-project-1' diff --git a/sdcm/nemesis_publisher.py b/sdcm/nemesis_publisher.py index 6abecefc73..0cce507374 100644 --- a/sdcm/nemesis_publisher.py +++ b/sdcm/nemesis_publisher.py @@ -33,9 +33,8 @@ def __init__(self, tester): def create_es_connection(self): ks = KeyStore() - es_conf = ks.get_elasticsearch_credentials() - self.es = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=False, # pylint: disable=invalid-name - http_auth=(es_conf["es_user"], es_conf["es_password"])) + es_conf = ks.get_elasticsearch_token() + self.es = Elasticsearch(**es_conf) @cached_property def stats(self): diff --git a/sdcm/results_analyze/__init__.py b/sdcm/results_analyze/__init__.py index b126857ce0..ea5e1ea557 100644 --- a/sdcm/results_analyze/__init__.py +++ b/sdcm/results_analyze/__init__.py @@ -47,7 +47,7 @@ class BaseResultsAnalyzer: # pylint: disable=too-many-instance-attributes def __init__(self, es_index, es_doc_type, email_recipients=(), email_template_fp="", query_limit=1000, logger=None, events=None): self._es = ES() - self._conf = self._es._conf # pylint: disable=protected-access + self._conf = self._es.conf # pylint: disable=protected-access self._es_index = es_index self._es_doc_type = es_doc_type self._limit = query_limit diff --git a/sdcm/results_analyze/test.py b/sdcm/results_analyze/test.py index 99dd408ff2..78c15805e7 100644 --- a/sdcm/results_analyze/test.py +++ b/sdcm/results_analyze/test.py @@ -520,7 +520,7 @@ def get_metric_class(cls, metric_path, default=__DEFAULT__): def gen_kibana_dashboard_url( dashboard_path="app/kibana#/dashboard/03414b70-0e89-11e9-a976-2fe0f5890cd0?_g=()" ): - return "%s/%s" % (ES()._conf.get('kibana_url'), dashboard_path) # pylint: disable=protected-access + return "%s/%s" % (ES().conf.get('kibana_url'), dashboard_path) # pylint: disable=protected-access def get_subtests(self): return self.get_by_params(es_index=self.es_index, main_test_id=self.test_id, subtest_name='*') diff --git a/utils/fix_es_mapping.py b/utils/fix_es_mapping.py index c15686baac..ffa4cc9806 100755 --- a/utils/fix_es_mapping.py +++ b/utils/fix_es_mapping.py @@ -15,10 +15,11 @@ @click.argument('index_name', type=str) def fix_es_mapping(index_name): ks = KeyStore() - es_conf = ks.get_elasticsearch_credentials() + es_conf = ks.get_elasticsearch_token() - mapping_url = "{es_url}/{index_name}/_mapping".format(index_name=index_name, **es_conf) - res = requests.get(mapping_url, auth=(es_conf["es_user"], es_conf["es_password"])) + mapping_url = "{es_url}/{index_name}/_mapping".format(index_name=index_name, es_url=es_conf["es_url"]) + res = requests.get(mapping_url, headers={'Authorization': f'ApiKey {es_conf["api_key"]}'}) + res.raise_for_status() output = res.json()[index_name] output['mappings']['test_stats']['dynamic'] = False @@ -29,7 +30,7 @@ def fix_es_mapping(index_name): output['mappings']['test_stats']['properties']['system_details'] = {"dynamic": False, "properties": {}} res = requests.put(mapping_url + "/test_stats", - json=output['mappings'], auth=(es_conf["es_user"], es_conf["es_password"])) + json=output['mappings'], headers={'Authorization': f'ApiKey {es_conf["api_key"]}'}) print(res.text) res.raise_for_status() diff --git a/utils/migrate_nemesis_data.py b/utils/migrate_nemesis_data.py index 21c5518cb3..03d97955e0 100755 --- a/utils/migrate_nemesis_data.py +++ b/utils/migrate_nemesis_data.py @@ -55,9 +55,8 @@ def migrate(old_index_name, dry_run, new_index, days): # pylint: disable=too-ma } ) ks = KeyStore() - es_conf = ks.get_elasticsearch_credentials() - elastic_search = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=True, - http_auth=(es_conf["es_user"], es_conf["es_password"])) + es_conf = ks.get_elasticsearch_token() + elastic_search = Elasticsearch(**es_conf) if not elastic_search.indices.exists(index=new_index): elastic_search.indices.create(index=new_index)