feature(elasticsearch): move to using api key

switching form user/password to using api keys (tokens)
also switching from using urls to using the elastic cloud_id
which is what recommanded by elasticsearch cloud, and enable
compression.

Ref: https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/connecting.html#connect-ec
Ref: scylladb/qa-tasks#1747
(cherry picked from commit 767d199)

# Conflicts:
#	utils/update_field.py

pylintrc

@@ -370,6 +370,7 @@ good-names=i,
            ks,
            cf,
            pytestmark,
+           es,
 
 # Include a hint for the correct naming format with invalid-name.
 include-naming-hint=no

sdcm/es.py

@@ -1,4 +1,5 @@
 import logging
+from functools import cached_property
 
 import elasticsearch
 
@@ -13,13 +14,11 @@ class ES(elasticsearch.Elasticsearch):
     """
 
     def __init__(self):
-        self._conf = self.get_conf()
-        super().__init__(hosts=[self._conf["es_url"]], verify_certs=False,
-                         http_auth=(self._conf["es_user"], self._conf["es_password"]))
+        super().__init__(**self.conf)
 
-    def get_conf(self):
-        self.key_store = KeyStore()
-        return self.key_store.get_elasticsearch_credentials()
+    @cached_property
+    def conf(self):
+        return KeyStore().get_elasticsearch_token()
 
     def _create_index(self, index):
         self.indices.create(index=index, ignore=400)  # pylint: disable=unexpected-keyword-arg

sdcm/keystore.py

@@ -43,8 +43,8 @@ def download_file(self, filename, dest_filename):
     def get_email_credentials(self):
         return self.get_json("email_config.json")
 
-    def get_elasticsearch_credentials(self):
-        return self.get_json("es.json")
+    def get_elasticsearch_token(self):
+        return self.get_json("es_token.json")
 
     def get_gcp_credentials(self):
         project = os.environ.get('SCT_GCE_PROJECT') or 'gcp-sct-project-1'

sdcm/nemesis_publisher.py

@@ -33,9 +33,8 @@ def __init__(self, tester):
 
     def create_es_connection(self):
         ks = KeyStore()
-        es_conf = ks.get_elasticsearch_credentials()
-        self.es = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=False,  # pylint: disable=invalid-name
-                                http_auth=(es_conf["es_user"], es_conf["es_password"]))
+        es_conf = ks.get_elasticsearch_token()
+        self.es = Elasticsearch(**es_conf)
 
     @cached_property
     def stats(self):

sdcm/results_analyze/__init__.py

@@ -47,7 +47,7 @@ class BaseResultsAnalyzer:  # pylint: disable=too-many-instance-attributes
     def __init__(self, es_index, es_doc_type, email_recipients=(), email_template_fp="", query_limit=1000, logger=None,
                  events=None):
         self._es = ES()
-        self._conf = self._es._conf  # pylint: disable=protected-access
+        self._conf = self._es.conf  # pylint: disable=protected-access
         self._es_index = es_index
         self._es_doc_type = es_doc_type
         self._limit = query_limit

sdcm/results_analyze/test.py

@@ -520,7 +520,7 @@ def get_metric_class(cls, metric_path, default=__DEFAULT__):
     def gen_kibana_dashboard_url(
             dashboard_path="app/kibana#/dashboard/03414b70-0e89-11e9-a976-2fe0f5890cd0?_g=()"
     ):
-        return "%s/%s" % (ES()._conf.get('kibana_url'), dashboard_path)  # pylint: disable=protected-access
+        return "%s/%s" % (ES().conf.get('kibana_url'), dashboard_path)  # pylint: disable=protected-access
 
     def get_subtests(self):
         return self.get_by_params(es_index=self.es_index, main_test_id=self.test_id, subtest_name='*')

utils/fix_es_mapping.py

@@ -15,10 +15,11 @@
 @click.argument('index_name', type=str)
 def fix_es_mapping(index_name):
     ks = KeyStore()
-    es_conf = ks.get_elasticsearch_credentials()
+    es_conf = ks.get_elasticsearch_token()
 
-    mapping_url = "{es_url}/{index_name}/_mapping".format(index_name=index_name, **es_conf)
-    res = requests.get(mapping_url, auth=(es_conf["es_user"], es_conf["es_password"]))
+    mapping_url = "{es_url}/{index_name}/_mapping".format(index_name=index_name, es_url=es_conf["es_url"])
+    res = requests.get(mapping_url, headers={'Authorization': f'ApiKey {es_conf["api_key"]}'})
+    res.raise_for_status()
     output = res.json()[index_name]
 
     output['mappings']['test_stats']['dynamic'] = False
@@ -29,7 +30,7 @@ def fix_es_mapping(index_name):
     output['mappings']['test_stats']['properties']['system_details'] = {"dynamic": False, "properties": {}}
 
     res = requests.put(mapping_url + "/test_stats",
-                       json=output['mappings'], auth=(es_conf["es_user"], es_conf["es_password"]))
+                       json=output['mappings'], headers={'Authorization': f'ApiKey {es_conf["api_key"]}'})
     print(res.text)
     res.raise_for_status()
 

utils/migrate_nemesis_data.py

@@ -55,9 +55,8 @@ def migrate(old_index_name, dry_run, new_index, days):  # pylint: disable=too-ma
     }
     )
     ks = KeyStore()
-    es_conf = ks.get_elasticsearch_credentials()
-    elastic_search = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=True,
-                                   http_auth=(es_conf["es_user"], es_conf["es_password"]))
+    es_conf = ks.get_elasticsearch_token()
+    elastic_search = Elasticsearch(**es_conf)
 
     if not elastic_search.indices.exists(index=new_index):
         elastic_search.indices.create(index=new_index)