Add tactical work around for traefik

zaza/openstack/charm_tests/tempest/tests.py

@@ -17,6 +17,7 @@
 import logging
 import os
 import subprocess
+import yaml
 
 import zaza
 import zaza.charm_lifecycle.utils
@@ -184,6 +185,39 @@ def wait_for_dead_units(self, application_name, dead_count):
                       if ustatus.agent_status.life == 'dying']
         assert len(dead_units) == dead_count
 
+    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
+                    reraise=True, stop=tenacity.stop_after_attempt(8))
+    def wait_for_traefik(self, application_name):
+        """Wait for traefk to finish processing lb changes."""
+        logging.warning(
+            "Waiting for traefik to process changes. This is a temporary "
+            "workaround and should be removed when there is a way to "
+            "determine when traefik has processed all requests")
+        units_count = len(zaza.model.get_units(application_name))
+        container_cmd = (
+            "cat /opt/traefik/juju/juju_ingress_ingress_*_{}.yaml").format(
+                application_name)
+        container_name = "traefik"
+        for unit in zaza.model.get_units("traefik"):
+            config = subprocess.check_output([
+                "juju",
+                "ssh",
+                "-m", zaza.model.get_juju_model(),
+                "--container",
+                container_name,
+                unit.entity_id,
+                container_cmd]).decode()
+            service_config = yaml.safe_load(config)
+            loadBalancers = [
+                lb['loadBalancer']
+                for lb in service_config['http']['services'].values()]
+            assert len(loadBalancers) == 1
+            unit_count_in_lb = len(loadBalancers[0]['servers'])
+            logging.info("Traefik LB server count: {} unit count: {}".format(
+                unit_count_in_lb,
+                units_count))
+            assert unit_count_in_lb == units_count
+
     def run(self):
         """Run tempest tests as specified in tests/tests.yaml.
 
@@ -197,6 +231,7 @@ def run(self):
         render_tempest_config_keystone_v3 = tenacity.retry(
             wait=tenacity.wait_fixed(10), stop=tenacity.stop_after_attempt(3)
         )(tempest_utils.render_tempest_config_keystone_v3)
+        self.wait_for_traefik(self.application_name)
         zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints()
         render_tempest_config_keystone_v3(minimal=True)
         if not super().run():
@@ -208,6 +243,7 @@ def run(self):
         zaza.model.block_until_all_units_idle()
         logging.info("Wait for status ready ...")
         zaza.model.wait_for_application_states(states=self.expected_statuses)
+        self.wait_for_traefik(self.application_name)
         zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints()
         render_tempest_config_keystone_v3(minimal=True)
         if not super().run():
@@ -222,6 +258,7 @@ def run(self):
         zaza.model.block_until_all_units_idle()
         logging.info("Wait for status ready ...")
         zaza.model.wait_for_application_states(states=self.expected_statuses)
+        self.wait_for_traefik(self.application_name)
         zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints()
         render_tempest_config_keystone_v3(minimal=True)
         return super().run()