diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index a7b063dfc..3bf9696a9 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -17,6 +17,7 @@ import logging import os import subprocess +import yaml import zaza import zaza.charm_lifecycle.utils @@ -184,6 +185,39 @@ def wait_for_dead_units(self, application_name, dead_count): if ustatus.agent_status.life == 'dying'] assert len(dead_units) == dead_count + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8)) + def wait_for_traefik(self, application_name): + """Wait for traefk to finish processing lb changes.""" + logging.warning( + "Waiting for traefik to process changes. This is a temporary " + "workaround and should be removed when there is a way to " + "determine when traefik has processed all requests") + units_count = len(zaza.model.get_units(application_name)) + container_cmd = ( + "cat /opt/traefik/juju/juju_ingress_ingress_*_{}.yaml").format( + application_name) + container_name = "traefik" + for unit in zaza.model.get_units("traefik"): + config = subprocess.check_output([ + "juju", + "ssh", + "-m", zaza.model.get_juju_model(), + "--container", + container_name, + unit.entity_id, + container_cmd]).decode() + service_config = yaml.safe_load(config) + loadBalancers = [ + lb['loadBalancer'] + for lb in service_config['http']['services'].values()] + assert len(loadBalancers) == 1 + unit_count_in_lb = len(loadBalancers[0]['servers']) + logging.info("Traefik LB server count: {} unit count: {}".format( + unit_count_in_lb, + units_count)) + assert unit_count_in_lb == units_count + def run(self): """Run tempest tests as specified in tests/tests.yaml. @@ -197,6 +231,7 @@ def run(self): render_tempest_config_keystone_v3 = tenacity.retry( wait=tenacity.wait_fixed(10), stop=tenacity.stop_after_attempt(3) )(tempest_utils.render_tempest_config_keystone_v3) + self.wait_for_traefik(self.application_name) zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints() render_tempest_config_keystone_v3(minimal=True) if not super().run(): @@ -208,6 +243,7 @@ def run(self): zaza.model.block_until_all_units_idle() logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.expected_statuses) + self.wait_for_traefik(self.application_name) zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints() render_tempest_config_keystone_v3(minimal=True) if not super().run(): @@ -222,6 +258,7 @@ def run(self): zaza.model.block_until_all_units_idle() logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.expected_statuses) + self.wait_for_traefik(self.application_name) zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints() render_tempest_config_keystone_v3(minimal=True) return super().run()