From 8b7f8d6522600789549bf407d8f7c7bae09288f6 Mon Sep 17 00:00:00 2001 From: Qin Zhao Date: Mon, 31 May 2021 12:46:29 +0800 Subject: [PATCH] Refactor extended profile implementation (cherry picked from commit 6a4a95fd2427e7bf24f784db76913d409e02ee9f) --- .../lbaasv2/drivers/bigip/resource_manager.py | 326 +++++++++++------- 1 file changed, 202 insertions(+), 124 deletions(-) diff --git a/f5_openstack_agent/lbaasv2/drivers/bigip/resource_manager.py b/f5_openstack_agent/lbaasv2/drivers/bigip/resource_manager.py index ed9ab0187..fbd4f2323 100644 --- a/f5_openstack_agent/lbaasv2/drivers/bigip/resource_manager.py +++ b/f5_openstack_agent/lbaasv2/drivers/bigip/resource_manager.py @@ -15,7 +15,6 @@ # import json import os -import re import urllib from f5_openstack_agent.lbaasv2.drivers.bigip import exceptions as f5_ex @@ -107,23 +106,26 @@ def _create(self, bigip, payload, resource, service, **kwargs): resource_type, payload['name']) resource_helper.create(bigip, payload) - def _update(self, bigip, payload, old_resource, resource, service): - if self.resource_helper.exists(bigip, name=payload['name'], - partition=payload['partition']): - LOG.debug("%s already exists ... updating", self._resource) - self.resource_helper.update(bigip, payload) + def _update(self, bigip, payload, old_resource, resource, service, + **kwargs): + resource_helper = kwargs.get("helper", self.resource_helper) + resource_type = kwargs.get("type", self._resource) + if resource_helper.exists(bigip, name=payload['name'], + partition=payload['partition']): + LOG.debug("%s already exists ... updating", resource_type) + resource_helper.update(bigip, payload) else: - LOG.debug("%s does not exist ... creating", self._resource) + LOG.debug("%s does not exist ... creating", resource_type) payload = self._create_payload(resource, service) - LOG.debug("%s payload is %s", self._resource, payload) - self.resource_helper.create(bigip, payload) + LOG.debug("%s payload is %s", resource_type, payload) + resource_helper.create(bigip, payload) def _delete(self, bigip, payload, resource, service, **kwargs): resource_helper = kwargs.get("helper", self.resource_helper) resource_helper.delete(bigip, name=payload['name'], partition=payload['partition']) - def _check_update_needed(self, payload, old_resource, resource): + def _update_needed(self, payload, old_resource, resource): if not payload or len(payload.keys()) == 0: return False return True @@ -158,7 +160,7 @@ def update(self, old_resource, resource, service, **kwargs): self._update_payload(old_resource, resource, service)) - if self._check_update_needed(payload, old_resource, resource) is False: + if self._update_needed(payload, old_resource, resource) is False: LOG.debug("Do not need to update %s", self._resource) return @@ -335,23 +337,44 @@ def __init__(self, driver): resource_helper.ResourceType.cookie_persistence) self.source_addr_persist_helper = resource_helper.BigIPResourceHelper( resource_helper.ResourceType.source_addr_persistence) - self.http_profile_helper = resource_helper.BigIPResourceHelper( - resource_helper.ResourceType.http_profile) self.ftp_helper = FTPProfileHelper() self.mutable_props = { "name": "description", "default_pool_id": "pool", "connection_limit": "connectionLimit" } + self.profile_map = { + "http_profile": { + "condition": self._isHTTPorTLS, + "customize": self._customize, + "helper": resource_helper.BigIPResourceHelper( + resource_helper.ResourceType.http_profile) + } + } + self.extended_profiles = {} + self._load_extended_profiles() + + def _isHTTPorTLS(self, listener): + if listener['protocol'] == "HTTP" or \ + listener['protocol'] == "TERMINATED_HTTPS": + return True + else: + return False + + def _customize(self, profile_type, profile, listener): + # NOTE(qzhao): The default behavior is to merge customized properties + # into profile payload. That is for the profile like http_profile, who + # supports to fetch addtional properties from customized json via API. + # If any other profile requires different behavior, need to implement + # that specific behavior by itself. + customized = self._customized_profile(profile_type, listener) + profile.update(customized) def _create_payload(self, listener, service): payload = self.driver.service_adapter.get_virtual(service) profiles = payload.get('profiles', []) if '/Common/http' in profiles: profiles.remove('/Common/http') - profile_name = '/' + payload['partition'] + '/' \ - + 'http_profile_' + payload['name'] - profiles.append(profile_name) return payload def _update_payload(self, old_listener, listener, service, **kwargs): @@ -407,13 +430,18 @@ def _check_customized_changed(self, old_listener, listener): return True return False - def _check_update_needed(self, payload, old_listener, listener): - if not payload or len(payload.keys()) == 0: - if self._check_customized_changed(old_listener, listener) \ - is False and \ - self._check_tls_changed(old_listener, listener) is False: - return False - return True + def _check_http2_changed(self, old_listener, listener): + if old_listener.get("http2") != listener.get("http2"): + return True + return False + + def _update_needed(self, payload, old_listener, listener): + if self._check_tls_changed(old_listener, listener) or \ + self._check_customized_changed(old_listener, listener) or \ + self._check_http2_changed(old_listener, listener): + return True + return super(ListenerManager, self)._update_needed( + payload, old_listener, listener) def _create_persist_profile(self, bigip, vs, persist): persist_type = persist.get('type', "") @@ -523,74 +551,152 @@ def _delete_ssl_profiles(self, bigip, vs, service): tls['name'] = vs['name'] listener_builder.remove_ssl_profiles(tls, bigip) - def _delete_http_profile(self, bigip, vs): - payload = { - "name": "http_profile_" + vs['name'], - "partition": vs['partition'], - } - super(ListenerManager, self)._delete( - bigip, payload, None, None, - helper=self.http_profile_helper) + def _load_extended_profiles(self): + if not self.driver.conf.f5_extended_profile: + return - def _create_http_profile(self, bigip, listener, vs): + file_name = self.driver.conf.f5_extended_profile + if not os.path.exists(file_name): + LOG.warning("Extended profile %s doesn't exist", file_name) + return - http_profile = self.__create_http_profile_content(bigip, listener, vs) - http_profile['partition'] = vs['partition'] - http_profile['name'] = "http_profile_" + vs['name'] - super(ListenerManager, self)._create( - bigip, http_profile, None, None, type="http-profile", - helper=self.http_profile_helper, overwrite=True) - - def __create_http_profile_content(self, bigip, listener, vs): - # If the f5_extended_profile is configured in .ini, then - # 1) check if the configured file exists or not. - # 2) parse the content in the file - # If customized is provided from cli, then - # 1) parse vs['customized'] as json format. - # 2) merge existing http_profile dict with parsed 'http_profile' - - http_profile = {} - if self.driver.conf.f5_extended_profile: - # check if the file exists or not. - # check the content of the file content - file_name = self.driver.conf.f5_extended_profile - LOG.debug("extended profile file configured is %s", - file_name) - if not os.path.exists(file_name): - LOG.warning("extended profile %s doesn't exist", - file_name) - else: - try: - with open(file_name) as fp: - payload = json.load(fp) - if 'http_profile' not in payload: - LOG.debug("http profile is not defined in %s", - file_name) - else: - http_profile = payload.get('http_profile', {}) - except ValueError: - LOG.error("extended profile %s is not a valid json file", - file_name) - - LOG.debug("http profile content from file is %s", http_profile) - - if 'customized' in listener and listener['customized']: - try: - payload = json.loads(listener['customized']) - http_profile_arg = payload.get('http_profile', {}) - http_profile.update(http_profile_arg) - except ValueError: - LOG.error("Invalid json format: %s", listener['customized']) + try: + with open(file_name) as file: + self.extended_profiles = json.load(file) + except ValueError: + LOG.error("Extended profile %s is a invalid json", file_name) + return - # The name and parition items in the file will be overwriten - if 'name' in http_profile: - del http_profile['name'] + # Remove name and partition attributes + for profile_type in self.extended_profiles.keys(): + profile = self.extended_profiles[profile_type] + for key in ["name", "partition"]: + if key in profile: + del profile[key] + return + + def _customized_profile(self, profile_type, listener): + if 'customized' not in listener or not listener['customized']: + return {} - if 'partition' in http_profile: - del http_profile['partition'] + try: + customized = json.loads(listener['customized']) + except ValueError: + LOG.error("Invalid json format: %s", listener['customized']) + return {} + + if profile_type not in customized: + return {} + + return customized.get(profile_type, {}) + + def _profile_condition(self, profile_type, listener): + condition = self.profile_map[profile_type]["condition"] + if callable(condition): + return condition(listener) + elif isinstance(condition, bool): + return condition + else: + return False - LOG.debug("http profile content merged is %s", http_profile) - return http_profile + def _create_extended_profiles(self, bigip, listener, vs): + for profile_type in self.profile_map.keys(): + if self._profile_condition(profile_type, listener): + profile = self.extended_profiles.get(profile_type, {}) + customize = self.profile_map[profile_type].get("customize") + if callable(customize): + customize(profile_type, profile, listener) + profile['partition'] = vs['partition'] + profile['name'] = profile_type + "_" + vs['name'] + helper = self.profile_map[profile_type]['helper'] + super(ListenerManager, self)._create( + bigip, profile, None, None, type=profile_type, + helper=helper, overwrite=True) + loc = "/" + profile['partition'] + "/" + profile['name'] + if "profiles" not in vs: + vs['profiles'] = list() + if loc not in vs['profiles']: + vs['profiles'].append(loc) + + def _update_extended_profiles(self, bigip, old_listener, listener, vs): + for profile_type in self.profile_map.keys(): + old_cond = self._profile_condition(profile_type, old_listener) + new_cond = self._profile_condition(profile_type, listener) + + if old_cond == new_cond and new_cond: + # Profile should be already there. Perhaps we need to update + # profile content, because some profiles (eg. http_profile) + # support to fetch property via customized json from API. + # In this case, needn't to consider the initial profile + # properties defined in json file. + old_profile = {} + profile = {} + customize = self.profile_map[profile_type].get("customize") + if callable(customize): + customize(profile_type, old_profile, old_listener) + customize(profile_type, profile, listener) + if profile != old_profile: + profile['partition'] = vs['partition'] + profile['name'] = profile_type + "_" + vs['name'] + helper = self.profile_map[profile_type]['helper'] + super(ListenerManager, self)._update( + bigip, profile, None, None, None, type=profile_type, + helper=helper) + elif old_cond != new_cond and new_cond: + # Need to create and attach profile + profile = self.extended_profiles.get(profile_type, {}) + customize = self.profile_map[profile_type].get("customize") + if callable(customize): + customize(profile_type, profile, listener) + profile['partition'] = vs['partition'] + profile['name'] = profile_type + "_" + vs['name'] + helper = self.profile_map[profile_type]['helper'] + super(ListenerManager, self)._create( + bigip, profile, None, None, type=profile_type, + helper=helper, overwrite=True) + self._attach_profile(bigip, vs, profile) + elif old_cond != new_cond and not new_cond: + # Need to detach and delete profile + profile = {} + profile['partition'] = vs['partition'] + profile['name'] = profile_type + "_" + vs['name'] + self._detach_profile(bigip, vs, profile) + helper = self.profile_map[profile_type]['helper'] + super(ListenerManager, self)._delete( + bigip, profile, None, None, type=profile_type, + helper=helper) + + def _delete_extended_profiles(self, bigip, listener, vs): + for profile_type in self.profile_map.keys(): + profile = {} + profile['partition'] = vs['partition'] + profile['name'] = profile_type + "_" + vs['name'] + helper = self.profile_map[profile_type]['helper'] + super(ListenerManager, self)._delete( + bigip, profile, None, None, type=profile_type, helper=helper) + + def _attach_profile(self, bigip, vs, profile): + v = self.resource_helper.load(bigip, name=vs['name'], + partition=vs['partition']) + if v.profiles_s.profiles.exists(name=profile['name'], + partition=profile['partition']): + LOG.debug("Profile %s has already been attached to vs %s", + profile['name'], vs['name']) + else: + v.profiles_s.profiles.create(name=profile['name'], + partition=profile['partition']) + + def _detach_profile(self, bigip, vs, profile): + v = self.resource_helper.load(bigip, name=vs['name'], + partition=vs['partition']) + if v.profiles_s.profiles.exists(name=profile['name'], + partition=profile['partition']): + p = v.profiles_s.profiles.load(name=profile['name'], + partition=profile['partition']) + p.delete() + else: + LOG.debug("Profile %s is not attached to vs %s", + profile['name'], vs['name']) def _create(self, bigip, vs, listener, service): tls = self.driver.service_adapter.get_tls(service) @@ -607,9 +713,11 @@ def _create(self, bigip, vs, listener, service): loadbalancer = service.get('loadbalancer', dict()) network_id = loadbalancer.get('network_id', "") self.driver.service_adapter.get_vlan(vs, bigip, network_id) - if listener['protocol'] == "HTTP" or \ - listener['protocol'] == "TERMINATED_HTTPS": - self._create_http_profile(bigip, listener, vs) + + # Create the following profiles required by this VS: + # HTTP profile (if listener is HTTP or TERMINATED_HTTPS) + self._create_extended_profiles(bigip, listener, vs) + super(ListenerManager, self)._create(bigip, vs, listener, service) def __get_profiles_from_bigip(self, bigip, vs): @@ -647,40 +755,10 @@ def _update(self, bigip, vs, old_listener, listener, service): profile = self._create_persist_profile(bigip, vs, persist) vs['persist'] = [{"name": profile}] - if self._check_customized_changed(old_listener, listener) is True: - self._create_http_profile(bigip, listener, vs) - # load the porfiles from bigip if needed - if not orig_profiles: - orig_profiles = self.__get_profiles_from_bigip(bigip, vs) - - # build the profiles property for the update payload - profiles = list() - profile_exists = False - profile_name = '/' + vs['partition'] + '/' \ - + 'http_profile_' + vs['name'] - http_pattern = "https://localhost/mgmt/tm/ltm/profile/http/" - profiles.append(profile_name) - - for profile in orig_profiles['items']: - # check if the new http profile is already there. - # if yes, don't do anything - if profile['fullPath'] == profile_name: - LOG.debug("The http profile is already bound to vs.") - profile_exists = True - break - - link = profile['nameReference']['link'] - if not re.search(http_pattern, link): - profiles.append(profile['fullPath']) - - # only build the profiles body if needed - if profile_exists is False: - if 'profiles' not in vs: - vs['profiles'] = list() - # add the profile which is from above profiles list and - # doesn't exist in origin vs[profiles'] - vs['profiles'] += filter( - lambda x: x not in vs['profiles'], profiles) + # Other code might call ListenerManager to post vs payload directly. + # Only need to refresh profile when a real listener update occurs. + if old_listener and listener: + self._update_extended_profiles(bigip, old_listener, listener, vs) # If no vs property to update, do not call icontrol patch api. # This happens, when vs payload only contains 'customized'. @@ -699,7 +777,7 @@ def _delete(self, bigip, vs, listener, service): super(ListenerManager, self)._delete(bigip, vs, listener, service) self._delete_persist_profile(bigip, vs) self._delete_ssl_profiles(bigip, vs, service) - self._delete_http_profile(bigip, vs) + self._delete_extended_profiles(bigip, listener, vs) ftp_enable = self.ftp_helper.enable_ftp(service) if ftp_enable: self.ftp_helper.remove_profile(service, vs, bigip)