From b26554160ce2a089dee9b242dfb624868656267e Mon Sep 17 00:00:00 2001 From: Szymon Basan Date: Tue, 2 Jul 2024 17:06:27 +0200 Subject: [PATCH 1/4] draft: system profile update with device access parcel is working --- .../device_template/device_template.py | 16 +- .../models/configuration/config_migration.py | 37 ++++- .../sdwan/system/device_access.py | 4 +- .../sdwan/system/device_access_ipv6.py | 4 +- .../creators/config_pusher.py | 12 +- .../creators/localized_policy_pusher.py | 139 ++++++++++++++++++ catalystwan/workflows/config_migration.py | 23 ++- 7 files changed, 227 insertions(+), 8 deletions(-) create mode 100644 catalystwan/utils/config_migration/creators/localized_policy_pusher.py diff --git a/catalystwan/api/templates/device_template/device_template.py b/catalystwan/api/templates/device_template/device_template.py index bf803d31..39c2067b 100644 --- a/catalystwan/api/templates/device_template/device_template.py +++ b/catalystwan/api/templates/device_template/device_template.py @@ -4,7 +4,8 @@ import logging from pathlib import Path -from typing import TYPE_CHECKING, ClassVar, List +from typing import TYPE_CHECKING, ClassVar, List, Optional +from uuid import UUID from jinja2 import DebugUndefined, Environment, FileSystemLoader, meta # type: ignore from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -15,6 +16,13 @@ logger = logging.getLogger(__name__) +def str_to_uuid(s: str) -> Optional[UUID]: + try: + return UUID(s) + except ValueError: + return None + + class GeneralTemplate(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) @@ -54,6 +62,12 @@ class DeviceTemplate(BaseModel): ) policy_id: str = Field(default="", serialization_alias="policyId", validation_alias="policyId") + def get_security_policy_uuid(self) -> Optional[UUID]: + return str_to_uuid(self.security_policy_id) + + def get_policy_uuid(self) -> Optional[UUID]: + return str_to_uuid(self.policy_id) + def generate_payload(self) -> str: env = Environment( loader=FileSystemLoader(self.payload_path.parent), diff --git a/catalystwan/models/configuration/config_migration.py b/catalystwan/models/configuration/config_migration.py index 2ee6f992..ce55614a 100644 --- a/catalystwan/models/configuration/config_migration.py +++ b/catalystwan/models/configuration/config_migration.py @@ -137,6 +137,17 @@ class UX1Templates(BaseModel): validation_alias="deviceTemplates", ) + def create_device_template_by_policy_id_lookup(self) -> Dict[Literal["policy", "security"], Dict[UUID, UUID]]: + lookup: Dict[Literal["policy", "security"], Dict[UUID, UUID]] = {"policy": {}, "security": {}} + for dt in self.device_templates: + policy_id = dt.get_policy_uuid() + security_policy_id = dt.get_security_policy_uuid() + if policy_id is not None: + lookup["policy"][policy_id] = UUID(dt.template_id) + if security_policy_id is not None: + lookup["security"][security_policy_id] = UUID(dt.template_id) + return lookup + class UX1Config(BaseModel): # All UX1 Configuration items - Mega Model @@ -150,6 +161,7 @@ class UX1Config(BaseModel): class TransformHeader(BaseModel): + model_config = ConfigDict(populate_by_name=True) type: str = Field( description="Needed to push item to specific endpoint." "Type discriminator is not present in many UX2 item payloads" @@ -157,6 +169,9 @@ class TransformHeader(BaseModel): origin: UUID = Field(description="Original UUID of converted item") origname: Optional[str] = None subelements: Set[UUID] = Field(default_factory=set) + localized_policy_subelements: Optional[Set[UUID]] = Field( + default=None, serialization_alias="localizedPolicySubelements", validation_alias="localizedPolicySubelements" + ) status: ConvertOutputStatus = Field(default="complete") info: List[str] = Field(default_factory=list) @@ -251,9 +266,29 @@ def insert_parcel_type_from_headers(cls, values: Dict[str, Any]): profile_parcel["parcel"]["type_"] = profile_parcel["header"]["type"] return values - def transformed_parcels_with_origin(self, origin: Set[UUID]) -> List[TransformedParcel]: + def list_transformed_parcels_with_origin(self, origin: Set[UUID]) -> List[TransformedParcel]: return [p for p in self.profile_parcels if p.header.origin in origin] + def add_subelement_in_config_group( + self, profile_type: ProfileType, device_template_id: UUID, subelement: UUID + ) -> bool: + profile_ids: Set[UUID] = set() + for config_group in self.config_groups: + if config_group.header.origin == device_template_id: + profile_ids = config_group.header.subelements + break + if not profile_ids: + return False + for feature_profile in self.feature_profiles: + if feature_profile.header.type == profile_type and feature_profile.header.origin in profile_ids: + head = feature_profile.header + if head.localized_policy_subelements is None: + head.localized_policy_subelements = {subelement} + else: + head.localized_policy_subelements.add(subelement) + return True + return False + class ConfigTransformResult(BaseModel): # https://docs.pydantic.dev/2.0/usage/models/#fields-with-dynamic-default-values diff --git a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py index ad2c81e9..1fa19137 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py @@ -64,8 +64,8 @@ class Sequence(BaseModel): base_action: Union[Global[AcceptDropActionType], Default[Literal[AcceptDropActionType]]] = Field( default=Default[Literal[AcceptDropActionType]](value="accept"), - validation_alias="BasicPolicyActionType", - serialization_alias="BasicPolicyActionType", + validation_alias="baseAction", + serialization_alias="baseAction", ) match_entries: MatchEntries = Field( validation_alias="matchEntries", serialization_alias="matchEntries", description="Define match conditions" diff --git a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py index 74cacde9..22085fc0 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py @@ -64,8 +64,8 @@ class Sequence(BaseModel): base_action: Union[Global[AcceptDropActionType], Default[Literal[AcceptDropActionType]]] = Field( default=Default[Literal[AcceptDropActionType]](value="accept"), - validation_alias="BasicPolicyActionType", - serialization_alias="BasicPolicyActionType", + validation_alias="baseAction", + serialization_alias="baseAction", ) match_entries: MatchEntries = Field( validation_alias="matchEntries", serialization_alias="matchEntries", description="Define match conditions" diff --git a/catalystwan/utils/config_migration/creators/config_pusher.py b/catalystwan/utils/config_migration/creators/config_pusher.py index b8178848..2bd1e157 100644 --- a/catalystwan/utils/config_migration/creators/config_pusher.py +++ b/catalystwan/utils/config_migration/creators/config_pusher.py @@ -22,6 +22,7 @@ from catalystwan.models.configuration.feature_profile.sdwan.topology.mesh import MeshParcel from catalystwan.session import ManagerSession from catalystwan.utils.config_migration.creators.groups_of_interests_pusher import GroupsOfInterestPusher +from catalystwan.utils.config_migration.creators.localized_policy_pusher import LocalizedPolicyPusher from catalystwan.utils.config_migration.creators.security_policy_pusher import SecurityPolicyPusher from catalystwan.utils.config_migration.factories.parcel_pusher import ParcelPusherFactory @@ -52,6 +53,13 @@ def __init__( push_result=self._push_result, push_context=self._push_context, ) + self._localized_policy_feature_updater = LocalizedPolicyPusher( + ux2_config=ux2_config, + session=session, + progress=progress, + push_result=self._push_result, + push_context=self._push_context, + ) self._security_policy_pusher = SecurityPolicyPusher( ux2_config=ux2_config, session=session, @@ -73,6 +81,7 @@ def push(self) -> UX2ConfigPushResult: self._create_cloud_credentials() self._create_config_groups() self._groups_of_interests_pusher.push() + self._localized_policy_feature_updater.push() self._security_policy_pusher.push() self._create_topology_groups( self._push_context.default_policy_object_profile_id @@ -144,6 +153,7 @@ def _create_feature_profile_and_parcels(self, feature_profiles_ids: List[UUID]) profile = pusher.push(transformed_feature_profile.feature_profile, parcels, self._config_map.parcel_map) feature_profiles.append(profile) self._push_result.rollback.add_feature_profile(profile.profile_uuid, profile_type) + self._push_context.id_lookup[feature_profile_id] = profile.profile_uuid except ManagerHTTPError as e: logger.error(f"Error occured during [{fp_name}] feature profile creation: {e}") except Exception: @@ -205,7 +215,7 @@ def _create_topology_groups(self, default_policy_object_profile_id: Optional[UUI logger.error(f"Error occured during topology group creation: {e}") continue - for transformed_parcel in self._ux2_config.transformed_parcels_with_origin(origins): + for transformed_parcel in self._ux2_config.list_transformed_parcels_with_origin(origins): parcel = transformed_parcel.parcel if isinstance(parcel, (CustomControlParcel, HubSpokeParcel, MeshParcel)): try: diff --git a/catalystwan/utils/config_migration/creators/localized_policy_pusher.py b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py new file mode 100644 index 00000000..4f7fb88d --- /dev/null +++ b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py @@ -0,0 +1,139 @@ +import logging +from typing import Callable, Dict, List, Tuple, Union, cast +from uuid import UUID + +from pydantic import Field +from typing_extensions import Annotated + +from catalystwan.endpoints.configuration_group import ConfigGroup +from catalystwan.exceptions import ManagerHTTPError +from catalystwan.models.configuration.config_migration import ( + PushContext, + TransformedParcel, + UX2Config, + UX2ConfigPushResult, +) +from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv4acl import Ipv4AclParcel +from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv6acl import Ipv6AclParcel +from catalystwan.models.configuration.feature_profile.sdwan.system.device_access import DeviceAccessIPv4Parcel +from catalystwan.models.configuration.feature_profile.sdwan.system.device_access_ipv6 import DeviceAccessIPv6Parcel +from catalystwan.models.configuration.profile_type import ProfileType +from catalystwan.session import ManagerSession +from catalystwan.utils.config_migration.creators.references_updater import update_parcel_references + +LOCALIZED_POLICY_PARCEL_TYPES = [ + "ipv4-device-access-policy", + "ipv6-device-access-policy", + "ipv4-acl", + "ipv6-acl", + "route", +] +AnyDeviceAccessParcel = Annotated[ + Union[DeviceAccessIPv4Parcel, DeviceAccessIPv6Parcel], + Field(discriminator="type_"), +] +AnyAclParcel = Annotated[ + Union[Ipv4AclParcel, Ipv6AclParcel], + Field(discriminator="type_"), +] + +logger = logging.getLogger(__name__) + + +class LocalizedPolicyPusher: + """ + 1. Associate selected Config Group with Default_Policy_Object_Profile + 2. Update selected Feature Profiles with Parcels originating from Localized Policy items (eg. acl, route) + Update needs to be performed after Feature Profiles are already populated with VPN parcels + and Default_Policy_Object_Profile is populated with Groups of Interest + """ + + def __init__( + self, + ux2_config: UX2Config, + session: ManagerSession, + push_result: UX2ConfigPushResult, + push_context: PushContext, + progress: Callable[[str, int, int], None], + ) -> None: + self._ux2_config = ux2_config + self._fp_api = session.api.sdwan_feature_profiles + self._cg_api = session.api.config_group + self._system_api = self._fp_api.system + self._transport_api = self._fp_api.transport + self._service_api = self._fp_api.service + self.dns = session.api.sdwan_feature_profiles.dns_security + self._push_result: UX2ConfigPushResult = push_result + self._progress: Callable[[str, int, int], None] = progress + self.push_context = push_context + self._parcel_by_id = self._create_parcel_by_id_lookup() + + def _create_parcel_by_id_lookup(self) -> Dict[UUID, TransformedParcel]: + lookup: Dict[UUID, TransformedParcel] = dict() + for transformed_parcel in self._ux2_config.profile_parcels: + if transformed_parcel.header.type in LOCALIZED_POLICY_PARCEL_TYPES: + lookup[transformed_parcel.header.origin] = transformed_parcel + return lookup + + def get_parcels_to_push(self, parcel_ids: List[UUID]) -> List[TransformedParcel]: + result: List[TransformedParcel] = list() + for parcel_id in parcel_ids: + if parcel := self._parcel_by_id.get(parcel_id): + result.append(parcel) + return result + + def find_config_groups_to_update(self) -> List[UUID]: + result: List[UUID] = list() + for transformed_cg in self._ux2_config.config_groups: + if transformed_cg.header.localized_policy_subelements is not None: + updated_id = self.push_context.id_lookup[transformed_cg.header.origin] + result.append(updated_id) + return result + + def get_config_group_contents(self, cg_ids: List[UUID]) -> Dict[UUID, ConfigGroup]: + result: Dict[UUID, ConfigGroup] = dict() + for cg_id in cg_ids: + cg = self._cg_api.get(cg_id) + result[cg_id] = cg + return result + + def find_profiles_to_update(self) -> List[Tuple[ProfileType, UUID, List[TransformedParcel]]]: + profiles: List[Tuple[ProfileType, UUID, List[TransformedParcel]]] = list() + for transformed_profile in self._ux2_config.feature_profiles: + if transformed_profile.header.localized_policy_subelements is not None: + updated_id = self.push_context.id_lookup[transformed_profile.header.origin] + parcels = self.get_parcels_to_push(list(transformed_profile.header.localized_policy_subelements)) + profiles.append((cast(ProfileType, transformed_profile.header.type), updated_id, parcels)) + return profiles + + def update_system_profile(self, profile_id: UUID, device_access: AnyDeviceAccessParcel): + try: + self._system_api.create_parcel( + profile_id=profile_id, payload=update_parcel_references(device_access, self.push_context.id_lookup) + ) + except ManagerHTTPError as e: + logger.error(f"Error occured during creation of {device_access.type_} {device_access.parcel_name}: {e}") + + def associate_config_groups_with_default_policy_object_profile(self): + for cg_id, cg in self.get_config_group_contents(self.find_config_groups_to_update()).items(): + profile_ids = [p.id for p in cg.profiles] + profile_ids.append(self.push_context.default_policy_object_profile_id) + try: + self._cg_api.edit( + cg_id=str(cg_id), + name=cg.name, + description=cg.description, + solution=cg.solution, + profile_ids=profile_ids, + ) + except ManagerHTTPError as e: + logger.error(f"Error occured during config group edit: {e}") + + def push(self): + self.associate_config_groups_with_default_policy_object_profile() + for profile_tuple in self.find_profiles_to_update(): + type_, uuid_, transformed_parcels = profile_tuple + for transformed_parcel in transformed_parcels: + parcel = transformed_parcel.parcel + if type_ == "system" and isinstance(parcel, (DeviceAccessIPv4Parcel, DeviceAccessIPv6Parcel)): + self.update_system_profile(profile_id=uuid_, device_access=parcel) diff --git a/catalystwan/workflows/config_migration.py b/catalystwan/workflows/config_migration.py index fc123c4b..03037957 100644 --- a/catalystwan/workflows/config_migration.py +++ b/catalystwan/workflows/config_migration.py @@ -329,7 +329,7 @@ def transform(ux1: UX1Config, add_suffix: bool = True) -> ConfigTransformResult: subtemplates_mapping[UUID(subtemplate_level_1.templateId)].add( UUID(subtemplate_level_2.templateId) ) - + policy_id = dt.get_policy_uuid() transformed_cg = TransformedConfigGroup( header=TransformHeader( type="config_group", @@ -337,6 +337,7 @@ def transform(ux1: UX1Config, add_suffix: bool = True) -> ConfigTransformResult: subelements=set( [fp_system_uuid, fp_other_uuid, fp_service_uuid, fp_transport_and_management_uuid, fp_cli_uuid] ), + localized_policy_subelements=set([policy_id]) if policy_id else None, ), config_group=ConfigGroupCreationPayload( name=dt.template_name, @@ -421,6 +422,26 @@ def transform(ux1: UX1Config, add_suffix: bool = True) -> ConfigTransformResult: ) ux2.profile_parcels.append(TransformedParcel(header=header, parcel=pd_parcel)) + # Localized Policies + _lookup = ux1.templates.create_device_template_by_policy_id_lookup() + dt_by_policy_id = _lookup["policy"] + for localized_policy in ux1.policies.localized_policies: + if not isinstance(localized_policy.policy_definition, str): + if dt_id := dt_by_policy_id.get(localized_policy.policy_id): + for item in localized_policy.policy_definition.assembly: + if item.type == "deviceaccesspolicy" or item.type == "deviceaccesspolicyv6": + ux2.add_subelement_in_config_group( + profile_type="system", device_template_id=dt_id, subelement=item.definition_id + ) + elif item.type == "acl" or item.type == "aclv6": + ux2.add_subelement_in_config_group( + profile_type="transport", device_template_id=dt_id, subelement=item.definition_id + ) + elif item.type == "vedgeRoute": + ux2.add_subelement_in_config_group( + profile_type="service", device_template_id=dt_id, subelement=item.definition_id + ) + # Security policies for security_policy in ux1.policies.security_policies: sp_result = convert_security_policy(security_policy, security_policy.policy_id, policy_context) From c8daa91d5539a6a39315a50eb13e832d9260d0b7 Mon Sep 17 00:00:00 2001 From: Szymon Basan Date: Wed, 3 Jul 2024 12:02:15 +0200 Subject: [PATCH 2/4] refactor --- .../creators/localized_policy_pusher.py | 64 ++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/catalystwan/utils/config_migration/creators/localized_policy_pusher.py b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py index 4f7fb88d..cb1151e5 100644 --- a/catalystwan/utils/config_migration/creators/localized_policy_pusher.py +++ b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py @@ -5,6 +5,7 @@ from pydantic import Field from typing_extensions import Annotated +from catalystwan.api.builders.feature_profiles.report import FeatureProfileBuildReport from catalystwan.endpoints.configuration_group import ConfigGroup from catalystwan.exceptions import ManagerHTTPError from catalystwan.models.configuration.config_migration import ( @@ -15,6 +16,7 @@ ) from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv4acl import Ipv4AclParcel from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv6acl import Ipv6AclParcel +from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel from catalystwan.models.configuration.feature_profile.sdwan.system.device_access import DeviceAccessIPv4Parcel from catalystwan.models.configuration.feature_profile.sdwan.system.device_access_ipv6 import DeviceAccessIPv6Parcel from catalystwan.models.configuration.profile_type import ProfileType @@ -37,6 +39,8 @@ Field(discriminator="type_"), ] +ProfileInfo = Tuple[ProfileType, UUID, str, List[TransformedParcel]] + logger = logging.getLogger(__name__) @@ -75,14 +79,22 @@ def _create_parcel_by_id_lookup(self) -> Dict[UUID, TransformedParcel]: lookup[transformed_parcel.header.origin] = transformed_parcel return lookup - def get_parcels_to_push(self, parcel_ids: List[UUID]) -> List[TransformedParcel]: + def _create_profile_report_by_id_lookup(self, profile_ids: List[UUID]) -> Dict[UUID, FeatureProfileBuildReport]: + lookup: Dict[UUID, FeatureProfileBuildReport] = dict() + for cg_report in self._push_result.report.config_groups: + for fp_report in cg_report.feature_profiles: + if fp_report.profile_uuid in profile_ids: + lookup[fp_report.profile_uuid] = fp_report + return lookup + + def _get_parcels_to_push(self, parcel_ids: List[UUID]) -> List[TransformedParcel]: result: List[TransformedParcel] = list() for parcel_id in parcel_ids: if parcel := self._parcel_by_id.get(parcel_id): result.append(parcel) return result - def find_config_groups_to_update(self) -> List[UUID]: + def _find_config_groups_to_update(self) -> List[UUID]: result: List[UUID] = list() for transformed_cg in self._ux2_config.config_groups: if transformed_cg.header.localized_policy_subelements is not None: @@ -90,32 +102,40 @@ def find_config_groups_to_update(self) -> List[UUID]: result.append(updated_id) return result - def get_config_group_contents(self, cg_ids: List[UUID]) -> Dict[UUID, ConfigGroup]: + def _get_config_group_contents(self, cg_ids: List[UUID]) -> Dict[UUID, ConfigGroup]: result: Dict[UUID, ConfigGroup] = dict() for cg_id in cg_ids: cg = self._cg_api.get(cg_id) result[cg_id] = cg return result - def find_profiles_to_update(self) -> List[Tuple[ProfileType, UUID, List[TransformedParcel]]]: - profiles: List[Tuple[ProfileType, UUID, List[TransformedParcel]]] = list() + def _find_profiles_to_update(self) -> List[ProfileInfo]: + profiles: List[ProfileInfo] = list() for transformed_profile in self._ux2_config.feature_profiles: if transformed_profile.header.localized_policy_subelements is not None: + profile_type = cast(ProfileType, transformed_profile.header.type) + name = transformed_profile.feature_profile.name updated_id = self.push_context.id_lookup[transformed_profile.header.origin] - parcels = self.get_parcels_to_push(list(transformed_profile.header.localized_policy_subelements)) - profiles.append((cast(ProfileType, transformed_profile.header.type), updated_id, parcels)) + parcels = self._get_parcels_to_push(list(transformed_profile.header.localized_policy_subelements)) + profiles.append((profile_type, updated_id, name, parcels)) return profiles - def update_system_profile(self, profile_id: UUID, device_access: AnyDeviceAccessParcel): + def update_system_profile( + self, profile_id: UUID, device_access: AnyDeviceAccessParcel, report: FeatureProfileBuildReport + ): try: - self._system_api.create_parcel( + parcel_id = self._system_api.create_parcel( profile_id=profile_id, payload=update_parcel_references(device_access, self.push_context.id_lookup) - ) + ).id + report.add_created_parcel(parcel_name=device_access.parcel_name, parcel_uuid=parcel_id) except ManagerHTTPError as e: logger.error(f"Error occured during creation of {device_access.type_} {device_access.parcel_name}: {e}") + report.add_failed_parcel( + parcel_name=device_access.parcel_name, parcel_type=device_access.type_, error_info=e.info + ) def associate_config_groups_with_default_policy_object_profile(self): - for cg_id, cg in self.get_config_group_contents(self.find_config_groups_to_update()).items(): + for cg_id, cg in self._get_config_group_contents(self._find_config_groups_to_update()).items(): profile_ids = [p.id for p in cg.profiles] profile_ids.append(self.push_context.default_policy_object_profile_id) try: @@ -130,10 +150,24 @@ def associate_config_groups_with_default_policy_object_profile(self): logger.error(f"Error occured during config group edit: {e}") def push(self): + self._progress("Associating Config Groups with Default Policy Object Profile", 0, 1) self.associate_config_groups_with_default_policy_object_profile() - for profile_tuple in self.find_profiles_to_update(): - type_, uuid_, transformed_parcels = profile_tuple + self._progress("Associating Config Groups with Default Policy Object Profile", 1, 1) + profile_infos = self._find_profiles_to_update() + profile_ids = [t[1] for t in profile_infos] + profile_reports = self._create_profile_report_by_id_lookup(profile_ids) + for i, profile_info in enumerate(profile_infos): + profile_type, profile_id, profile_name, transformed_parcels = profile_info + self._progress(f"Updating {profile_name} profile with policy parcels", i + 1, len(profile_ids)) for transformed_parcel in transformed_parcels: parcel = transformed_parcel.parcel - if type_ == "system" and isinstance(parcel, (DeviceAccessIPv4Parcel, DeviceAccessIPv6Parcel)): - self.update_system_profile(profile_id=uuid_, device_access=parcel) + if profile_type == "system" and isinstance(parcel, (DeviceAccessIPv4Parcel, DeviceAccessIPv6Parcel)): + self.update_system_profile( + profile_id=profile_id, device_access=parcel, report=profile_reports[profile_id] + ) + elif profile_type == "service" and isinstance(parcel, RoutePolicyParcel): + logger.warning(f"not implemented: adding {parcel.parcel_name} parcel to profile: {profile_id}") + elif profile_type == "transport" and isinstance(parcel, (Ipv4AclParcel, Ipv6AclParcel)): + logger.warning(f"not implemented: adding {parcel.parcel_name} parcel to profile: {profile_id}") + else: + logger.warning(f"Unexpected profile type {profile_type} to add Localized Policy items, skipping") From e57fef636c208784db48b6971c8d9aa1806971f9 Mon Sep 17 00:00:00 2001 From: Szymon Basan Date: Wed, 3 Jul 2024 22:37:08 +0200 Subject: [PATCH 3/4] all items working and reported --- catalystwan/api/feature_profile_api.py | 19 ++- .../sdwan/test_extended_community.py | 2 +- .../sdwan/test_ssl_decryption.py | 2 +- .../models/configuration/config_migration.py | 11 +- .../feature_profile/sdwan/acl/ipv6acl.py | 2 +- .../sdwan/service/route_policy.py | 6 +- .../sdwan/transport/__init__.py | 2 + catalystwan/models/policy/policy.py | 4 - .../creators/config_pusher.py | 4 +- .../creators/groups_of_interests_pusher.py | 2 +- .../creators/localized_policy_pusher.py | 123 +++++++++++------- .../creators/references_updater.py | 9 +- catalystwan/utils/config_migration/runner.py | 34 ++--- catalystwan/workflows/config_migration.py | 10 +- examples/parcel_configuration_guide.py | 4 +- 15 files changed, 131 insertions(+), 103 deletions(-) diff --git a/catalystwan/api/feature_profile_api.py b/catalystwan/api/feature_profile_api.py index 1ff6fa9b..b98167fc 100644 --- a/catalystwan/api/feature_profile_api.py +++ b/catalystwan/api/feature_profile_api.py @@ -46,6 +46,7 @@ from catalystwan.models.configuration.feature_profile.sdwan.routing import AnyRoutingParcel, RoutingBgpParcel from catalystwan.models.configuration.feature_profile.sdwan.service import AnyServiceParcel from catalystwan.models.configuration.feature_profile.sdwan.service.multicast import MulticastParcel +from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel from catalystwan.models.configuration.feature_profile.sdwan.sig_security.sig_security import SIGParcel from catalystwan.models.configuration.feature_profile.sdwan.topology import AnyTopologyParcel from catalystwan.models.configuration.feature_profile.sdwan.topology.custom_control import CustomControlParcel @@ -306,6 +307,12 @@ def get_parcel(self, profile_id: UUID, parcel_type: Type[Ipv4AclParcel], parcel_ def get_parcel(self, profile_id: UUID, parcel_type: Type[Ipv6AclParcel], parcel_id: UUID) -> Parcel[Ipv6AclParcel]: ... + @overload + def get_parcel( + self, profile_id: UUID, parcel_type: Type[RoutePolicyParcel], parcel_id: UUID + ) -> Parcel[RoutePolicyParcel]: + ... + def get_parcel( self, profile_id: UUID, parcel_type: Type[Union[AnyTransportParcel, AnyRoutingParcel]], parcel_id: UUID ) -> Parcel: @@ -431,21 +438,21 @@ def delete_all_profiles(self) -> None: self.delete_profile(profile.profile_id) def create_parcel( - self, profile_uuid: UUID, payload: AnyServiceParcel, vpn_uuid: Optional[UUID] = None + self, profile_id: UUID, payload: AnyServiceParcel, vpn_uuid: Optional[UUID] = None ) -> ParcelCreationResponse: """ Create Service Parcel for selected profile_id based on payload type """ if vpn_uuid is not None: if isinstance(payload, MulticastParcel): - response = self.endpoint.create_service_parcel(profile_uuid, payload._get_parcel_type(), payload) + response = self.endpoint.create_service_parcel(profile_id, payload._get_parcel_type(), payload) return self.endpoint.associate_parcel_with_vpn( - profile_uuid, vpn_uuid, payload._get_parcel_type(), ParcelAssociationPayload(parcel_id=response.id) + profile_id, vpn_uuid, payload._get_parcel_type(), ParcelAssociationPayload(parcel_id=response.id) ) else: parcel_type = payload._get_parcel_type().replace("lan/vpn/", "") - return self.endpoint.create_lan_vpn_sub_parcel(profile_uuid, vpn_uuid, parcel_type, payload) - return self.endpoint.create_service_parcel(profile_uuid, payload._get_parcel_type(), payload) + return self.endpoint.create_lan_vpn_sub_parcel(profile_id, vpn_uuid, parcel_type, payload) + return self.endpoint.create_service_parcel(profile_id, payload._get_parcel_type(), payload) def delete_parcel(self, profile_uuid: UUID, parcel_type: Type[AnyServiceParcel], parcel_uuid: UUID) -> None: """ @@ -1194,7 +1201,7 @@ def get( profile_id=profile_id, policy_object_list_type=policy_object_list_type, list_object_id=parcel_id ) - def create(self, profile_id: UUID, payload: AnyPolicyObjectParcel) -> ParcelCreationResponse: + def create_parcel(self, profile_id: UUID, payload: AnyPolicyObjectParcel) -> ParcelCreationResponse: """ Create Policy Object for selected profile_id based on payload type """ diff --git a/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py b/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py index d8d06d72..01a3879f 100644 --- a/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py +++ b/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py @@ -42,7 +42,7 @@ def test_create_extended_community_parcel(self): ext.add_site_of_origin_community("1.2.3.4", 1000) ext.add_site_of_origin_community("10.20.30.40", 3000) - self.created_id = self.policy_api.create(self.profile_uuid, ext).id + self.created_id = self.policy_api.create_parcel(self.profile_uuid, ext).id parcel = self.policy_api.get(self.profile_uuid, ExtendedCommunityParcel, parcel_id=self.created_id) assert parcel.payload.parcel_name == "ExampleTestName" diff --git a/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py b/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py index e9c2af2c..a20ae949 100644 --- a/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py +++ b/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py @@ -57,7 +57,7 @@ def test_create_ssl_decryption_parcel(self): min_tls_ver="TLSv1.1", ) - self.created_id = self.policy_api.create(self.profile_uuid, ssl_decryption_parcel).id + self.created_id = self.policy_api.create_parcel(self.profile_uuid, ssl_decryption_parcel).id read_parcel = self.policy_api.get(self.profile_uuid, SslDecryptionParcel, parcel_id=self.created_id) assert read_parcel.payload.parcel_name == "test_ssl_profile" diff --git a/catalystwan/models/configuration/config_migration.py b/catalystwan/models/configuration/config_migration.py index ce55614a..3aaf96be 100644 --- a/catalystwan/models/configuration/config_migration.py +++ b/catalystwan/models/configuration/config_migration.py @@ -270,24 +270,25 @@ def list_transformed_parcels_with_origin(self, origin: Set[UUID]) -> List[Transf return [p for p in self.profile_parcels if p.header.origin in origin] def add_subelement_in_config_group( - self, profile_type: ProfileType, device_template_id: UUID, subelement: UUID + self, profile_types: List[ProfileType], device_template_id: UUID, subelement: UUID ) -> bool: profile_ids: Set[UUID] = set() + added = False for config_group in self.config_groups: if config_group.header.origin == device_template_id: profile_ids = config_group.header.subelements break if not profile_ids: - return False + return added for feature_profile in self.feature_profiles: - if feature_profile.header.type == profile_type and feature_profile.header.origin in profile_ids: + if feature_profile.header.type in profile_types and feature_profile.header.origin in profile_ids: head = feature_profile.header if head.localized_policy_subelements is None: head.localized_policy_subelements = {subelement} else: head.localized_policy_subelements.add(subelement) - return True - return False + added = True + return added class ConfigTransformResult(BaseModel): diff --git a/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py b/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py index 93608d94..c56da0ab 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py @@ -127,7 +127,7 @@ class MatchEntry(BaseModel): packet_length: Optional[Global[Union[str, int]]] = Field( default=None, validation_alias="packetLength", serialization_alias="packetLength" ) - source_data_prefix: Union[SourceDataPrefix, SourceDataPrefixList] = Field( + source_data_prefix: Union[SourceDataPrefix, SourceDataPrefixList, None] = Field( default=None, validation_alias="sourceDataPrefix", serialization_alias="sourceDataPrefix" ) source_ports: Optional[List[SourcePorts]] = Field( diff --git a/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py b/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py index 6a3ef4e9..ab102cb4 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py @@ -255,8 +255,8 @@ class RoutePolicySequence(BaseModel): ) base_action: Union[Global[AcceptRejectActionType], Default[AcceptRejectActionType]] = Field( default=as_default("reject", AcceptRejectActionType), - serialization_alias="routePolicyActionType", - validation_alias="routePolicyActionType", + serialization_alias="baseAction", + validation_alias="baseAction", description="Base Action", ) protocol: Union[Global[Protocol], Default[Protocol]] = Field( @@ -411,7 +411,7 @@ class RoutePolicyParcel(_ParcelBase): ) default_action: Union[Global[AcceptRejectActionType], Default[AcceptRejectActionType]] = Field( default=as_default("reject", AcceptRejectActionType), - validation_alias=AliasPath("data", "routePolicyActionType"), + validation_alias=AliasPath("data", "defaultAction"), description="Default Action", ) sequences: List[RoutePolicySequence] = Field( diff --git a/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py b/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py index fc78d9a0..ac637d74 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py @@ -6,6 +6,7 @@ from typing_extensions import Annotated from catalystwan.models.configuration.feature_profile.sdwan.acl import AnyAclParcel +from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel from .cellular_controller import CellularControllerParcel from .cellular_profile import CellularProfileParcel @@ -65,6 +66,7 @@ AnyTransportVpnParcel, AnyTransportVpnSubParcel, AnyManagementVpnSubParcel, + RoutePolicyParcel, ], Field(discriminator="type_"), ] diff --git a/catalystwan/models/policy/policy.py b/catalystwan/models/policy/policy.py index f3f358b6..4d7b11a6 100644 --- a/catalystwan/models/policy/policy.py +++ b/catalystwan/models/policy/policy.py @@ -77,10 +77,6 @@ class PolicyCreationPayload(BaseModel): default="default description", serialization_alias="policyDescription", validation_alias="policyDescription" ) policy_type: str = Field(serialization_alias="policyType", validation_alias="policyType") - policy_definition: Union[PolicyDefinition, str] = Field( - serialization_alias="policyDefinition", - validation_alias="policyDefinition", - ) is_policy_activated: bool = Field( default=False, serialization_alias="isPolicyActivated", validation_alias="isPolicyActivated" ) diff --git a/catalystwan/utils/config_migration/creators/config_pusher.py b/catalystwan/utils/config_migration/creators/config_pusher.py index 2bd1e157..b0ba81a2 100644 --- a/catalystwan/utils/config_migration/creators/config_pusher.py +++ b/catalystwan/utils/config_migration/creators/config_pusher.py @@ -53,7 +53,7 @@ def __init__( push_result=self._push_result, push_context=self._push_context, ) - self._localized_policy_feature_updater = LocalizedPolicyPusher( + self._localized_policy_feature_pusher = LocalizedPolicyPusher( ux2_config=ux2_config, session=session, progress=progress, @@ -81,7 +81,7 @@ def push(self) -> UX2ConfigPushResult: self._create_cloud_credentials() self._create_config_groups() self._groups_of_interests_pusher.push() - self._localized_policy_feature_updater.push() + self._localized_policy_feature_pusher.push() self._security_policy_pusher.push() self._create_topology_groups( self._push_context.default_policy_object_profile_id diff --git a/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py b/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py index b7c05983..3e548d0f 100644 --- a/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py +++ b/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py @@ -98,7 +98,7 @@ def push(self) -> None: try: parcel = update_parcel_references(parcel, self.push_context.id_lookup) - parcel_id = self._policy_object_api.create(profile_id=profile_id, payload=parcel).id + parcel_id = self._policy_object_api.create_parcel(profile_id=profile_id, payload=parcel).id profile_rollback.add_parcel(parcel.type_, parcel_id) self._push_result.report.groups_of_interest.add_created(parcel.parcel_name, parcel_id) self.push_context.id_lookup[transformed_parcel.header.origin] = parcel_id diff --git a/catalystwan/utils/config_migration/creators/localized_policy_pusher.py b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py index cb1151e5..a07a0c36 100644 --- a/catalystwan/utils/config_migration/creators/localized_policy_pusher.py +++ b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py @@ -1,45 +1,48 @@ import logging -from typing import Callable, Dict, List, Tuple, Union, cast +from typing import Callable, Dict, List, Literal, Tuple, Union, cast from uuid import UUID -from pydantic import Field +from pydantic import Field, ValidationError from typing_extensions import Annotated from catalystwan.api.builders.feature_profiles.report import FeatureProfileBuildReport from catalystwan.endpoints.configuration_group import ConfigGroup -from catalystwan.exceptions import ManagerHTTPError +from catalystwan.exceptions import ManagerErrorInfo, ManagerHTTPError from catalystwan.models.configuration.config_migration import ( PushContext, TransformedParcel, UX2Config, UX2ConfigPushResult, ) +from catalystwan.models.configuration.feature_profile.parcel import list_types from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv4acl import Ipv4AclParcel from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv6acl import Ipv6AclParcel from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel from catalystwan.models.configuration.feature_profile.sdwan.system.device_access import DeviceAccessIPv4Parcel from catalystwan.models.configuration.feature_profile.sdwan.system.device_access_ipv6 import DeviceAccessIPv6Parcel -from catalystwan.models.configuration.profile_type import ProfileType from catalystwan.session import ManagerSession from catalystwan.utils.config_migration.creators.references_updater import update_parcel_references -LOCALIZED_POLICY_PARCEL_TYPES = [ - "ipv4-device-access-policy", - "ipv6-device-access-policy", - "ipv4-acl", - "ipv6-acl", - "route", +_AnyTransportPolicyFeatureParcel = Annotated[ + Union[Ipv4AclParcel, Ipv6AclParcel, RoutePolicyParcel], + Field(discriminator="type_"), +] +_AnyServicePolicyFeatureParcel = Annotated[ + Union[Ipv4AclParcel, Ipv6AclParcel, RoutePolicyParcel], + Field(discriminator="type_"), ] -AnyDeviceAccessParcel = Annotated[ +_AnySystemPolicyFeatureParcel = Annotated[ Union[DeviceAccessIPv4Parcel, DeviceAccessIPv6Parcel], Field(discriminator="type_"), ] -AnyAclParcel = Annotated[ - Union[Ipv4AclParcel, Ipv6AclParcel], +_AnyLocalizedPolicyParcel = Annotated[ + Union[_AnyTransportPolicyFeatureParcel, _AnyServicePolicyFeatureParcel, _AnySystemPolicyFeatureParcel], Field(discriminator="type_"), ] +_LocalizedPolicyProfileTypes = Literal["transport", "service", "system"] +LOCALIZED_POLICY_PARCEL_TYPES = [t._get_parcel_type() for t in list_types(_AnyLocalizedPolicyParcel)] -ProfileInfo = Tuple[ProfileType, UUID, str, List[TransformedParcel]] +_ProfileInfo = Tuple[_LocalizedPolicyProfileTypes, UUID, str, List[TransformedParcel]] logger = logging.getLogger(__name__) @@ -47,7 +50,7 @@ class LocalizedPolicyPusher: """ 1. Associate selected Config Group with Default_Policy_Object_Profile - 2. Update selected Feature Profiles with Parcels originating from Localized Policy items (eg. acl, route) + 2. Update selected Feature Profiles by pushing Parcels originating from Localized Policy items (eg. acl, route) Update needs to be performed after Feature Profiles are already populated with VPN parcels and Default_Policy_Object_Profile is populated with Groups of Interest """ @@ -61,15 +64,11 @@ def __init__( progress: Callable[[str, int, int], None], ) -> None: self._ux2_config = ux2_config - self._fp_api = session.api.sdwan_feature_profiles + self._session = session self._cg_api = session.api.config_group - self._system_api = self._fp_api.system - self._transport_api = self._fp_api.transport - self._service_api = self._fp_api.service - self.dns = session.api.sdwan_feature_profiles.dns_security self._push_result: UX2ConfigPushResult = push_result + self._push_context = push_context self._progress: Callable[[str, int, int], None] = progress - self.push_context = push_context self._parcel_by_id = self._create_parcel_by_id_lookup() def _create_parcel_by_id_lookup(self) -> Dict[UUID, TransformedParcel]: @@ -98,7 +97,7 @@ def _find_config_groups_to_update(self) -> List[UUID]: result: List[UUID] = list() for transformed_cg in self._ux2_config.config_groups: if transformed_cg.header.localized_policy_subelements is not None: - updated_id = self.push_context.id_lookup[transformed_cg.header.origin] + updated_id = self._push_context.id_lookup[transformed_cg.header.origin] result.append(updated_id) return result @@ -109,37 +108,51 @@ def _get_config_group_contents(self, cg_ids: List[UUID]) -> Dict[UUID, ConfigGro result[cg_id] = cg return result - def _find_profiles_to_update(self) -> List[ProfileInfo]: - profiles: List[ProfileInfo] = list() + def _find_profiles_to_update(self) -> List[_ProfileInfo]: + profiles: List[_ProfileInfo] = list() for transformed_profile in self._ux2_config.feature_profiles: if transformed_profile.header.localized_policy_subelements is not None: - profile_type = cast(ProfileType, transformed_profile.header.type) + profile_type = cast(_LocalizedPolicyProfileTypes, transformed_profile.header.type) name = transformed_profile.feature_profile.name - updated_id = self.push_context.id_lookup[transformed_profile.header.origin] + updated_id = self._push_context.id_lookup[transformed_profile.header.origin] parcels = self._get_parcels_to_push(list(transformed_profile.header.localized_policy_subelements)) profiles.append((profile_type, updated_id, name, parcels)) return profiles - def update_system_profile( - self, profile_id: UUID, device_access: AnyDeviceAccessParcel, report: FeatureProfileBuildReport + def _update_transport_profile( + self, + profile_id: UUID, + parcel: _AnyTransportPolicyFeatureParcel, + ) -> UUID: + api = self._session.api.sdwan_feature_profiles.transport + parcel.parcel_name += "_transport" + return api.create_parcel(profile_id=profile_id, payload=parcel).id + + def _update_service_profile( + self, + profile_id: UUID, + parcel: _AnyServicePolicyFeatureParcel, + ): + api = self._session.api.sdwan_feature_profiles.service + parcel.parcel_name += "_service" + return api.create_parcel(profile_id=profile_id, payload=parcel).id + + def _update_system_profile( + self, + profile_id: UUID, + parcel: _AnySystemPolicyFeatureParcel, ): - try: - parcel_id = self._system_api.create_parcel( - profile_id=profile_id, payload=update_parcel_references(device_access, self.push_context.id_lookup) - ).id - report.add_created_parcel(parcel_name=device_access.parcel_name, parcel_uuid=parcel_id) - except ManagerHTTPError as e: - logger.error(f"Error occured during creation of {device_access.type_} {device_access.parcel_name}: {e}") - report.add_failed_parcel( - parcel_name=device_access.parcel_name, parcel_type=device_access.type_, error_info=e.info - ) + api = self._session.api.sdwan_feature_profiles.system + parcel.parcel_name += "_system" + return api.create_parcel(profile_id=profile_id, payload=parcel).id def associate_config_groups_with_default_policy_object_profile(self): for cg_id, cg in self._get_config_group_contents(self._find_config_groups_to_update()).items(): profile_ids = [p.id for p in cg.profiles] - profile_ids.append(self.push_context.default_policy_object_profile_id) + profile_ids.append(self._push_context.default_policy_object_profile_id) try: - self._cg_api.edit( + api = self._session.api.config_group + api.edit( cg_id=str(cg_id), name=cg.name, description=cg.description, @@ -160,14 +173,24 @@ def push(self): profile_type, profile_id, profile_name, transformed_parcels = profile_info self._progress(f"Updating {profile_name} profile with policy parcels", i + 1, len(profile_ids)) for transformed_parcel in transformed_parcels: - parcel = transformed_parcel.parcel - if profile_type == "system" and isinstance(parcel, (DeviceAccessIPv4Parcel, DeviceAccessIPv6Parcel)): - self.update_system_profile( - profile_id=profile_id, device_access=parcel, report=profile_reports[profile_id] + report = profile_reports[profile_id] + error_parcel = transformed_parcel.parcel + error_info: Union[ManagerErrorInfo, str, None] = None + try: + parcel_copy = update_parcel_references(transformed_parcel.parcel, self._push_context.id_lookup) + error_parcel = parcel_copy + if profile_type == "transport": + parcel_id = self._update_transport_profile(profile_id=profile_id, parcel=parcel_copy) + elif profile_type == "service": + parcel_id = self._update_service_profile(profile_id=profile_id, parcel=parcel_copy) + elif profile_type == "system": + parcel_id = self._update_system_profile(profile_id=profile_id, parcel=parcel_copy) + report.add_created_parcel(parcel_name=parcel_copy.parcel_name, parcel_uuid=parcel_id) + except ValidationError as validation_error: + error_info = str(validation_error) + except ManagerHTTPError as http_error: + error_info = http_error.info + if error_info is not None: + report.add_failed_parcel( + parcel_name=error_parcel.parcel_name, parcel_type=error_parcel.type_, error_info=error_info ) - elif profile_type == "service" and isinstance(parcel, RoutePolicyParcel): - logger.warning(f"not implemented: adding {parcel.parcel_name} parcel to profile: {profile_id}") - elif profile_type == "transport" and isinstance(parcel, (Ipv4AclParcel, Ipv6AclParcel)): - logger.warning(f"not implemented: adding {parcel.parcel_name} parcel to profile: {profile_id}") - else: - logger.warning(f"Unexpected profile type {profile_type} to add Localized Policy items, skipping") diff --git a/catalystwan/utils/config_migration/creators/references_updater.py b/catalystwan/utils/config_migration/creators/references_updater.py index b6a72c70..940246b6 100644 --- a/catalystwan/utils/config_migration/creators/references_updater.py +++ b/catalystwan/utils/config_migration/creators/references_updater.py @@ -3,7 +3,6 @@ from uuid import UUID from pydantic import BaseModel, ValidationError -from pydantic_core import from_json logger = logging.getLogger(__name__) @@ -11,8 +10,7 @@ def update_parcel_references(parcel: T, uuid_map: Dict[UUID, UUID]) -> T: - t = type(parcel) - origin_dump = target_dump = parcel.model_dump_json(by_alias=True) + target_dump = parcel.model_dump_json(by_alias=True) pattern = '"{}"' for origin_uuid, target_uuid in uuid_map.items(): @@ -20,11 +18,8 @@ def update_parcel_references(parcel: T, uuid_map: Dict[UUID, UUID]) -> T: target_uuid_str = pattern.format(str(target_uuid)) target_dump = target_dump.replace(origin_uuid_str, target_uuid_str) - if origin_dump == target_dump: - return parcel - try: - return t.model_validate(from_json(target_dump)) + return parcel.model_validate_json(target_dump) except ValidationError as e: logging.error(f"Cannot validate model after references update: {e}") raise e diff --git a/catalystwan/utils/config_migration/runner.py b/catalystwan/utils/config_migration/runner.py index 20611119..43ab3e22 100644 --- a/catalystwan/utils/config_migration/runner.py +++ b/catalystwan/utils/config_migration/runner.py @@ -105,10 +105,25 @@ def clear_ux2(self) -> None: self.progress("deleting other profiles...", 6, 12) fp_api.other.delete_all_profiles() - self.progress("deleting default policy object profile parcels...", 7, 12) + self.progress("deleting service profiles...", 7, 12) + fp_api.service.delete_all_profiles() + + self.progress("deleting sig security profiles...", 8, 12) + fp_api.sig_security.delete_all_profiles() + + self.progress("deleting system profiles...", 9, 12) + fp_api.system.delete_all_profiles() + + self.progress("deleting transport profiles...", 10, 12) + fp_api.transport.delete_all_profiles() + + self.progress("deleting topology profiles...", 11, 12) + fp_api.topology.delete_all_profiles() + + self.progress("deleting default policy object profile parcels...", 12, 12) po_profiles = fp_api.policy_object.get_profiles() if len(po_profiles) > 1: - print("WARNING MORE THAN ONE DEFAULT POLICY OBJECT PROFILE DETECTED") + print("WARNING! MORE THAN ONE DEFAULT POLICY OBJECT PROFILE DETECTED") for po_profile in po_profiles: sorted_parcel_types = sorted( @@ -124,21 +139,6 @@ def clear_ux2(self) -> None: parcel_uuid = UUID(str(parcel.parcel_id)) fp_api.policy_object.delete(po_profile.profile_id, type(parcel.payload), parcel_uuid) - self.progress("deleting service profiles...", 8, 12) - fp_api.service.delete_all_profiles() - - self.progress("deleting sig security profiles...", 9, 12) - fp_api.sig_security.delete_all_profiles() - - self.progress("deleting system profiles...", 10, 12) - fp_api.system.delete_all_profiles() - - self.progress("deleting transport profiles...", 11, 12) - fp_api.transport.delete_all_profiles() - - self.progress("deleting topology profiles...", 12, 12) - fp_api.topology.delete_all_profiles() - def run(self): with self.session.login() as session: # collext and dump ux1 to json file diff --git a/catalystwan/workflows/config_migration.py b/catalystwan/workflows/config_migration.py index 03037957..293ffb45 100644 --- a/catalystwan/workflows/config_migration.py +++ b/catalystwan/workflows/config_migration.py @@ -431,15 +431,19 @@ def transform(ux1: UX1Config, add_suffix: bool = True) -> ConfigTransformResult: for item in localized_policy.policy_definition.assembly: if item.type == "deviceaccesspolicy" or item.type == "deviceaccesspolicyv6": ux2.add_subelement_in_config_group( - profile_type="system", device_template_id=dt_id, subelement=item.definition_id + profile_types=["system"], device_template_id=dt_id, subelement=item.definition_id ) elif item.type == "acl" or item.type == "aclv6": ux2.add_subelement_in_config_group( - profile_type="transport", device_template_id=dt_id, subelement=item.definition_id + profile_types=["transport", "service"], + device_template_id=dt_id, + subelement=item.definition_id, ) elif item.type == "vedgeRoute": ux2.add_subelement_in_config_group( - profile_type="service", device_template_id=dt_id, subelement=item.definition_id + profile_types=["transport", "service"], + device_template_id=dt_id, + subelement=item.definition_id, ) # Security policies diff --git a/examples/parcel_configuration_guide.py b/examples/parcel_configuration_guide.py index 4fb5e388..4e7c2e45 100644 --- a/examples/parcel_configuration_guide.py +++ b/examples/parcel_configuration_guide.py @@ -255,7 +255,7 @@ def configure_groups_of_interest(profile_id: UUID, api: PolicyObjectFeatureProfi print(item.model_dump_json(by_alias=True, indent=4)) for item in items: - items_ids.append((api.create(profile_id, item), item.__class__)) + items_ids.append((api.create_parcel(profile_id, item), item.__class__)) id_, _ = items_ids[-1] @@ -263,7 +263,7 @@ def configure_groups_of_interest(profile_id: UUID, api: PolicyObjectFeatureProfi sla.add_entry(app_probe_class_id=id_.id, jitter=20, latency=50, loss=100) sla.add_fallback(criteria="jitter-latency-loss", latency_variance=10, jitter_variance=10, loss_variance=10) - items_ids.append((api.create(profile_id, sla), sla.__class__)) + items_ids.append((api.create_parcel(profile_id, sla), sla.__class__)) input("Press enter to delete...") From 27a8d9d93c3663d75c779b6b83dcb553b501aa98 Mon Sep 17 00:00:00 2001 From: Szymon Basan Date: Wed, 3 Jul 2024 22:46:08 +0200 Subject: [PATCH 4/4] fix: reference updater test --- .../policy_converters/test_references_updater.py | 2 +- .../utils/config_migration/creators/references_updater.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/catalystwan/tests/config_migration/policy_converters/test_references_updater.py b/catalystwan/tests/config_migration/policy_converters/test_references_updater.py index 9dc32611..bf196a92 100644 --- a/catalystwan/tests/config_migration/policy_converters/test_references_updater.py +++ b/catalystwan/tests/config_migration/policy_converters/test_references_updater.py @@ -57,4 +57,4 @@ def test_not_update_parcel_references(self): updated_parcel = update_parcel_references(aip_parcel, pushed_objects) - assert updated_parcel is aip_parcel + assert updated_parcel.model_dump_json() == aip_parcel.model_dump_json() diff --git a/catalystwan/utils/config_migration/creators/references_updater.py b/catalystwan/utils/config_migration/creators/references_updater.py index 940246b6..34759386 100644 --- a/catalystwan/utils/config_migration/creators/references_updater.py +++ b/catalystwan/utils/config_migration/creators/references_updater.py @@ -10,6 +10,11 @@ def update_parcel_references(parcel: T, uuid_map: Dict[UUID, UUID]) -> T: + """ + Replaces UUID strings found in json dump based on provided map + always returns a copy of original even when there was no substitution + always perform output validation + """ target_dump = parcel.model_dump_json(by_alias=True) pattern = '"{}"'