diff --git a/catalystwan/api/feature_profile_api.py b/catalystwan/api/feature_profile_api.py index 1ff6fa9b..b98167fc 100644 --- a/catalystwan/api/feature_profile_api.py +++ b/catalystwan/api/feature_profile_api.py @@ -46,6 +46,7 @@ from catalystwan.models.configuration.feature_profile.sdwan.routing import AnyRoutingParcel, RoutingBgpParcel from catalystwan.models.configuration.feature_profile.sdwan.service import AnyServiceParcel from catalystwan.models.configuration.feature_profile.sdwan.service.multicast import MulticastParcel +from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel from catalystwan.models.configuration.feature_profile.sdwan.sig_security.sig_security import SIGParcel from catalystwan.models.configuration.feature_profile.sdwan.topology import AnyTopologyParcel from catalystwan.models.configuration.feature_profile.sdwan.topology.custom_control import CustomControlParcel @@ -306,6 +307,12 @@ def get_parcel(self, profile_id: UUID, parcel_type: Type[Ipv4AclParcel], parcel_ def get_parcel(self, profile_id: UUID, parcel_type: Type[Ipv6AclParcel], parcel_id: UUID) -> Parcel[Ipv6AclParcel]: ... + @overload + def get_parcel( + self, profile_id: UUID, parcel_type: Type[RoutePolicyParcel], parcel_id: UUID + ) -> Parcel[RoutePolicyParcel]: + ... + def get_parcel( self, profile_id: UUID, parcel_type: Type[Union[AnyTransportParcel, AnyRoutingParcel]], parcel_id: UUID ) -> Parcel: @@ -431,21 +438,21 @@ def delete_all_profiles(self) -> None: self.delete_profile(profile.profile_id) def create_parcel( - self, profile_uuid: UUID, payload: AnyServiceParcel, vpn_uuid: Optional[UUID] = None + self, profile_id: UUID, payload: AnyServiceParcel, vpn_uuid: Optional[UUID] = None ) -> ParcelCreationResponse: """ Create Service Parcel for selected profile_id based on payload type """ if vpn_uuid is not None: if isinstance(payload, MulticastParcel): - response = self.endpoint.create_service_parcel(profile_uuid, payload._get_parcel_type(), payload) + response = self.endpoint.create_service_parcel(profile_id, payload._get_parcel_type(), payload) return self.endpoint.associate_parcel_with_vpn( - profile_uuid, vpn_uuid, payload._get_parcel_type(), ParcelAssociationPayload(parcel_id=response.id) + profile_id, vpn_uuid, payload._get_parcel_type(), ParcelAssociationPayload(parcel_id=response.id) ) else: parcel_type = payload._get_parcel_type().replace("lan/vpn/", "") - return self.endpoint.create_lan_vpn_sub_parcel(profile_uuid, vpn_uuid, parcel_type, payload) - return self.endpoint.create_service_parcel(profile_uuid, payload._get_parcel_type(), payload) + return self.endpoint.create_lan_vpn_sub_parcel(profile_id, vpn_uuid, parcel_type, payload) + return self.endpoint.create_service_parcel(profile_id, payload._get_parcel_type(), payload) def delete_parcel(self, profile_uuid: UUID, parcel_type: Type[AnyServiceParcel], parcel_uuid: UUID) -> None: """ @@ -1194,7 +1201,7 @@ def get( profile_id=profile_id, policy_object_list_type=policy_object_list_type, list_object_id=parcel_id ) - def create(self, profile_id: UUID, payload: AnyPolicyObjectParcel) -> ParcelCreationResponse: + def create_parcel(self, profile_id: UUID, payload: AnyPolicyObjectParcel) -> ParcelCreationResponse: """ Create Policy Object for selected profile_id based on payload type """ diff --git a/catalystwan/api/templates/device_template/device_template.py b/catalystwan/api/templates/device_template/device_template.py index bf803d31..39c2067b 100644 --- a/catalystwan/api/templates/device_template/device_template.py +++ b/catalystwan/api/templates/device_template/device_template.py @@ -4,7 +4,8 @@ import logging from pathlib import Path -from typing import TYPE_CHECKING, ClassVar, List +from typing import TYPE_CHECKING, ClassVar, List, Optional +from uuid import UUID from jinja2 import DebugUndefined, Environment, FileSystemLoader, meta # type: ignore from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -15,6 +16,13 @@ logger = logging.getLogger(__name__) +def str_to_uuid(s: str) -> Optional[UUID]: + try: + return UUID(s) + except ValueError: + return None + + class GeneralTemplate(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) @@ -54,6 +62,12 @@ class DeviceTemplate(BaseModel): ) policy_id: str = Field(default="", serialization_alias="policyId", validation_alias="policyId") + def get_security_policy_uuid(self) -> Optional[UUID]: + return str_to_uuid(self.security_policy_id) + + def get_policy_uuid(self) -> Optional[UUID]: + return str_to_uuid(self.policy_id) + def generate_payload(self) -> str: env = Environment( loader=FileSystemLoader(self.payload_path.parent), diff --git a/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py b/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py index d8d06d72..01a3879f 100644 --- a/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py +++ b/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py @@ -42,7 +42,7 @@ def test_create_extended_community_parcel(self): ext.add_site_of_origin_community("1.2.3.4", 1000) ext.add_site_of_origin_community("10.20.30.40", 3000) - self.created_id = self.policy_api.create(self.profile_uuid, ext).id + self.created_id = self.policy_api.create_parcel(self.profile_uuid, ext).id parcel = self.policy_api.get(self.profile_uuid, ExtendedCommunityParcel, parcel_id=self.created_id) assert parcel.payload.parcel_name == "ExampleTestName" diff --git a/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py b/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py index e9c2af2c..a20ae949 100644 --- a/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py +++ b/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py @@ -57,7 +57,7 @@ def test_create_ssl_decryption_parcel(self): min_tls_ver="TLSv1.1", ) - self.created_id = self.policy_api.create(self.profile_uuid, ssl_decryption_parcel).id + self.created_id = self.policy_api.create_parcel(self.profile_uuid, ssl_decryption_parcel).id read_parcel = self.policy_api.get(self.profile_uuid, SslDecryptionParcel, parcel_id=self.created_id) assert read_parcel.payload.parcel_name == "test_ssl_profile" diff --git a/catalystwan/models/configuration/config_migration.py b/catalystwan/models/configuration/config_migration.py index 2ee6f992..3aaf96be 100644 --- a/catalystwan/models/configuration/config_migration.py +++ b/catalystwan/models/configuration/config_migration.py @@ -137,6 +137,17 @@ class UX1Templates(BaseModel): validation_alias="deviceTemplates", ) + def create_device_template_by_policy_id_lookup(self) -> Dict[Literal["policy", "security"], Dict[UUID, UUID]]: + lookup: Dict[Literal["policy", "security"], Dict[UUID, UUID]] = {"policy": {}, "security": {}} + for dt in self.device_templates: + policy_id = dt.get_policy_uuid() + security_policy_id = dt.get_security_policy_uuid() + if policy_id is not None: + lookup["policy"][policy_id] = UUID(dt.template_id) + if security_policy_id is not None: + lookup["security"][security_policy_id] = UUID(dt.template_id) + return lookup + class UX1Config(BaseModel): # All UX1 Configuration items - Mega Model @@ -150,6 +161,7 @@ class UX1Config(BaseModel): class TransformHeader(BaseModel): + model_config = ConfigDict(populate_by_name=True) type: str = Field( description="Needed to push item to specific endpoint." "Type discriminator is not present in many UX2 item payloads" @@ -157,6 +169,9 @@ class TransformHeader(BaseModel): origin: UUID = Field(description="Original UUID of converted item") origname: Optional[str] = None subelements: Set[UUID] = Field(default_factory=set) + localized_policy_subelements: Optional[Set[UUID]] = Field( + default=None, serialization_alias="localizedPolicySubelements", validation_alias="localizedPolicySubelements" + ) status: ConvertOutputStatus = Field(default="complete") info: List[str] = Field(default_factory=list) @@ -251,9 +266,30 @@ def insert_parcel_type_from_headers(cls, values: Dict[str, Any]): profile_parcel["parcel"]["type_"] = profile_parcel["header"]["type"] return values - def transformed_parcels_with_origin(self, origin: Set[UUID]) -> List[TransformedParcel]: + def list_transformed_parcels_with_origin(self, origin: Set[UUID]) -> List[TransformedParcel]: return [p for p in self.profile_parcels if p.header.origin in origin] + def add_subelement_in_config_group( + self, profile_types: List[ProfileType], device_template_id: UUID, subelement: UUID + ) -> bool: + profile_ids: Set[UUID] = set() + added = False + for config_group in self.config_groups: + if config_group.header.origin == device_template_id: + profile_ids = config_group.header.subelements + break + if not profile_ids: + return added + for feature_profile in self.feature_profiles: + if feature_profile.header.type in profile_types and feature_profile.header.origin in profile_ids: + head = feature_profile.header + if head.localized_policy_subelements is None: + head.localized_policy_subelements = {subelement} + else: + head.localized_policy_subelements.add(subelement) + added = True + return added + class ConfigTransformResult(BaseModel): # https://docs.pydantic.dev/2.0/usage/models/#fields-with-dynamic-default-values diff --git a/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py b/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py index 93608d94..c56da0ab 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py @@ -127,7 +127,7 @@ class MatchEntry(BaseModel): packet_length: Optional[Global[Union[str, int]]] = Field( default=None, validation_alias="packetLength", serialization_alias="packetLength" ) - source_data_prefix: Union[SourceDataPrefix, SourceDataPrefixList] = Field( + source_data_prefix: Union[SourceDataPrefix, SourceDataPrefixList, None] = Field( default=None, validation_alias="sourceDataPrefix", serialization_alias="sourceDataPrefix" ) source_ports: Optional[List[SourcePorts]] = Field( diff --git a/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py b/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py index 6a3ef4e9..ab102cb4 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py @@ -255,8 +255,8 @@ class RoutePolicySequence(BaseModel): ) base_action: Union[Global[AcceptRejectActionType], Default[AcceptRejectActionType]] = Field( default=as_default("reject", AcceptRejectActionType), - serialization_alias="routePolicyActionType", - validation_alias="routePolicyActionType", + serialization_alias="baseAction", + validation_alias="baseAction", description="Base Action", ) protocol: Union[Global[Protocol], Default[Protocol]] = Field( @@ -411,7 +411,7 @@ class RoutePolicyParcel(_ParcelBase): ) default_action: Union[Global[AcceptRejectActionType], Default[AcceptRejectActionType]] = Field( default=as_default("reject", AcceptRejectActionType), - validation_alias=AliasPath("data", "routePolicyActionType"), + validation_alias=AliasPath("data", "defaultAction"), description="Default Action", ) sequences: List[RoutePolicySequence] = Field( diff --git a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py index ad2c81e9..1fa19137 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py @@ -64,8 +64,8 @@ class Sequence(BaseModel): base_action: Union[Global[AcceptDropActionType], Default[Literal[AcceptDropActionType]]] = Field( default=Default[Literal[AcceptDropActionType]](value="accept"), - validation_alias="BasicPolicyActionType", - serialization_alias="BasicPolicyActionType", + validation_alias="baseAction", + serialization_alias="baseAction", ) match_entries: MatchEntries = Field( validation_alias="matchEntries", serialization_alias="matchEntries", description="Define match conditions" diff --git a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py index 74cacde9..22085fc0 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py @@ -64,8 +64,8 @@ class Sequence(BaseModel): base_action: Union[Global[AcceptDropActionType], Default[Literal[AcceptDropActionType]]] = Field( default=Default[Literal[AcceptDropActionType]](value="accept"), - validation_alias="BasicPolicyActionType", - serialization_alias="BasicPolicyActionType", + validation_alias="baseAction", + serialization_alias="baseAction", ) match_entries: MatchEntries = Field( validation_alias="matchEntries", serialization_alias="matchEntries", description="Define match conditions" diff --git a/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py b/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py index fc78d9a0..ac637d74 100644 --- a/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py +++ b/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py @@ -6,6 +6,7 @@ from typing_extensions import Annotated from catalystwan.models.configuration.feature_profile.sdwan.acl import AnyAclParcel +from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel from .cellular_controller import CellularControllerParcel from .cellular_profile import CellularProfileParcel @@ -65,6 +66,7 @@ AnyTransportVpnParcel, AnyTransportVpnSubParcel, AnyManagementVpnSubParcel, + RoutePolicyParcel, ], Field(discriminator="type_"), ] diff --git a/catalystwan/models/policy/policy.py b/catalystwan/models/policy/policy.py index f3f358b6..4d7b11a6 100644 --- a/catalystwan/models/policy/policy.py +++ b/catalystwan/models/policy/policy.py @@ -77,10 +77,6 @@ class PolicyCreationPayload(BaseModel): default="default description", serialization_alias="policyDescription", validation_alias="policyDescription" ) policy_type: str = Field(serialization_alias="policyType", validation_alias="policyType") - policy_definition: Union[PolicyDefinition, str] = Field( - serialization_alias="policyDefinition", - validation_alias="policyDefinition", - ) is_policy_activated: bool = Field( default=False, serialization_alias="isPolicyActivated", validation_alias="isPolicyActivated" ) diff --git a/catalystwan/tests/config_migration/policy_converters/test_references_updater.py b/catalystwan/tests/config_migration/policy_converters/test_references_updater.py index 9dc32611..bf196a92 100644 --- a/catalystwan/tests/config_migration/policy_converters/test_references_updater.py +++ b/catalystwan/tests/config_migration/policy_converters/test_references_updater.py @@ -57,4 +57,4 @@ def test_not_update_parcel_references(self): updated_parcel = update_parcel_references(aip_parcel, pushed_objects) - assert updated_parcel is aip_parcel + assert updated_parcel.model_dump_json() == aip_parcel.model_dump_json() diff --git a/catalystwan/utils/config_migration/creators/config_pusher.py b/catalystwan/utils/config_migration/creators/config_pusher.py index b8178848..b0ba81a2 100644 --- a/catalystwan/utils/config_migration/creators/config_pusher.py +++ b/catalystwan/utils/config_migration/creators/config_pusher.py @@ -22,6 +22,7 @@ from catalystwan.models.configuration.feature_profile.sdwan.topology.mesh import MeshParcel from catalystwan.session import ManagerSession from catalystwan.utils.config_migration.creators.groups_of_interests_pusher import GroupsOfInterestPusher +from catalystwan.utils.config_migration.creators.localized_policy_pusher import LocalizedPolicyPusher from catalystwan.utils.config_migration.creators.security_policy_pusher import SecurityPolicyPusher from catalystwan.utils.config_migration.factories.parcel_pusher import ParcelPusherFactory @@ -52,6 +53,13 @@ def __init__( push_result=self._push_result, push_context=self._push_context, ) + self._localized_policy_feature_pusher = LocalizedPolicyPusher( + ux2_config=ux2_config, + session=session, + progress=progress, + push_result=self._push_result, + push_context=self._push_context, + ) self._security_policy_pusher = SecurityPolicyPusher( ux2_config=ux2_config, session=session, @@ -73,6 +81,7 @@ def push(self) -> UX2ConfigPushResult: self._create_cloud_credentials() self._create_config_groups() self._groups_of_interests_pusher.push() + self._localized_policy_feature_pusher.push() self._security_policy_pusher.push() self._create_topology_groups( self._push_context.default_policy_object_profile_id @@ -144,6 +153,7 @@ def _create_feature_profile_and_parcels(self, feature_profiles_ids: List[UUID]) profile = pusher.push(transformed_feature_profile.feature_profile, parcels, self._config_map.parcel_map) feature_profiles.append(profile) self._push_result.rollback.add_feature_profile(profile.profile_uuid, profile_type) + self._push_context.id_lookup[feature_profile_id] = profile.profile_uuid except ManagerHTTPError as e: logger.error(f"Error occured during [{fp_name}] feature profile creation: {e}") except Exception: @@ -205,7 +215,7 @@ def _create_topology_groups(self, default_policy_object_profile_id: Optional[UUI logger.error(f"Error occured during topology group creation: {e}") continue - for transformed_parcel in self._ux2_config.transformed_parcels_with_origin(origins): + for transformed_parcel in self._ux2_config.list_transformed_parcels_with_origin(origins): parcel = transformed_parcel.parcel if isinstance(parcel, (CustomControlParcel, HubSpokeParcel, MeshParcel)): try: diff --git a/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py b/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py index b7c05983..3e548d0f 100644 --- a/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py +++ b/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py @@ -98,7 +98,7 @@ def push(self) -> None: try: parcel = update_parcel_references(parcel, self.push_context.id_lookup) - parcel_id = self._policy_object_api.create(profile_id=profile_id, payload=parcel).id + parcel_id = self._policy_object_api.create_parcel(profile_id=profile_id, payload=parcel).id profile_rollback.add_parcel(parcel.type_, parcel_id) self._push_result.report.groups_of_interest.add_created(parcel.parcel_name, parcel_id) self.push_context.id_lookup[transformed_parcel.header.origin] = parcel_id diff --git a/catalystwan/utils/config_migration/creators/localized_policy_pusher.py b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py new file mode 100644 index 00000000..a07a0c36 --- /dev/null +++ b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py @@ -0,0 +1,196 @@ +import logging +from typing import Callable, Dict, List, Literal, Tuple, Union, cast +from uuid import UUID + +from pydantic import Field, ValidationError +from typing_extensions import Annotated + +from catalystwan.api.builders.feature_profiles.report import FeatureProfileBuildReport +from catalystwan.endpoints.configuration_group import ConfigGroup +from catalystwan.exceptions import ManagerErrorInfo, ManagerHTTPError +from catalystwan.models.configuration.config_migration import ( + PushContext, + TransformedParcel, + UX2Config, + UX2ConfigPushResult, +) +from catalystwan.models.configuration.feature_profile.parcel import list_types +from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv4acl import Ipv4AclParcel +from catalystwan.models.configuration.feature_profile.sdwan.acl.ipv6acl import Ipv6AclParcel +from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel +from catalystwan.models.configuration.feature_profile.sdwan.system.device_access import DeviceAccessIPv4Parcel +from catalystwan.models.configuration.feature_profile.sdwan.system.device_access_ipv6 import DeviceAccessIPv6Parcel +from catalystwan.session import ManagerSession +from catalystwan.utils.config_migration.creators.references_updater import update_parcel_references + +_AnyTransportPolicyFeatureParcel = Annotated[ + Union[Ipv4AclParcel, Ipv6AclParcel, RoutePolicyParcel], + Field(discriminator="type_"), +] +_AnyServicePolicyFeatureParcel = Annotated[ + Union[Ipv4AclParcel, Ipv6AclParcel, RoutePolicyParcel], + Field(discriminator="type_"), +] +_AnySystemPolicyFeatureParcel = Annotated[ + Union[DeviceAccessIPv4Parcel, DeviceAccessIPv6Parcel], + Field(discriminator="type_"), +] +_AnyLocalizedPolicyParcel = Annotated[ + Union[_AnyTransportPolicyFeatureParcel, _AnyServicePolicyFeatureParcel, _AnySystemPolicyFeatureParcel], + Field(discriminator="type_"), +] +_LocalizedPolicyProfileTypes = Literal["transport", "service", "system"] +LOCALIZED_POLICY_PARCEL_TYPES = [t._get_parcel_type() for t in list_types(_AnyLocalizedPolicyParcel)] + +_ProfileInfo = Tuple[_LocalizedPolicyProfileTypes, UUID, str, List[TransformedParcel]] + +logger = logging.getLogger(__name__) + + +class LocalizedPolicyPusher: + """ + 1. Associate selected Config Group with Default_Policy_Object_Profile + 2. Update selected Feature Profiles by pushing Parcels originating from Localized Policy items (eg. acl, route) + Update needs to be performed after Feature Profiles are already populated with VPN parcels + and Default_Policy_Object_Profile is populated with Groups of Interest + """ + + def __init__( + self, + ux2_config: UX2Config, + session: ManagerSession, + push_result: UX2ConfigPushResult, + push_context: PushContext, + progress: Callable[[str, int, int], None], + ) -> None: + self._ux2_config = ux2_config + self._session = session + self._cg_api = session.api.config_group + self._push_result: UX2ConfigPushResult = push_result + self._push_context = push_context + self._progress: Callable[[str, int, int], None] = progress + self._parcel_by_id = self._create_parcel_by_id_lookup() + + def _create_parcel_by_id_lookup(self) -> Dict[UUID, TransformedParcel]: + lookup: Dict[UUID, TransformedParcel] = dict() + for transformed_parcel in self._ux2_config.profile_parcels: + if transformed_parcel.header.type in LOCALIZED_POLICY_PARCEL_TYPES: + lookup[transformed_parcel.header.origin] = transformed_parcel + return lookup + + def _create_profile_report_by_id_lookup(self, profile_ids: List[UUID]) -> Dict[UUID, FeatureProfileBuildReport]: + lookup: Dict[UUID, FeatureProfileBuildReport] = dict() + for cg_report in self._push_result.report.config_groups: + for fp_report in cg_report.feature_profiles: + if fp_report.profile_uuid in profile_ids: + lookup[fp_report.profile_uuid] = fp_report + return lookup + + def _get_parcels_to_push(self, parcel_ids: List[UUID]) -> List[TransformedParcel]: + result: List[TransformedParcel] = list() + for parcel_id in parcel_ids: + if parcel := self._parcel_by_id.get(parcel_id): + result.append(parcel) + return result + + def _find_config_groups_to_update(self) -> List[UUID]: + result: List[UUID] = list() + for transformed_cg in self._ux2_config.config_groups: + if transformed_cg.header.localized_policy_subelements is not None: + updated_id = self._push_context.id_lookup[transformed_cg.header.origin] + result.append(updated_id) + return result + + def _get_config_group_contents(self, cg_ids: List[UUID]) -> Dict[UUID, ConfigGroup]: + result: Dict[UUID, ConfigGroup] = dict() + for cg_id in cg_ids: + cg = self._cg_api.get(cg_id) + result[cg_id] = cg + return result + + def _find_profiles_to_update(self) -> List[_ProfileInfo]: + profiles: List[_ProfileInfo] = list() + for transformed_profile in self._ux2_config.feature_profiles: + if transformed_profile.header.localized_policy_subelements is not None: + profile_type = cast(_LocalizedPolicyProfileTypes, transformed_profile.header.type) + name = transformed_profile.feature_profile.name + updated_id = self._push_context.id_lookup[transformed_profile.header.origin] + parcels = self._get_parcels_to_push(list(transformed_profile.header.localized_policy_subelements)) + profiles.append((profile_type, updated_id, name, parcels)) + return profiles + + def _update_transport_profile( + self, + profile_id: UUID, + parcel: _AnyTransportPolicyFeatureParcel, + ) -> UUID: + api = self._session.api.sdwan_feature_profiles.transport + parcel.parcel_name += "_transport" + return api.create_parcel(profile_id=profile_id, payload=parcel).id + + def _update_service_profile( + self, + profile_id: UUID, + parcel: _AnyServicePolicyFeatureParcel, + ): + api = self._session.api.sdwan_feature_profiles.service + parcel.parcel_name += "_service" + return api.create_parcel(profile_id=profile_id, payload=parcel).id + + def _update_system_profile( + self, + profile_id: UUID, + parcel: _AnySystemPolicyFeatureParcel, + ): + api = self._session.api.sdwan_feature_profiles.system + parcel.parcel_name += "_system" + return api.create_parcel(profile_id=profile_id, payload=parcel).id + + def associate_config_groups_with_default_policy_object_profile(self): + for cg_id, cg in self._get_config_group_contents(self._find_config_groups_to_update()).items(): + profile_ids = [p.id for p in cg.profiles] + profile_ids.append(self._push_context.default_policy_object_profile_id) + try: + api = self._session.api.config_group + api.edit( + cg_id=str(cg_id), + name=cg.name, + description=cg.description, + solution=cg.solution, + profile_ids=profile_ids, + ) + except ManagerHTTPError as e: + logger.error(f"Error occured during config group edit: {e}") + + def push(self): + self._progress("Associating Config Groups with Default Policy Object Profile", 0, 1) + self.associate_config_groups_with_default_policy_object_profile() + self._progress("Associating Config Groups with Default Policy Object Profile", 1, 1) + profile_infos = self._find_profiles_to_update() + profile_ids = [t[1] for t in profile_infos] + profile_reports = self._create_profile_report_by_id_lookup(profile_ids) + for i, profile_info in enumerate(profile_infos): + profile_type, profile_id, profile_name, transformed_parcels = profile_info + self._progress(f"Updating {profile_name} profile with policy parcels", i + 1, len(profile_ids)) + for transformed_parcel in transformed_parcels: + report = profile_reports[profile_id] + error_parcel = transformed_parcel.parcel + error_info: Union[ManagerErrorInfo, str, None] = None + try: + parcel_copy = update_parcel_references(transformed_parcel.parcel, self._push_context.id_lookup) + error_parcel = parcel_copy + if profile_type == "transport": + parcel_id = self._update_transport_profile(profile_id=profile_id, parcel=parcel_copy) + elif profile_type == "service": + parcel_id = self._update_service_profile(profile_id=profile_id, parcel=parcel_copy) + elif profile_type == "system": + parcel_id = self._update_system_profile(profile_id=profile_id, parcel=parcel_copy) + report.add_created_parcel(parcel_name=parcel_copy.parcel_name, parcel_uuid=parcel_id) + except ValidationError as validation_error: + error_info = str(validation_error) + except ManagerHTTPError as http_error: + error_info = http_error.info + if error_info is not None: + report.add_failed_parcel( + parcel_name=error_parcel.parcel_name, parcel_type=error_parcel.type_, error_info=error_info + ) diff --git a/catalystwan/utils/config_migration/creators/references_updater.py b/catalystwan/utils/config_migration/creators/references_updater.py index b6a72c70..34759386 100644 --- a/catalystwan/utils/config_migration/creators/references_updater.py +++ b/catalystwan/utils/config_migration/creators/references_updater.py @@ -3,7 +3,6 @@ from uuid import UUID from pydantic import BaseModel, ValidationError -from pydantic_core import from_json logger = logging.getLogger(__name__) @@ -11,8 +10,12 @@ def update_parcel_references(parcel: T, uuid_map: Dict[UUID, UUID]) -> T: - t = type(parcel) - origin_dump = target_dump = parcel.model_dump_json(by_alias=True) + """ + Replaces UUID strings found in json dump based on provided map + always returns a copy of original even when there was no substitution + always perform output validation + """ + target_dump = parcel.model_dump_json(by_alias=True) pattern = '"{}"' for origin_uuid, target_uuid in uuid_map.items(): @@ -20,11 +23,8 @@ def update_parcel_references(parcel: T, uuid_map: Dict[UUID, UUID]) -> T: target_uuid_str = pattern.format(str(target_uuid)) target_dump = target_dump.replace(origin_uuid_str, target_uuid_str) - if origin_dump == target_dump: - return parcel - try: - return t.model_validate(from_json(target_dump)) + return parcel.model_validate_json(target_dump) except ValidationError as e: logging.error(f"Cannot validate model after references update: {e}") raise e diff --git a/catalystwan/utils/config_migration/runner.py b/catalystwan/utils/config_migration/runner.py index 20611119..43ab3e22 100644 --- a/catalystwan/utils/config_migration/runner.py +++ b/catalystwan/utils/config_migration/runner.py @@ -105,10 +105,25 @@ def clear_ux2(self) -> None: self.progress("deleting other profiles...", 6, 12) fp_api.other.delete_all_profiles() - self.progress("deleting default policy object profile parcels...", 7, 12) + self.progress("deleting service profiles...", 7, 12) + fp_api.service.delete_all_profiles() + + self.progress("deleting sig security profiles...", 8, 12) + fp_api.sig_security.delete_all_profiles() + + self.progress("deleting system profiles...", 9, 12) + fp_api.system.delete_all_profiles() + + self.progress("deleting transport profiles...", 10, 12) + fp_api.transport.delete_all_profiles() + + self.progress("deleting topology profiles...", 11, 12) + fp_api.topology.delete_all_profiles() + + self.progress("deleting default policy object profile parcels...", 12, 12) po_profiles = fp_api.policy_object.get_profiles() if len(po_profiles) > 1: - print("WARNING MORE THAN ONE DEFAULT POLICY OBJECT PROFILE DETECTED") + print("WARNING! MORE THAN ONE DEFAULT POLICY OBJECT PROFILE DETECTED") for po_profile in po_profiles: sorted_parcel_types = sorted( @@ -124,21 +139,6 @@ def clear_ux2(self) -> None: parcel_uuid = UUID(str(parcel.parcel_id)) fp_api.policy_object.delete(po_profile.profile_id, type(parcel.payload), parcel_uuid) - self.progress("deleting service profiles...", 8, 12) - fp_api.service.delete_all_profiles() - - self.progress("deleting sig security profiles...", 9, 12) - fp_api.sig_security.delete_all_profiles() - - self.progress("deleting system profiles...", 10, 12) - fp_api.system.delete_all_profiles() - - self.progress("deleting transport profiles...", 11, 12) - fp_api.transport.delete_all_profiles() - - self.progress("deleting topology profiles...", 12, 12) - fp_api.topology.delete_all_profiles() - def run(self): with self.session.login() as session: # collext and dump ux1 to json file diff --git a/catalystwan/workflows/config_migration.py b/catalystwan/workflows/config_migration.py index fc123c4b..293ffb45 100644 --- a/catalystwan/workflows/config_migration.py +++ b/catalystwan/workflows/config_migration.py @@ -329,7 +329,7 @@ def transform(ux1: UX1Config, add_suffix: bool = True) -> ConfigTransformResult: subtemplates_mapping[UUID(subtemplate_level_1.templateId)].add( UUID(subtemplate_level_2.templateId) ) - + policy_id = dt.get_policy_uuid() transformed_cg = TransformedConfigGroup( header=TransformHeader( type="config_group", @@ -337,6 +337,7 @@ def transform(ux1: UX1Config, add_suffix: bool = True) -> ConfigTransformResult: subelements=set( [fp_system_uuid, fp_other_uuid, fp_service_uuid, fp_transport_and_management_uuid, fp_cli_uuid] ), + localized_policy_subelements=set([policy_id]) if policy_id else None, ), config_group=ConfigGroupCreationPayload( name=dt.template_name, @@ -421,6 +422,30 @@ def transform(ux1: UX1Config, add_suffix: bool = True) -> ConfigTransformResult: ) ux2.profile_parcels.append(TransformedParcel(header=header, parcel=pd_parcel)) + # Localized Policies + _lookup = ux1.templates.create_device_template_by_policy_id_lookup() + dt_by_policy_id = _lookup["policy"] + for localized_policy in ux1.policies.localized_policies: + if not isinstance(localized_policy.policy_definition, str): + if dt_id := dt_by_policy_id.get(localized_policy.policy_id): + for item in localized_policy.policy_definition.assembly: + if item.type == "deviceaccesspolicy" or item.type == "deviceaccesspolicyv6": + ux2.add_subelement_in_config_group( + profile_types=["system"], device_template_id=dt_id, subelement=item.definition_id + ) + elif item.type == "acl" or item.type == "aclv6": + ux2.add_subelement_in_config_group( + profile_types=["transport", "service"], + device_template_id=dt_id, + subelement=item.definition_id, + ) + elif item.type == "vedgeRoute": + ux2.add_subelement_in_config_group( + profile_types=["transport", "service"], + device_template_id=dt_id, + subelement=item.definition_id, + ) + # Security policies for security_policy in ux1.policies.security_policies: sp_result = convert_security_policy(security_policy, security_policy.policy_id, policy_context) diff --git a/examples/parcel_configuration_guide.py b/examples/parcel_configuration_guide.py index 4fb5e388..4e7c2e45 100644 --- a/examples/parcel_configuration_guide.py +++ b/examples/parcel_configuration_guide.py @@ -255,7 +255,7 @@ def configure_groups_of_interest(profile_id: UUID, api: PolicyObjectFeatureProfi print(item.model_dump_json(by_alias=True, indent=4)) for item in items: - items_ids.append((api.create(profile_id, item), item.__class__)) + items_ids.append((api.create_parcel(profile_id, item), item.__class__)) id_, _ = items_ids[-1] @@ -263,7 +263,7 @@ def configure_groups_of_interest(profile_id: UUID, api: PolicyObjectFeatureProfi sla.add_entry(app_probe_class_id=id_.id, jitter=20, latency=50, loss=100) sla.add_fallback(criteria="jitter-latency-loss", latency_variance=10, jitter_variance=10, loss_variance=10) - items_ids.append((api.create(profile_id, sla), sla.__class__)) + items_ids.append((api.create_parcel(profile_id, sla), sla.__class__)) input("Press enter to delete...")