dev-uxmt: push converted localized policy elements to config group pr…

…ofile parcels (#747) * draft: system profile update with device access parcel is working * refactor * all items working and reported * fix: reference updater test
cisco-open · Jul 3, 2024 · a9ba557 · a9ba557
1 parent 12d21df
commit a9ba557
Show file tree

Hide file tree

Showing 19 changed files with 338 additions and 52 deletions.
diff --git a/catalystwan/api/feature_profile_api.py b/catalystwan/api/feature_profile_api.py
@@ -46,6 +46,7 @@
 from catalystwan.models.configuration.feature_profile.sdwan.routing import AnyRoutingParcel, RoutingBgpParcel
 from catalystwan.models.configuration.feature_profile.sdwan.service import AnyServiceParcel
 from catalystwan.models.configuration.feature_profile.sdwan.service.multicast import MulticastParcel
+from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel
 from catalystwan.models.configuration.feature_profile.sdwan.sig_security.sig_security import SIGParcel
 from catalystwan.models.configuration.feature_profile.sdwan.topology import AnyTopologyParcel
 from catalystwan.models.configuration.feature_profile.sdwan.topology.custom_control import CustomControlParcel
@@ -306,6 +307,12 @@ def get_parcel(self, profile_id: UUID, parcel_type: Type[Ipv4AclParcel], parcel_
     def get_parcel(self, profile_id: UUID, parcel_type: Type[Ipv6AclParcel], parcel_id: UUID) -> Parcel[Ipv6AclParcel]:
         ...
 
+    @overload
+    def get_parcel(
+        self, profile_id: UUID, parcel_type: Type[RoutePolicyParcel], parcel_id: UUID
+    ) -> Parcel[RoutePolicyParcel]:
+        ...
+
     def get_parcel(
         self, profile_id: UUID, parcel_type: Type[Union[AnyTransportParcel, AnyRoutingParcel]], parcel_id: UUID
     ) -> Parcel:
@@ -431,21 +438,21 @@ def delete_all_profiles(self) -> None:
             self.delete_profile(profile.profile_id)
 
     def create_parcel(
-        self, profile_uuid: UUID, payload: AnyServiceParcel, vpn_uuid: Optional[UUID] = None
+        self, profile_id: UUID, payload: AnyServiceParcel, vpn_uuid: Optional[UUID] = None
     ) -> ParcelCreationResponse:
         """
         Create Service Parcel for selected profile_id based on payload type
         """
         if vpn_uuid is not None:
             if isinstance(payload, MulticastParcel):
-                response = self.endpoint.create_service_parcel(profile_uuid, payload._get_parcel_type(), payload)
+                response = self.endpoint.create_service_parcel(profile_id, payload._get_parcel_type(), payload)
                 return self.endpoint.associate_parcel_with_vpn(
-                    profile_uuid, vpn_uuid, payload._get_parcel_type(), ParcelAssociationPayload(parcel_id=response.id)
+                    profile_id, vpn_uuid, payload._get_parcel_type(), ParcelAssociationPayload(parcel_id=response.id)
                 )
             else:
                 parcel_type = payload._get_parcel_type().replace("lan/vpn/", "")
-                return self.endpoint.create_lan_vpn_sub_parcel(profile_uuid, vpn_uuid, parcel_type, payload)
-        return self.endpoint.create_service_parcel(profile_uuid, payload._get_parcel_type(), payload)
+                return self.endpoint.create_lan_vpn_sub_parcel(profile_id, vpn_uuid, parcel_type, payload)
+        return self.endpoint.create_service_parcel(profile_id, payload._get_parcel_type(), payload)
 
     def delete_parcel(self, profile_uuid: UUID, parcel_type: Type[AnyServiceParcel], parcel_uuid: UUID) -> None:
         """
@@ -1194,7 +1201,7 @@ def get(
             profile_id=profile_id, policy_object_list_type=policy_object_list_type, list_object_id=parcel_id
         )
 
-    def create(self, profile_id: UUID, payload: AnyPolicyObjectParcel) -> ParcelCreationResponse:
+    def create_parcel(self, profile_id: UUID, payload: AnyPolicyObjectParcel) -> ParcelCreationResponse:
         """
         Create Policy Object for selected profile_id based on payload type
         """

diff --git a/catalystwan/api/templates/device_template/device_template.py b/catalystwan/api/templates/device_template/device_template.py
@@ -4,7 +4,8 @@
 
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, List
+from typing import TYPE_CHECKING, ClassVar, List, Optional
+from uuid import UUID
 
 from jinja2 import DebugUndefined, Environment, FileSystemLoader, meta  # type: ignore
 from pydantic import BaseModel, ConfigDict, Field, field_validator
@@ -15,6 +16,13 @@
 logger = logging.getLogger(__name__)
 
 
+def str_to_uuid(s: str) -> Optional[UUID]:
+    try:
+        return UUID(s)
+    except ValueError:
+        return None
+
+
 class GeneralTemplate(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -54,6 +62,12 @@ class DeviceTemplate(BaseModel):
     )
     policy_id: str = Field(default="", serialization_alias="policyId", validation_alias="policyId")
 
+    def get_security_policy_uuid(self) -> Optional[UUID]:
+        return str_to_uuid(self.security_policy_id)
+
+    def get_policy_uuid(self) -> Optional[UUID]:
+        return str_to_uuid(self.policy_id)
+
     def generate_payload(self) -> str:
         env = Environment(
             loader=FileSystemLoader(self.payload_path.parent),

diff --git a/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py b/catalystwan/integration_tests/feature_profile/sdwan/test_extended_community.py
@@ -42,7 +42,7 @@ def test_create_extended_community_parcel(self):
         ext.add_site_of_origin_community("1.2.3.4", 1000)
         ext.add_site_of_origin_community("10.20.30.40", 3000)
 
-        self.created_id = self.policy_api.create(self.profile_uuid, ext).id
+        self.created_id = self.policy_api.create_parcel(self.profile_uuid, ext).id
         parcel = self.policy_api.get(self.profile_uuid, ExtendedCommunityParcel, parcel_id=self.created_id)
 
         assert parcel.payload.parcel_name == "ExampleTestName"

diff --git a/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py b/catalystwan/integration_tests/feature_profile/sdwan/test_ssl_decryption.py
@@ -57,7 +57,7 @@ def test_create_ssl_decryption_parcel(self):
             min_tls_ver="TLSv1.1",
         )
 
-        self.created_id = self.policy_api.create(self.profile_uuid, ssl_decryption_parcel).id
+        self.created_id = self.policy_api.create_parcel(self.profile_uuid, ssl_decryption_parcel).id
         read_parcel = self.policy_api.get(self.profile_uuid, SslDecryptionParcel, parcel_id=self.created_id)
 
         assert read_parcel.payload.parcel_name == "test_ssl_profile"

diff --git a/catalystwan/models/configuration/config_migration.py b/catalystwan/models/configuration/config_migration.py
@@ -137,6 +137,17 @@ class UX1Templates(BaseModel):
         validation_alias="deviceTemplates",
     )
 
+    def create_device_template_by_policy_id_lookup(self) -> Dict[Literal["policy", "security"], Dict[UUID, UUID]]:
+        lookup: Dict[Literal["policy", "security"], Dict[UUID, UUID]] = {"policy": {}, "security": {}}
+        for dt in self.device_templates:
+            policy_id = dt.get_policy_uuid()
+            security_policy_id = dt.get_security_policy_uuid()
+            if policy_id is not None:
+                lookup["policy"][policy_id] = UUID(dt.template_id)
+            if security_policy_id is not None:
+                lookup["security"][security_policy_id] = UUID(dt.template_id)
+        return lookup
+
 
 class UX1Config(BaseModel):
     # All UX1 Configuration items - Mega Model
@@ -150,13 +161,17 @@ class UX1Config(BaseModel):
 
 
 class TransformHeader(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
     type: str = Field(
         description="Needed to push item to specific endpoint."
         "Type discriminator is not present in many UX2 item payloads"
     )
     origin: UUID = Field(description="Original UUID of converted item")
     origname: Optional[str] = None
     subelements: Set[UUID] = Field(default_factory=set)
+    localized_policy_subelements: Optional[Set[UUID]] = Field(
+        default=None, serialization_alias="localizedPolicySubelements", validation_alias="localizedPolicySubelements"
+    )
     status: ConvertOutputStatus = Field(default="complete")
     info: List[str] = Field(default_factory=list)
 
@@ -251,9 +266,30 @@ def insert_parcel_type_from_headers(cls, values: Dict[str, Any]):
             profile_parcel["parcel"]["type_"] = profile_parcel["header"]["type"]
         return values
 
-    def transformed_parcels_with_origin(self, origin: Set[UUID]) -> List[TransformedParcel]:
+    def list_transformed_parcels_with_origin(self, origin: Set[UUID]) -> List[TransformedParcel]:
         return [p for p in self.profile_parcels if p.header.origin in origin]
 
+    def add_subelement_in_config_group(
+        self, profile_types: List[ProfileType], device_template_id: UUID, subelement: UUID
+    ) -> bool:
+        profile_ids: Set[UUID] = set()
+        added = False
+        for config_group in self.config_groups:
+            if config_group.header.origin == device_template_id:
+                profile_ids = config_group.header.subelements
+                break
+        if not profile_ids:
+            return added
+        for feature_profile in self.feature_profiles:
+            if feature_profile.header.type in profile_types and feature_profile.header.origin in profile_ids:
+                head = feature_profile.header
+                if head.localized_policy_subelements is None:
+                    head.localized_policy_subelements = {subelement}
+                else:
+                    head.localized_policy_subelements.add(subelement)
+                added = True
+        return added
+
 
 class ConfigTransformResult(BaseModel):
     # https://docs.pydantic.dev/2.0/usage/models/#fields-with-dynamic-default-values

diff --git a/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py b/catalystwan/models/configuration/feature_profile/sdwan/acl/ipv6acl.py
@@ -127,7 +127,7 @@ class MatchEntry(BaseModel):
     packet_length: Optional[Global[Union[str, int]]] = Field(
         default=None, validation_alias="packetLength", serialization_alias="packetLength"
     )
-    source_data_prefix: Union[SourceDataPrefix, SourceDataPrefixList] = Field(
+    source_data_prefix: Union[SourceDataPrefix, SourceDataPrefixList, None] = Field(
         default=None, validation_alias="sourceDataPrefix", serialization_alias="sourceDataPrefix"
     )
     source_ports: Optional[List[SourcePorts]] = Field(

diff --git a/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py b/catalystwan/models/configuration/feature_profile/sdwan/service/route_policy.py
@@ -255,8 +255,8 @@ class RoutePolicySequence(BaseModel):
     )
     base_action: Union[Global[AcceptRejectActionType], Default[AcceptRejectActionType]] = Field(
         default=as_default("reject", AcceptRejectActionType),
-        serialization_alias="routePolicyActionType",
-        validation_alias="routePolicyActionType",
+        serialization_alias="baseAction",
+        validation_alias="baseAction",
         description="Base Action",
     )
     protocol: Union[Global[Protocol], Default[Protocol]] = Field(
@@ -411,7 +411,7 @@ class RoutePolicyParcel(_ParcelBase):
     )
     default_action: Union[Global[AcceptRejectActionType], Default[AcceptRejectActionType]] = Field(
         default=as_default("reject", AcceptRejectActionType),
-        validation_alias=AliasPath("data", "routePolicyActionType"),
+        validation_alias=AliasPath("data", "defaultAction"),
         description="Default Action",
     )
     sequences: List[RoutePolicySequence] = Field(

diff --git a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access.py
@@ -64,8 +64,8 @@ class Sequence(BaseModel):
 
     base_action: Union[Global[AcceptDropActionType], Default[Literal[AcceptDropActionType]]] = Field(
         default=Default[Literal[AcceptDropActionType]](value="accept"),
-        validation_alias="BasicPolicyActionType",
-        serialization_alias="BasicPolicyActionType",
+        validation_alias="baseAction",
+        serialization_alias="baseAction",
     )
     match_entries: MatchEntries = Field(
         validation_alias="matchEntries", serialization_alias="matchEntries", description="Define match conditions"

diff --git a/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py b/catalystwan/models/configuration/feature_profile/sdwan/system/device_access_ipv6.py
@@ -64,8 +64,8 @@ class Sequence(BaseModel):
 
     base_action: Union[Global[AcceptDropActionType], Default[Literal[AcceptDropActionType]]] = Field(
         default=Default[Literal[AcceptDropActionType]](value="accept"),
-        validation_alias="BasicPolicyActionType",
-        serialization_alias="BasicPolicyActionType",
+        validation_alias="baseAction",
+        serialization_alias="baseAction",
     )
     match_entries: MatchEntries = Field(
         validation_alias="matchEntries", serialization_alias="matchEntries", description="Define match conditions"

diff --git a/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py b/catalystwan/models/configuration/feature_profile/sdwan/transport/__init__.py
@@ -6,6 +6,7 @@
 from typing_extensions import Annotated
 
 from catalystwan.models.configuration.feature_profile.sdwan.acl import AnyAclParcel
+from catalystwan.models.configuration.feature_profile.sdwan.service.route_policy import RoutePolicyParcel
 
 from .cellular_controller import CellularControllerParcel
 from .cellular_profile import CellularProfileParcel
@@ -65,6 +66,7 @@
         AnyTransportVpnParcel,
         AnyTransportVpnSubParcel,
         AnyManagementVpnSubParcel,
+        RoutePolicyParcel,
     ],
     Field(discriminator="type_"),
 ]

diff --git a/catalystwan/models/policy/policy.py b/catalystwan/models/policy/policy.py
@@ -77,10 +77,6 @@ class PolicyCreationPayload(BaseModel):
         default="default description", serialization_alias="policyDescription", validation_alias="policyDescription"
     )
     policy_type: str = Field(serialization_alias="policyType", validation_alias="policyType")
-    policy_definition: Union[PolicyDefinition, str] = Field(
-        serialization_alias="policyDefinition",
-        validation_alias="policyDefinition",
-    )
     is_policy_activated: bool = Field(
         default=False, serialization_alias="isPolicyActivated", validation_alias="isPolicyActivated"
     )

diff --git a/catalystwan/tests/config_migration/policy_converters/test_references_updater.py b/catalystwan/tests/config_migration/policy_converters/test_references_updater.py
@@ -57,4 +57,4 @@ def test_not_update_parcel_references(self):
 
         updated_parcel = update_parcel_references(aip_parcel, pushed_objects)
 
-        assert updated_parcel is aip_parcel
+        assert updated_parcel.model_dump_json() == aip_parcel.model_dump_json()
diff --git a/catalystwan/utils/config_migration/creators/config_pusher.py b/catalystwan/utils/config_migration/creators/config_pusher.py
@@ -22,6 +22,7 @@
 from catalystwan.models.configuration.feature_profile.sdwan.topology.mesh import MeshParcel
 from catalystwan.session import ManagerSession
 from catalystwan.utils.config_migration.creators.groups_of_interests_pusher import GroupsOfInterestPusher
+from catalystwan.utils.config_migration.creators.localized_policy_pusher import LocalizedPolicyPusher
 from catalystwan.utils.config_migration.creators.security_policy_pusher import SecurityPolicyPusher
 from catalystwan.utils.config_migration.factories.parcel_pusher import ParcelPusherFactory
 
@@ -52,6 +53,13 @@ def __init__(
             push_result=self._push_result,
             push_context=self._push_context,
         )
+        self._localized_policy_feature_pusher = LocalizedPolicyPusher(
+            ux2_config=ux2_config,
+            session=session,
+            progress=progress,
+            push_result=self._push_result,
+            push_context=self._push_context,
+        )
         self._security_policy_pusher = SecurityPolicyPusher(
             ux2_config=ux2_config,
             session=session,
@@ -73,6 +81,7 @@ def push(self) -> UX2ConfigPushResult:
         self._create_cloud_credentials()
         self._create_config_groups()
         self._groups_of_interests_pusher.push()
+        self._localized_policy_feature_pusher.push()
         self._security_policy_pusher.push()
         self._create_topology_groups(
             self._push_context.default_policy_object_profile_id
@@ -144,6 +153,7 @@ def _create_feature_profile_and_parcels(self, feature_profiles_ids: List[UUID])
                 profile = pusher.push(transformed_feature_profile.feature_profile, parcels, self._config_map.parcel_map)
                 feature_profiles.append(profile)
                 self._push_result.rollback.add_feature_profile(profile.profile_uuid, profile_type)
+                self._push_context.id_lookup[feature_profile_id] = profile.profile_uuid
             except ManagerHTTPError as e:
                 logger.error(f"Error occured during [{fp_name}] feature profile creation: {e}")
             except Exception:
@@ -205,7 +215,7 @@ def _create_topology_groups(self, default_policy_object_profile_id: Optional[UUI
                 logger.error(f"Error occured during topology group creation: {e}")
                 continue
 
-            for transformed_parcel in self._ux2_config.transformed_parcels_with_origin(origins):
+            for transformed_parcel in self._ux2_config.list_transformed_parcels_with_origin(origins):
                 parcel = transformed_parcel.parcel
                 if isinstance(parcel, (CustomControlParcel, HubSpokeParcel, MeshParcel)):
                     try:

diff --git a/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py b/catalystwan/utils/config_migration/creators/groups_of_interests_pusher.py
@@ -98,7 +98,7 @@ def push(self) -> None:
 
             try:
                 parcel = update_parcel_references(parcel, self.push_context.id_lookup)
-                parcel_id = self._policy_object_api.create(profile_id=profile_id, payload=parcel).id
+                parcel_id = self._policy_object_api.create_parcel(profile_id=profile_id, payload=parcel).id
                 profile_rollback.add_parcel(parcel.type_, parcel_id)
                 self._push_result.report.groups_of_interest.add_created(parcel.parcel_name, parcel_id)
                 self.push_context.id_lookup[transformed_parcel.header.origin] = parcel_id
Original file line number	Diff line number	Diff line change
Expand Up		@@ -57,4 +57,4 @@ def test_not_update_parcel_references(self):

		updated_parcel = update_parcel_references(aip_parcel, pushed_objects)

		assert updated_parcel is aip_parcel
		assert updated_parcel.model_dump_json() == aip_parcel.model_dump_json()