Merge pull request #110 from nautobot/develop

v1.4.0 Release

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## v1.4.0 - 2022-10-21
+
+### Added
+
+- #101 - Added support for the Device Lifecycle Management plug-in. Addresses #87.
+- #99 - Added support for getting interface description.
+- #103 - Added support for importing IP Addresses from CloudVision. Addresses #52.
+- #108 - Added support for hostname parsing to dynamically create Site/Role. #51
+
+### Fixed
+
+- #101 - Fix Platform creation to be single item instead of matching DeviceType.
+- #101 - Corrected Nautobot adapter load method
+- #101 - Ensure only CustomFields with `arista_` prepend are loaded. Addresses #95.
+- #101 - Validated enabledState key in chassis interface. Addresses #102.
+- #101 - Fixed MAC address string to match CVP so diff is idempotent.
+- #104 - Fixes display of plugin settings. Addresses #86.
+
+### Performance
+
+- #101 - Optimized query for Devices to improve load times. Addresses #93.
+
+### Refactor
+
+- #104 - Refactored Jobs to use load source/target adapter methods for performance metrics.
+
 ## v1.3.0 - 2022-10-13
 
 ### Added

development/Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /source
 COPY poetry.lock pyproject.toml /source/
 # --no-root declares not to install the project package since we're wanting to take advantage of caching dependency installation
 # and the project is copied in and installed after this step
-RUN poetry install --no-interaction --no-ansi --no-root
+RUN poetry install --no-interaction --no-ansi --no-root --extras "nautobot-device-lifecycle-mgmt"
 
 # Copy in the rest of the source code and install local Nautobot plugin
 COPY . /source

development/nautobot_config.py

@@ -8,7 +8,7 @@
 import sys
 
 from nautobot.core.settings import *  # noqa: F401,F403
-from nautobot.core.settings_funcs import parse_redis_connection
+from nautobot.core.settings_funcs import parse_redis_connection, is_truthy
 
 TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
 
@@ -62,7 +62,7 @@
 SECRET_KEY = os.getenv("NAUTOBOT_SECRET_KEY", "")
 
 # Enable installed plugins. Add the name of each plugin to the list.
-PLUGINS = ["nautobot_ssot", "nautobot_ssot_aristacv"]
+PLUGINS = ["nautobot_ssot", "nautobot_ssot_aristacv", "nautobot_device_lifecycle_mgmt"]
 
 # Plugins configuration settings. These settings are used by various plugins that the user may have installed.
 # Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
@@ -71,13 +71,27 @@
         "hide_example_jobs": True,  # defaults to False if unspecified
     },
     "nautobot_ssot_aristacv": {
-        "delete_devices_on_sync": True,
+        "cvp_token": os.getenv("NAUTOBOT_ARISTACV_TOKEN", ""),
+        "cvp_host": os.getenv("NAUTOBOT_ARISTACV_HOST", ""),
+        "cvp_port": os.getenv("NAUTOBOT_ARISTACV_PORT", 443),
+        "cvp_user": os.getenv("NAUTOBOT_ARISTACV_USERNAME", ""),
+        "cvp_password": os.getenv("NAUTOBOT_ARISTACV_PASSWORD", ""),
+        "verify": is_truthy(os.getenv("NAUTOBOT_ARISTACV_VERIFY", True)),
         "from_cloudvision_default_site": "cloudvision_imported",
         "from_cloudvision_default_device_role": "network",
         "from_cloudvision_default_device_role_color": "ff0000",
-        "from_cloudvision_default_device_status": "Active",
-        "from_cloudvision_default_device_status_color": "ff0000",
-        "apply_import_tag": True,
-        "import_active": False,
+        "delete_devices_on_sync": is_truthy(os.getenv("NAUTOBOT_ARISTACV_DELETE_ON_SYNC", False)),
+        "apply_import_tag": is_truthy(os.getenv("NAUTOBOT_ARISTACV_IMPORT_TAG", False)),
+        "import_active": is_truthy(os.getenv("NAUTOBOT_ARISTACV_IMPORT_ACTIVE", False)),
+        "hostname_patterns": [[r"(?P<site>\w{2,3}\d+)-(?P<role>\w+)-\d+"]],
+        "site_mappings": {"ams01": "Amsterdam", "atl01": "Atlanta"},
+        "role_mappings": {
+            "bb": "backbone",
+            "edge": "edge",
+            "dist": "distribution",
+            "leaf": "leaf",
+            "rtr": "router",
+            "spine": "spine",
+        },
     },
 }

nautobot_ssot_aristacv/__init__.py

@@ -32,6 +32,9 @@ class NautobotSSOTAristaCVConfig(PluginConfig):
         "cvp_password": os.getenv("NAUTOBOT_ARISTACV_PASSWORD"),
         "verify": os.getenv("NAUTOBOT_ARISTACVP_VERIFY"),
         "cvp_token": os.getenv("NAUTOBOT_ARISTACV_TOKEN"),
+        "hostname_patterns": [],
+        "site_mappings": {},
+        "role_mappings": {},
     }
     caching_config = {}
 
@@ -41,9 +44,13 @@ def ready(self):
 
         from .signals import (  # pylint: disable=import-outside-toplevel
             post_migrate_create_custom_fields,
+            post_migrate_create_manufacturer,
+            post_migrate_create_platform,
         )
 
         post_migrate.connect(post_migrate_create_custom_fields)
+        post_migrate.connect(post_migrate_create_manufacturer)
+        post_migrate.connect(post_migrate_create_platform)
 
 
 config = NautobotSSOTAristaCVConfig  # pylint:disable=invalid-name

nautobot_ssot_aristacv/diffsync/adapters/cloudvision.py

@@ -1,14 +1,16 @@
 """DiffSync adapter for Arista CloudVision."""
+from django.conf import settings
 import distutils
 import re
 
 import arista.tag.v1 as TAG
 from diffsync import DiffSync
-from diffsync.exceptions import ObjectAlreadyExists
+from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound
 from nautobot_ssot_aristacv.diffsync.models.cloudvision import (
     CloudvisionCustomField,
     CloudvisionDevice,
     CloudvisionPort,
+    CloudvisionIPAddress,
 )
 from nautobot_ssot_aristacv.utils import cloudvision
 
@@ -18,9 +20,10 @@ class CloudvisionAdapter(DiffSync):
 
     device = CloudvisionDevice
     port = CloudvisionPort
+    ipaddr = CloudvisionIPAddress
     cf = CloudvisionCustomField
 
-    top_level = ["device", "cf"]
+    top_level = ["device", "ipaddr", "cf"]
 
     def __init__(self, *args, job=None, conn: cloudvision.CloudvisionApi, **kwargs):
         """Initialize the CloudVision DiffSync adapter."""
@@ -33,7 +36,12 @@ def load_devices(self):
         for dev in cloudvision.get_devices(client=self.conn.comm_channel):
             if dev["hostname"] != "":
                 new_device = self.device(
-                    name=dev["hostname"], serial=dev["device_id"], device_model=dev["model"], uuid=None
+                    name=dev["hostname"],
+                    serial=dev["device_id"],
+                    status=dev["status"],
+                    device_model=dev["model"],
+                    version=dev["sw_ver"],
+                    uuid=None,
                 )
                 try:
                     self.add(new_device)
@@ -43,6 +51,7 @@ def load_devices(self):
                     )
                     continue
                 self.load_interfaces(device=new_device)
+                self.load_ip_addresses(dev=new_device)
                 self.load_device_tags(device=new_device)
             else:
                 self.job.log_warning(message=f"Device {dev} is missing hostname so won't be imported.")
@@ -71,12 +80,16 @@ def load_interfaces(self, device):
                 transceiver = cloudvision.get_interface_transceiver(
                     client=self.conn, dId=device.serial, interface=base_port_name
                 )
+            port_description = cloudvision.get_interface_description(
+                client=self.conn, dId=device.serial, interface=port["interface"]
+            )
             port_status = cloudvision.get_interface_status(port_info=port)
             port_type = cloudvision.get_port_type(port_info=port, transceiver=transceiver)
             if port["interface"] != "":
                 new_port = self.port(
                     name=port["interface"],
                     device=device.name,
+                    description=port_description,
                     mac_addr=port["mac_addr"] if port.get("mac_addr") else "",
                     mode="tagged" if port_mode == "trunk" else "access",
                     mtu=port["mtu"],
@@ -93,6 +106,50 @@ def load_interfaces(self, device):
                         message=f"Duplicate port {port['interface']} found for {device.name} and ignored. {err}"
                     )
 
+    def load_ip_addresses(self, dev: device):
+        """Load IP addresses from CloudVision."""
+        dev_ip_intfs = cloudvision.get_ip_interfaces(client=self.conn, dId=dev.serial)
+        for intf in dev_ip_intfs:
+            try:
+                _ = self.get(self.port, {"name": intf["interface"], "device": dev.name})
+            except ObjectNotFound:
+                new_port = self.port(
+                    name=intf["interface"],
+                    device=dev.name,
+                    description=cloudvision.get_interface_description(
+                        client=self.conn, dId=dev.serial, interface=intf["interface"]
+                    ),
+                    mac_addr="",
+                    enabled=True,
+                    mode="access",
+                    mtu=65535,
+                    port_type=cloudvision.get_port_type(port_info={"interface": intf["interface"]}, transceiver=""),
+                    status="active",
+                    uuid=None,
+                )
+                self.add(new_port)
+                try:
+                    device = self.get(self.device, dev.name)
+                    device.add_child(new_port)
+                except ObjectNotFound as err:
+                    self.job.log_warning(
+                        message=f"Unable to find device {dev.name} to assign port {intf['interface']}. {err}"
+                    )
+
+            new_ip = self.ipaddr(
+                address=intf["address"],
+                interface=intf["interface"],
+                device=dev.name,
+                uuid=None,
+            )
+            try:
+                self.add(new_ip)
+            except ObjectAlreadyExists as err:
+                self.job.log_warning(
+                    message=f"Unable to load {intf['address']} for {dev.name} on {intf['interface']}. {err}"
+                )
+                continue
+
     def load_device_tags(self, device):
         """Load device tags from CloudVision."""
         system_tags = cloudvision.get_tags_by_type(
@@ -123,4 +180,11 @@ def load_device_tags(self, device):
 
     def load(self):
         """Load devices and associated data from CloudVision."""
+        PLUGIN_SETTINGS = settings.PLUGINS_CONFIG["nautobot_ssot_aristacv"]
+        if PLUGIN_SETTINGS.get("hostname_patterns") and not (
+            PLUGIN_SETTINGS.get("site_mappings") and PLUGIN_SETTINGS.get("role_mappings")
+        ):
+            self.job.log_warning(
+                message="Configuration found for hostname_patterns but no site_mappings or role_mappings. Please ensure your mappings are defined."
+            )
         self.load_devices()

nautobot_ssot_aristacv/diffsync/adapters/nautobot.py

@@ -1,20 +1,28 @@
 """DiffSync adapter for Nautobot."""
 from nautobot.dcim.models import Device as OrmDevice
 from nautobot.dcim.models import Interface as OrmInterface
+from nautobot.ipam.models import IPAddress as OrmIPAddress
 from diffsync import DiffSync
-from diffsync.exceptions import ObjectNotFound
+from diffsync.exceptions import ObjectNotFound, ObjectAlreadyExists
 
-from nautobot_ssot_aristacv.diffsync.models.nautobot import NautobotDevice, NautobotCustomField, NautobotPort
+from nautobot_ssot_aristacv.diffsync.models.nautobot import (
+    NautobotDevice,
+    NautobotCustomField,
+    NautobotIPAddress,
+    NautobotPort,
+)
+from nautobot_ssot_aristacv.utils import nautobot
 
 
 class NautobotAdapter(DiffSync):
     """DiffSync adapter implementation for Nautobot custom fields."""
 
     device = NautobotDevice
     port = NautobotPort
+    ipaddr = NautobotIPAddress
     cf = NautobotCustomField
 
-    top_level = ["device", "cf"]
+    top_level = ["device", "ipaddr", "cf"]
 
     def __init__(self, *args, job=None, **kwargs):
         """Initialize the Nautobot DiffSync adapter."""
@@ -23,33 +31,38 @@ def __init__(self, *args, job=None, **kwargs):
 
     def load(self):
         """Load device custom field data from Nautobot and populate DiffSync models."""
-        devices = OrmDevice.objects.all()
-        for dev in devices:
+        for dev in OrmDevice.objects.filter(device_type__manufacturer__slug="arista"):
             try:
-                if dev.device_type.manufacturer.name.lower() == "arista":
-                    new_device = self.device(
-                        name=dev.name, device_model=dev.device_type.name, serial=dev.serial, uuid=dev.id
-                    )
-                    self.add(new_device)
-                    dev_custom_fields = dev.custom_field_data
+                new_device = self.device(
+                    name=dev.name,
+                    device_model=dev.device_type.model,
+                    serial=dev.serial,
+                    status=dev.status.slug,
+                    version=nautobot.get_device_version(dev),
+                    uuid=dev.id,
+                )
+                self.add(new_device)
+            except ObjectAlreadyExists as err:
+                self.job.log_warning(message=f"Unable to load {dev.name} as it appears to be a duplicate. {err}")
+                continue
 
-                    for cf_name, cf_value in dev_custom_fields.items():
-                        if cf_value is None:
-                            cf_value = ""
-                        new_cf = self.cf(name=cf_name, value=cf_value, device_name=dev.name)
+            for cf_name, cf_value in dev.custom_field_data.items():
+                if cf_name.startswith("arista_"):
+                    try:
+                        new_cf = self.cf(
+                            name=cf_name, value=cf_value if cf_value is not None else "", device_name=dev.name
+                        )
                         self.add(new_cf)
+                    except AttributeError as err:
+                        self.job.log_warning(message=f"Unable to load {cf_name}. {err}")
+                        continue
 
-                    # Gets model from device and puts it into CustomField Object.
-                    new_cf = self.cf(name="arista_model", value=str(dev.platform), device_name=dev.name)
-                    self.add(new_cf)
-            except AttributeError:
-                continue
-
-        for intf in OrmInterface.objects.all():
+        for intf in OrmInterface.objects.filter(device__device_type__manufacturer__slug="arista"):
             new_port = self.port(
                 name=intf.name,
                 device=intf.device.name,
-                mac_addr=intf.mac_address if intf.mac_address else "",
+                description=intf.description,
+                mac_addr=str(intf.mac_address).lower() if intf.mac_address else "",
                 enabled=intf.enabled,
                 mode=intf.mode,
                 mtu=intf.mtu,
@@ -65,3 +78,15 @@ def load(self):
                 self.job.log_warning(
                     message=f"Unable to find Device {intf.device.name} in diff to assign to port {intf.name}. {err}"
                 )
+
+        for ipaddr in OrmIPAddress.objects.filter(interface__device__device_type__manufacturer__slug="arista"):
+            new_ip = self.ipaddr(
+                address=str(ipaddr.address),
+                interface=ipaddr.assigned_object.name,
+                device=ipaddr.assigned_object.device.name,
+                uuid=ipaddr.id,
+            )
+            try:
+                self.add(new_ip)
+            except ObjectAlreadyExists as err:
+                self.job.log_warning(message=f"Unable to load {ipaddr.address} as appears to be a duplicate. {err}")