From fa1842307912ee43ef7ef6152627706f939ff5e4 Mon Sep 17 00:00:00 2001 From: "Ben Astell (bastell)" Date: Fri, 19 May 2023 15:11:39 -0400 Subject: [PATCH] Abstract Doc Updates for Revision Co-authored-by: Thomas Ryan (thomryan) --- docs/Makefile | 2 +- docs/abstract/_dev.rst | 853 +++++++++++++++++++++++++++++ docs/abstract/concept.rst | 574 ++++++++++--------- docs/abstract/contributing.rst | 111 ++++ docs/abstract/conventions.rst | 144 ----- docs/abstract/index.rst | 11 +- docs/abstract/introduction.rst | 145 +---- docs/abstract/lookup_class.rst | 329 ----------- docs/abstract/lookup_decorator.rst | 239 -------- docs/abstract/revisions.rst | 125 +++++ docs/conf.py | 1 + docs/index.rst | 4 +- 12 files changed, 1413 insertions(+), 1125 deletions(-) create mode 100644 docs/abstract/_dev.rst create mode 100644 docs/abstract/contributing.rst delete mode 100644 docs/abstract/conventions.rst delete mode 100644 docs/abstract/lookup_class.rst delete mode 100644 docs/abstract/lookup_decorator.rst create mode 100644 docs/abstract/revisions.rst diff --git a/docs/Makefile b/docs/Makefile index cc64636..8bf9cdb 100755 --- a/docs/Makefile +++ b/docs/Makefile @@ -40,7 +40,7 @@ DEVNET = true # Dependencies for building documentation DEPENDENCIES = robotframework Sphinx==4.2.0 sphinxcontrib-napoleon \ sphinxcontrib-mockautodoc sphinx-rtd-theme - #sphinxcontrib_robotframework + #sphinxcontrib_robotframework sphinx-tabs # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) diff --git a/docs/abstract/_dev.rst b/docs/abstract/_dev.rst new file mode 100644 index 0000000..70855a8 --- /dev/null +++ b/docs/abstract/_dev.rst @@ -0,0 +1,853 @@ + +INTRODUCTION +============ + +.. code-block:: python + + # Example + # ------- + # + # with and without abstraction + + # typical non-abstracted script + # ----------------------------- + # import the proper function through if statements + if release == 'v2.1': + if context == 'YANG': + from my_library.v2_1.yang import configure_something + else: + from my_library.v2_1.cli import configure_something + elif release == 'v2.2': + if context == 'YANG': + from my_library.v2_2.yang import configure_something + else: + from my_library.v2_2.cli import configure_something + else: + if context == 'YANG': + from my_library.generic import configure_something + else: + from my_library.generic import configure_something + + # get result + result = configure_something() + + # using abstraction & properly abstracted libraries + # ------------------------------------------------- + from genie import abstract + + # build a lookup object and pass the release/context as tokens + lookup = abstract.Lookup(release, context) + + # collect result by looking up the corresponding API + result = lookup.my_library.configure_something() + + +As show above, through the use of ``abstract`` package, users can write +straightforward codes (single-source) that automatically invokes the right set +of library APIs (classes, functions, methods etc) based on given requirements, +without the repeated use of custom ``if..elif..elif..else`` statements +everywhere. This dynamic library referencing can be beneficial in many use +cases, including but not limited to: + + - handling minute release-to-release, image-to-image differences + - running the same tests/scripts across a variety of hardware (os/platforms, etc) + + +Concept +======= + +Abstract Mechanism +------------------ + + +The ``abstract`` module works most of its magic at the Python ``import`` and +``getattr()`` level. It does so by dissecting each lookup into three distinct +parts: + + - **relative path**: the primary lookup path that makes the most sense from + a functional perspective. This is what the user references directly, eg: + ``my_library.routing.ospf`` + + - **tokens**: the list of abstraction tokens currently known by the + abstraction engine. This portion is registered through the ``Lookup`` + object. Eg: ``iosxr``, ``fretta``, ``xml``. + + - **target**: the module/class/function/variable user is looking for. + +During runtime, the lookup engine dynamically pieces together the above +information into a list of possible candidate **absolute paths** (direct mapping +to python import statements). As the list of tokens is arbitrary, this candidate +list is built following the :ref:`abstract_search_algorithm`. + + + + + + +.. code-block:: python + + # Example + # ------- + # + # relative path & absolute path explained + + # Given the following tokens: + # - iosxe + # - polaris_dev + # - yang + os = 'iosxe' + branch = 'polaris_dev' + context = 'yang' + + # feed to to abstraction lookup engine. + library = abstract.Lookup(os, branch, context) + + # the relative call to + library.my_package.config.routing.ospf.Ospf() + + # could match, for example: + # + # my_package.iosxe.config.polaris_dev.routing.ospf.yang.Ospf + # | | | | | | | | + # abstraction | relative | relative | | class + # package | path | path | token + # token token relative + # path + # which translates to: + # from my_package.iosxe.config.polaris_dev.routing.ospf.yang import Ospf + # + # where + # ----- + # relative path = config, routing, ospf + # tokens = iosxe, polaris_dev, yang + # target = Ospf() + + + +.. _abstract_search_algorithm: + +Search Algorithm +---------------- + +The search engine combines the user's **relative path** and currently known +**tokens** into possible **absolute paths** (python module names) and searches +through them. A match occurs when an implementation is found (ie the target +exists at the candidate relative path). Otherwise, the next combination is +tried. If no target is found, a ``LookupError`` would be thrown. + +As the token names are not pre-defined, the search engine orders +all tokens in a pre-defined fashion: + + - token describes a set of *differences* + - token positions are always fixed w.r.t. to its left (parent) + - tokens on the right are more *specific* than tokens on the left + - each token may only appear *once* in a combination + - greedy match: more tokens matches is always better than less. + +.. code-block:: text + + Given tokens: a, b, c and d, the preferred token combination would be: + + a b c d + a b c + a b + a + (no tokens) + +These combinations are then *multiplexed* to user's **relative path** into +potential **absolute paths** to search for, using the following rules: + + - absolute paths must always start with the abstracted package name. + + - the order of relative path sections (words divided by ``.``) must be + preserved. + + - the order of token combinations must be preserved. + + - tokens may take place before and after each relative path section, and may + appear in multiples together. (eg, ``library.iosxr.google.latest.mpls``) + + - the last resort option is to try with "no token", eg, matching the + relative path directly. + +Combining the above rules, the ideal solution would be a multi-combinatory +mathematical function, whose search complexity is ... *(insert math here)* ... +exponential. + +.. code-block:: text + + Given Package: my_pkg + Relative Path: X, Y + Tokens: a, b + Target: MyClass() + + We could have the following mathmatical combinational possibilities: + + 1. my_pkg.a.X.b.Y.MyClass() + 2. my_pkg.a.X.Y.b.MyClass() + 3. my_pkg.X.a.Y.b.MyClass() + 4. my_pkg.X.a.b.Y.MyClass() + 5. my_pkg.X.Y.a.b.MyClass() + 6. my_pkg.a.X.Y.MyClass() + 7. my_pkg.X.a.Y.MyClass() + 8. my_pkg.X.Y.a.MyClass() + 9. my_pkg.X.Y.MyClass() + + And that's just with two tokens and two path sections! + +The actual implementation internally is much simpler. When an an abstracted +package is defined/declared and the lookup object is created, the package and +all of its child modules are *recursively imported*. This allows the abstraction +engine to build an internal table of relative paths, their available token +combinations learnt from the import and its corresponding module. This reduced +**relative path + tokens** relationship effectively simplies the above +brute-force search algorithm into an ``O(n)`` lookup, where ``n`` is the number +of tokens. + +.. code-block:: text + + Pseudo Lookup Table + =================== + + Relative Path Tokens Combos Corresponding Module + ------------- ------------- -------------------- + X.Y a, b X.a.Y.b + X.Y a X.a.Y + X.Y None X.Y + + (shown in order of preference, from top down) + +This algorithm limits to only dealing with what's been defined in the user +library, instead of going through all possible permutations of **relative path** +and **tokens**. The system assumes that it is unlikely for users to make +redundant declarations, such as defining both ``from X.a.Y.b import target`` and +``from X.a.b.Y import target`` within the same library. + +.. note:: + + The learning process safeguards against these redundant scenarios. + + +.. _token_builder: + +Token Builder +------------- + +The token builder is a simple function that implements the token permutation +portion of the :ref:`abstract_search_algorithm`. The default token builder is +available as ``abstract.magic.default_builder()``. + +.. csv-table:: default_builder Argument List + :header: "Argument", "Description" + + ``tokens``, "list of tokens to permute" + ``mandatory``, "list of tokens that must be used" + +.. code-block:: python + + # Example + # ------- + # + # pseudo code demonstrating the behavior of default token builder + + from abstract.magic import default_builder + + # without any mandatory tokens + default_builder(tokens = ['nxos', 'n7k', 'c7003', 'yang', 'R8_1']) + # [('nxos', 'n7k', 'c7003', 'yang', 'R8_1'), + # ('nxos', 'n7k', 'c7003', 'yang'), + # ('nxos', 'n7k', 'c7003'), + # ('nxos', 'n7k'), + # ('nxos',), + # ()] + + # a mandatory token is one that MUST be used in the search + default_builder(tokens = ['nxos', 'n7k', 'c7003', 'yang', 'R8_1'], + mandatory = ['yang']) + # [('nxos', 'n7k', 'c7003', 'yang', 'R8_1'), + # ('nxos', 'n7k', 'c7003', 'yang'), + # ('nxos', 'n7k', 'yang'), + # ('nxos', 'yang'), + # ('yang',)] + +In essence, the "tokens" input parameter to the builder is a reflection of +the actual, longest possible chain of tokens under any given relative path. If +no target is found at this token/relative path combination, the next, reduced +set of tokens is tried. This reduction mechanism always reduces from the right. + +Use the ``mandatory`` input argument when you absolutely require some tokens to +be present in any token permutations during abstraction. This can be useful when +you do not want the system to automatically fallback using the above logic and +remove it. This ensures the proper "set" of libraries is picked. + + + + +.. _abstract_lookup_cls: + +Lookup Class +============ + +``Lookup`` class is the main feature of ``abstract`` package. It implements +:ref:`Abstraction Concepts ` in a user-friendly fashion, +and allows users to perform dynamic lookups just as if they were accessing +object attributes. + +.. code-block:: text + + .------> TokenX.Y implementation + / + UserScript -> Lookup Target --+--------> Token X implementation + (func/cls/var) \ + `------> Default (no token) implementation + + +Usages +------ + +When instanciated with a list of :ref:`abstraction_tokens`, ``Lookup`` class +allows the user to reference any :ref:`abstraction_pkg` available in the current +namespace scope. This behavior can be generally summarized into the following: + +- at miminum, a list of :ref:`abstraction_tokens` is required in order to + instanciate a new ``Lookup`` object. + +- by default, all :ref:`Abstraction-Enabled Packages ` imported + and available at the scope where ``Lookup()`` is called, gets discovered and + registered internally. + +- if an package is a part of a parent package, it needs to be imported + directly into the current namespace. + + .. code-block:: python + + # instead of + import parent_package.my_abstracted_package + + # you must import it directly + from parent_package import my_abstracted_package + +- users can provide a dictionary of ``name: package`` to ``Lookup()`` and + override the default discovery behavior. ``name`` is the alias to refer to + the given package. + + .. code-block:: python + + import parent.my_package + + lookup = Lookup(*tokens, packages = {'pkg': parent.my_package}) + +- perform library lookups as if you were referencing attributes of an object. + + .. code-block:: python + + import my_abstracted_library + + lookup = Lookup(*tokens) + + # always start with the name of the library you want to search from + lookup.my_abstracted_library.some_module.some_other_module.Target() + +- the default :ref:`token_builder` supports specifying mandatory tokens. This + generator can be overwritten with ``builder`` argument to ``Lookup()`` (very + advanced functionality). + + .. code-block:: python + + from genie import abstract + from my_library import my_builder + + # use your default builder + lookup = Lookup(*tokens, builder = my_builder) + + +- in addition, this global default builder setting can be modified by setting + ``abstract.magic.DEFAULT_BUILDER`` to a builder of your liking. This will + affect **all** newly created ``Lookup()`` object from this point onwards. + + .. code-block:: python + + from genie import abstract + from my_library import my_default_builder + + # overwrite the default builder + abstract.magic.DEFAULT_BUILDER = my_default_builder + + # any lookup object created hereonward will take on your builder + lookup = Lookup(*tokens) + + +.. code-block:: python + + # Example + # ------- + # + # Lookup() class examples & features + + # import the class from abstract + from genie.abstract import Lookup + + # import any abstraction-enabled packages you need + import my_abstracted_library + from xbu_shared import genie, parser + + # create the lookup object and provide it with tokens + # this auto discovers and registers the above imported packages: + # my_abstracted_library, genie, parser + lookup = Lookup('iosxr') + + # now use the lookup object and reference the above imported + # libraries using attribute queries. Eg: + + result = lookup.my_abstracted_library.my_abstracted_function() + # runtime absolute path translation: + # from my_abstracted_library.iosxr import my_abstracted_function + # result = my_abstracted_function() + + ospf = lookup.genie.conf.ospf.Ospf() + # runtime absolute path translation: + # from xbu_shared.genie.conf.ospf.iosxr import Ospf + # ospf = Ospf() + + output = lookup.parser.ShowVersion(device = device) + # runtime absolute path translation: + # from xbu_shared.parser.iosxr import ShowVersion + # output = ShowVersion() + + # -------------------------------------------------------------------------- + + # create new Lookup() instances if tokens requirements change + # you can also change the set of packages available for it, + # as well as its base reference name. + lookup = Lookup('token_a', 'token_b', '...', 'etc', + packages = {'lib_1': my_abstracted_library, + 'lib_2': genie', + 'lib_3': parser}) + + # as new names are tokens are provided, we can now do: + result = lookup.lib_1.my_abstracted_function() + ospf = lookup.lib_2.conf.ospf.Ospf() + output = lookup.lib_3.ShowVersion(device = device) + +.. tip:: + + always use meaningful package names. + +.. csv-table:: Lookup Class Argument List + :header: "Argument", "Description" + + ``*token``, "list of tokens to be used as input requirements for to this + lookup" + ``packages``, "dictionary of name/abstraction package to lookup from + (optional)" + ``builder``, "token permutation builder (optional)" + ``**builder_kwargs``, "any keyword arguments/values to be passed to the + builder (optional)" + + +Integration with Topology +------------------------- + +``Lookup()`` class also features a classmethod constructor that enables it to +understand pyATS topology module's ``Device()`` object, and subsequently, create +lookup objects based on the tokens specified under ``Device.custom.abstraction`` +field. + +.. code-block:: yaml + + # Example + # ------- + # + # example pyATS topology device yaml + + device: + my-example-device: + type: router + os: iosxe + series: asr1k + custom: + abstraction: + order: [os, series, context] + context: yang + +.. code-block:: python + + # Example + # ------- + # + # using the above testbed definition with abstraction + + from pyats import topology + testbed = topology.loader.load('/path/to/above/testbed.yaml') + device = testbed.devices['my-example-device'] + + # create abstraction + from genie.abstract import Lookup + + lookup = Lookup.from_device(device) + # eg, the above is equivalent to: + # os = device.custom.abstraction.get('os', device.os) + # series = device.custom.abstraction.get('series', device.series) + # context = device.custom.abstraction.get('context') + # lookup = Lookup(os, series, context) + +In the above testbed YAML file, we defined a custom abstraction definition, +specifying the expected token list ``[os, series, context]``, and the expected +``context = 'yang'``. + +When ``Lookup.from_device()`` method is called, the tokens associated with that +device is automatically extracted following these rules: + + - ``device.custom.abstraction`` is a dictionary + - ``device.custom.abstraction['tokens']`` specifies the list of attributes + to read from this device object, and converted into token values. + - the code prefers to read the attributes from + ``device.custom.abstraction[attrbute]``, and falls back to + ``device.`` if needed. + +All other arguments to ``Lookup()``, such as ``builder, packages, +builder_kwargs`` also applies to this classmethod. + +If however you would like to not specify the ``device.custom.abstraction`` block +in your testbed YAML file all the time, you can provide ``default_tokens`` as a +list to ``Lookup.from_device()``. Any tokens specified there would be looked-up +from the provided device attribute. + +.. code-block:: python + + # Example + # ------- + # + # Lookup.from_device using defaults + + lookup = Lookup.from_device(device, default_tokens = ['os', 'series']) + # eg, the above is equivalent to: + # os = device.os + # series = device.serie + # lookup = Lookup(os, series) + +.. note:: + + note that when using ``default_tokens``, the lookup from device attribute + is non-strict, eg: if tokens ``a``, ``b``, ``c`` are specified, and only + ``a``, ``c`` exists, it will not error and just use these values instead. + + +Tips & Tricks +------------- + +Typically, abstraction should be used when the end library needs to handle +differences (such as OS/Release/Mgmt Interface) etc. This leads to a per-device +lookup model, where the set of :ref:`abstraction-tokens` per device differs. +The best, pythonic method to tackle this is to follow the natural patterns +of Python/pyATS programming: + +- ``import`` all your packages at the top of your script/code, including all + :ref:`Abstraction-Enabled Packages `. + +- inside AEtest ``CommonSetup`` section, as soon as you have connected to your + testbed devices and learnt about what they are, create your ``Lookup()`` + objects and assign them as an attribute to each ``Device`` instance. + +.. code-block:: python + + # Example + # ------- + # + # an example AEtest script with abstraction enabled + + # import everything at the top + import logging + from genie import abstract + from pyats import aetest + + # eg, these are my abstraction libraries + import my_abstracted_library + from xbu_shared import genie, parser + + logger = logging.getLogger(__name__) + + class CommonSetup(aetest.CommonSetup): + + @aetest.subsection + def connect_to_testbed(self, testbed): + for name, device in testbed.devices.items(): + device.connect() + logger.info('connected to device %s' % device.name) + + @aetest.subsection + def create_abstraction_lookup_objects(self, testbed, context): + '''create_abstraction_lookup_objects + + Subsection to create abstraction Lookup object and assigns it to + each corresponding device object as 'device.lib' attribute. + + In this example, we are using device object's attribute 'os', 'type' + (from testbed YAML file) and script input parameter 'context' as + tokens. + ''' + for device in testbed.devices.values(): + device.lib = Lookup(device.os, device.type, context) + + # ... other subsections + + # from here onwards, you can refer to libraries dynamically. + + class Configure_Ospf(aetest.Testcase): + + @aetest.setup + def setup(self, testbed): + # iterate through all devices and configure device... + for device in testbed.devices.values(): + device.lib.my_abstracted_library.configure_ospf(arg_1 = '...', + arg_2 = '...', + etc = '...') + + @aetest.test + def test(self, testbed): + for device in testbed.devices.values(): + output = device.lib.parser.ShowOspf(device = device) + + # validate values... etc + # ... + + +.. _abstract_lookup_decorator: + +Lookup Decorator +================ + +``LookupDecorator`` is a feature extension to :ref:`abstract_lookup_cls`. +Whereas the ``Lookup`` class allows users to write **different** classes, +functions and variables in tokenized modules and dynamically reference them, the +lookup decorator operates at the class method level, allowing users to write +a **single class** with different method implementations per each token variance +combination. + +.. code-block:: text + + .--> TokenX.Y class method + / + UserScript -> import cls -> call cls method --+----> TokenX class method + \ + `--> Default (no token) + class method + +.. code-block:: python + + # Example + # ------- + # + # a simple lookup decorator example + + # my_library/config.py + # -------------------- + + # import the decorator + # (note the lowercase 'lookup') + from abstract import lookup + + # define a class using the decorator on its methods + class ConfigureRouting(object) + def __init__(self, os): + self.os = os + + # apply the decorator on methods to be abstracted + @lookup('os') + def apply_config(self): + # ... insert generic/non-os specific code here + + + # my_library/nxos/config.py + # ------------------------- + from ..config import ConfigureRouting as BaseConfigRouting + + # inherit the parent class + class ConfigureRouting(BaseConfigRouting): + + # define the same method specific to this token + def apply_config(self): + # ... insert nxos specific code here + +The main benefit of using ``LookupDecorator`` is that it allows the user to +perform standard python ``import`` and deal with only one class instance. +During runtime, the engine looks up the class's attributes and forms a list of +tokens based on these values, and replaces the decorated methods during with a +"more" appropriate one from a tokenized search +(see :ref:`abstract_search_algorithm`). + +.. code-block:: python + + # Example + # ------- + # + # using the above code + + # import the main entry class directly + from my_library.config import ConfigureRouting + + # use it as you would naturally + obj = ConfigureRouting(os = 'nxos') + + # when a decorated method is called, the lookup occurs and the + # most appriorate method from one of its subclasses is called instead. + result = obj.apply_config() + # lookup information + # ------------------ + # attributes to read: os + # attribute value: os = 'nxos' + # + # thus, the search result equivalence is: + # from my_library.nxos.config import ConfigureRouting + # result = ConfigureRouting.apply_config(obj) + + +Usages +------ + +To use ``LookupDecorator``, start with writing your abstraction-enabled library +as you normally would. When arriving at defining classes that requires methods +level abstraction, simply apply the decorator onto each method that needs to be +abstracted. Behaviors: + +- Lookup decorator can be imported as ``lookup`` (note the lowercase), or as + ``decorator.LookupDecorator``. They are exactly the same, but some may prefer + one name over the other. + + .. code-block:: python + + from abstract import lookup + from abstract.decorator import LookupDecorator + +- The usage of lookup decorator does not mandate a top-level + :ref:`abstraction_pkg` declaration. It only requires :ref:`abstraction_tokens` + definitions under the module where the lookup decorator is used. + + .. code-block:: text + + Example: + if LookupDecorator is used in on class X under module A.B, + tokens should be defined as child modules under A.B. + +- Lookup decorator takes in a list of **attributes names** as arguments. During + runtime, the engine will lookup the given class instance for these attributes + to be used as tokens. This mechanism is called an *attribute getter*. The + default attribute getter looks up both the class instance and + ``instance.device`` (if exists) for the named attribute. + + .. code-block:: python + + class MyClass(object): + + @lookup('attr_1', 'attr_2') + def some_func(self): + # ... + + # equivalent to + # obj = MyClass() + # token_1 = getattr(obj, 'attr_1', getattr(obj.device, 'attr_1')) + # token_2 = getattr(obj, 'attr_2', getattr(obj.device, 'attr_2')) + +- The search for matching token combinations always begins at this class's + module declaration level onwards. It will match for the same **relative path** + as the current module, and the same class name (or names in nested class defs) + and target method. + + .. code-block:: text + + Example: + a search originating from: moduleX.moduleY.classA.classB.some_func() + may match: moduleX.moduleY.tokenJ.tokenK.classA.classB.some_func() + +- the default *attribute getter* can be replaced by providing a new function + through ``attr_getter`` argument. The provided function must take in two + arguments: ``obj`` and ``attr`` for both the object under scrutiny and the + attribute to lookup + +.. code-block:: python + + # Examples + # -------- + # + # lookup decorator usage + + # assuming we had a lookup-decorator enabled library + # my_library.my_module.ConfigureOspf + + # import it regularly + from my_library.my_module import ConfigureOspf + + # instaciate it naturally + # (in this case our class requires argument 'os' and mgmt_context) + routing = ConfigureOspf(os = 'iosxr', mgmt_context = 'yang') + + # if we call a decorated method, say, apply_configuration + # eg, code snippet: + # @lookup('os', 'mgmt_context') + # def apply_configuration(self): + # # ... code + + routing.apply_configuration() + # the engine translates this to: + # token_os = routing.os = 'iosxr' + # token_mgmt_context = routing.mgmt_context = 'yang' + # and the resulting lookup equivalent could be: + # from my_library.my_module.iosxr.yang import ConfigureOspf + # result = ConfigureOspf.apply_configuration(routing) + + # note + # ---- + # after lookup is performed, notice that the found target class's method + # is called directly with the original class instance as first argument. + # This is a python property: class methods can be treated as "functions" + # if you pass in a "similar" class instance as the first argument. + # See: https://docs.python.org/3.4/tutorial/classes.html#method-objects + +.. csv-table:: LookupDecorator Class Argument List + :header: "Argument", "Description" + + ``*attrs``, "list of attributes to be used read as input tokens for lookup" + ``attr_getter``, "class instance attribute getter (optional)" + ``builder``, "token permutation builder (optional)" + ``**builder_kwargs``, "any keyword arguments/values to be passed to the + builder (optional)" + +Lookup From Device Decorator +============================ + +``LookupDecorator.from_device`` is a feature extension to ``LookupDecorator``. +The lookup.from_device decorator operates at the runtime, allowing users to +write a **single class** with different method implementations and dynamically +based on the token variance combination from device's custom abstraction or +pre-defined at class method level. + +.. code-block:: python + + # Example + # ------- + # + # a simple lookup.from_device decorator example + + # my_library/config.py + # -------------------- + + # import the decorator + # (note the lowercase 'lookup') + from abstract import lookup + + # define a class using the decorator on its methods + class ConfigureRouting(object) + def __init__(self, os): + self.os = os + + # apply the decorator on methods to be abstracted dynamically based on + # custom abstraction data + @lookup.from_device + def apply_config(self): + # ... insert generic/non-os specific code here + + # apply the decorator on methods to be abstracted dynamically based on + # custom abstraction data or fallback to token 'os' + @lookup.from_device('os') + def check_config(self): + # ... insert generic/non-os specific code here diff --git a/docs/abstract/concept.rst b/docs/abstract/concept.rst index 0622577..a0247aa 100644 --- a/docs/abstract/concept.rst +++ b/docs/abstract/concept.rst @@ -12,58 +12,41 @@ Concept - :pythonimportsystem:`Python Import System ` -``abstract`` package is built upon the principle of dynamically looking up and -calling the right set of classes/functions/methods based on *requirements*. -These requirements are defined in each **abstraction enabled package** in the -form of **tokens**. +``abstract`` package is built upon the principle of dynamically looking up and +calling the right set of classes/functions/methods based on *requirements*. +These requirements are defined in the form of **tokens**, which describe the +types of device that a specific implementation supports. .. _abstraction_pkg: Abstraction-Enabled Package --------------------------- -An abstracton-enabled package is simply any regular Python package declared to -be abstraction-compatible using the ``abstract.declare_package()`` API. Beyond -that, an abstraction-enabled package behaves no differently than any other -standard Python modules. +An abstraction-enabled package is simply any regular Python package declared to +be abstraction-compatible using the ``abstract.declare_package()`` API. .. code-block:: python # Example # ------- - # # abstraction-enabled package example # assuming the following directory structure - # my_package/ - # | - # |-- __init__.py - # | - # `-- + # mypackage/ + # ├── __init__.py + # └── # declare abstraction-package at the top of my_package/__init__.py from genie import abstract - abstract.declare_package(__name__) - - # Note - # ---- - # - # - the above is equivalent to: - # from abstract import declare_package - # declare_package(__name__) - # - # - __name__ is the name of this module - -The call to ``abstract.declare_package(__name__)`` internally -flags the given module to be an abstraction package. This is a mandatory step -when creating a module to be abstraction-compatible, and does not change the -behavior of how this package normally behaves. + abstract.declare_package() + +Beyond that, an abstraction-enabled package behaves no differently than any other +standard Python module. .. code-block:: python # Example # ------- - # # an abstraction-enabled package is just like any other package # you can import it @@ -80,286 +63,341 @@ behavior of how this package normally behaves. Abstraction Tokens ------------------ -An abstraction token is simply a child-module within an abstraction-enabled -package. It is declared by calling ``abstract.declare_token()`` API. Similar to -the above, these are still... Python modules. +An abstraction token is a device description attached to a child-module within +an abstraction-enabled package. It is declared by calling +``abstract.declare_token(=)`` API, which is how we define what +kind of device this child-module is meant to support. Similar to the above, +these are still regular Python modules and can be imported and used +as such. -.. code-block:: python +.. code-block:: text # Example # ------- - # # abstraction-enabled package with tokens - # assuming the following directory sturcture - # my_package/ - # | - # |-- __init__.py - # | - # |-- token_one/ - # | `-- __init__.py - # | - # `-- token_two/ - # |-- __init__.py - # | - # `-- token_two_one/ - # `-- __init__.py - - # abstraction-token is declared at the top of - # - my_package/token_one/__init__.py - # - my_package/token_two/__init__.py - # - my_package/token_two_one/__init__.py + # assuming the following directory structure + # mypackage/ + # ├── __init__.py + # ├── nxos/ + # │ └── __init__.py + # └── iosxe/ + # ├── __init__.py + # └── cat9k/ + # └── __init__.py + + +Abstraction tokens are declared at the top of the ``__init__.py`` file + + +.. code-block:: python + + # - my_package/nxos/__init__.py from genie import abstract - abstract.declare_token(__name__) - - # Note - # ---- - # - # - the above is equivalent to: - # from abstract import declare_token - # declare_token(__name__) - # - # - __name__ is the name of this module - - # keep in mind that this does not alter the nature of python modules - # it can still be imported - from my_package.token_one import my_class - from my_package.token_two import token_two_one - from my_package.token_two_one.token_two_one import my_other_class - -Each abstraction token represents an alternate set of libraries, capable of -handling the differences introduced/labelled by the **token** name. For example, -if a package contains token ``nxos``, it suggests that the libraries following -this token module is specific to Cisco NXOS. - -In addition, tokens may be chained/nested. This allows for library tiering. For -example, if token ``yang`` is declared under token ``nxos``, it suggests that -these libraries would be specific to Cisco NXOS's NETCONF/YANG implementation. + abstract.declare_token(os='nxos') -.. note:: - Tokens may carry arbitrary names. Use token naming wisely to depict - differences where you want to abstract your libraries. For more details, - refer to :ref:`abstraction_conventions`. +.. code-block:: python -.. tip:: - - Follow PEP8 - :modulenamingconvention:`module naming convention `. + # - my_package/iosxe/__init__.py + from genie import abstract + abstract.declare_token(os='iosxe') + +.. code-block:: python + + # - my_package/iosxe/cat9k/__init__.py + from genie import abstract + abstract.declare_token(platform='cat9k') + +Keep in mind that this does not alter the nature of python modules, they +can still be imported as usual + +.. code-block:: python + + from my_package.nxos import my_class + from my_package.iosxe import my_class + from my_package.iosxe.cat9k import my_class + +Each abstraction token represents an alternate set of libraries, capable of +handling the differences introduced/labelled by the **token** value. For example, +if a package contains token ``os='nxos'``, it suggests that the libraries following +this token module is specific to Cisco NXOS. + +.. _token_order: + +In addition, tokens may be nested. For example, if the token ``platform='n7k'`` +is declared under token ``os='nxos'``, it suggests that these libraries would be +specific to Cisco Nexus 7000 Series Switches. There is a specific order of device +attributes for tokens, which **must** be followed while nesting: + +.. code-block:: markdown + + - os + - platform + - model + - pid + - version + - revision + +.. warning:: + + Tokens may *not* carry arbitrary names or values. They should match the values + defined in the Unicon `PID tokens`_ file which is the source of truth for device + definitions. If the token values for your particular device are not present + please consider contributing a new definition to the file. + +.. _PID tokens: https://github.com/CiscoTestAutomation/unicon.plugins/blob/master/src/unicon/plugins/pid_tokens.csv Abstraction Mechanism --------------------- -The ``abstract`` module works most of its magic at the Python ``import`` and -``getattr()`` level. It does so by dissecting each lookup into three distinct -parts: +Retrieving features with abstraction will attempt to find the most number of +token matches to return an implementation for that feature. An implementation +matching both OS and platform would be returned instead of just the +implementation matching OS, as the first would be more specific for the given +device. - - **relative path**: the primary lookup path that makes the most sense from - a functional perspective. This is what the user references directly, eg: - ``my_library.routing.ospf`` - - **tokens**: the list of abstraction tokens currently known by the - abstraction engine. This portion is registered through the ``Lookup`` - object. Eg: ``iosxr``, ``fretta``, ``xml``. +Examples +######## - - **target**: the module/class/function/variable user is looking for. +Parser code exists in a directory structure like so: -During runtime, the lookup engine dynamically pieces together the above -information into a list of possible candidate **absolute paths** (direct mapping -to python import statements). As the list of tokens is arbitrary, this candidate -list is built following the :ref:`abstract_search_algorithm`. +.. code-block:: text -.. code-block:: python + parser/ + ├── __init__.py + └── iosxe/ + ├── __init__.py + ├── show_feature.py + └── cat9k/ + ├── __init__.py + ├── show_feature.py + └── c9300/ + ├── __init__.py + └── show_feature.py + +Each level of ``show_feature.py`` has an implementation for the `show feature` +parser, but only ``parser/iosxe/show_feature.py`` and +``parser/iosxe/cat9k/show_feature.py`` have implementations for the `show other feature` +parser. Note that the `c9700` folder does **not** exist. + +.. list-table:: Show Feature Parser + :header-rows: 1 + :align: center + + * - Parser Path + - show feature + - show other feature + * - iosxe + - ✔️ + - ✔️ + * - iosxe/cat9k + - ✔️ + - ✔️ + * - iosxe/cat9k/c9300 + - ✔️ + - ❌ + * - iosxe/cat9k/c9700 + - ❌ + - ❌ - # Example - # ------- - # - # relative path & absolute path explained - - # Given the following tokens: - # - iosxe - # - polaris_dev - # - yang - os = 'iosxe' - branch = 'polaris_dev' - context = 'yang' - - # feed to to abstraction lookup engine. - library = abstract.Lookup(os, branch, context) - - # the relative call to - library.my_package.config.routing.ospf.Ospf() - - # could match, for example: - # - # my_package.iosxe.config.polaris_dev.routing.ospf.yang.Ospf - # | | | | | | | | - # abstraction | relative | relative | | class - # package | path | path | token - # token token relative - # path - # which translates to: - # from my_package.iosxe.config.polaris_dev.routing.ospf.yang import Ospf - # - # where - # ----- - # relative path = config, routing, ospf - # tokens = iosxe, polaris_dev, yang - # target = Ospf() - - -.. _abstract_search_algorithm: - -Search Algorithm ----------------- - -The search engine combines the user's **relative path** and currently known -**tokens** into possible **absolute paths** (python module names) and searches -through them. A match occurs when an implementation is found (ie the target -exists at the candidate relative path). Otherwise, the next combination is -tried. If no target is found, a ``LookupError`` would be thrown. - -As the token names are not pre-defined, the search engine orders -all tokens in a pre-defined fashion: - - - token describes a set of *differences* - - token positions are always fixed w.r.t. to its left (parent) - - tokens on the right are more *specific* than tokens on the left - - each token may only appear *once* in a combination - - greedy match: more tokens matches is always better than less. +.. note:: -.. code-block:: text + A reminder of purpose of the different levels of ``show_feature.py`` - Given tokens: a, b, c and d, the preferred token combination would be: + - Generic implementations for any IOSXE device. + - More specific implementations for the Catalyst 9000 Platform. + - Even more specific implementations for only the Catalyst 9300 model. - a b c d - a b c - a b - a - (no tokens) +We have two devices, -These combinations are then *multiplexed* to user's **relative path** into -potential **absolute paths** to search for, using the following rules: - - - absolute paths must always start with the abstracted package name. +- ``Device_A`` which can be defined by the tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'``. +- ``Device_B`` which can be defined by the tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9700'``. - - the order of relative path sections (words divided by ``.``) must be - preserved. +Note the different models of the two devices. - - the order of token combinations must be preserved. +Example 1 +^^^^^^^^^ - - tokens may take place before and after each relative path section, and may - appear in multiples together. (eg, ``library.iosxr.google.latest.mpls``) +.. tabs:: - - the last resort option is to try with "no token", eg, matching the - relative path directly. + .. tab:: 1 -Combining the above rules, the ideal solution would be a multi-combinatory -mathematical function, whose search complexity is ... *(insert math here)* ... -exponential. + We want to parse `show feature` with ``Device_A``. -.. code-block:: text - - Given Package: my_pkg - Relative Path: X, Y - Tokens: a, b - Target: MyClass() - - We could have the following mathmatical combinational possibilities: - - 1. my_pkg.a.X.b.Y.MyClass() - 2. my_pkg.a.X.Y.b.MyClass() - 3. my_pkg.X.a.Y.b.MyClass() - 4. my_pkg.X.a.b.Y.MyClass() - 5. my_pkg.X.Y.a.b.MyClass() - 6. my_pkg.a.X.Y.MyClass() - 7. my_pkg.X.a.Y.MyClass() - 8. my_pkg.X.Y.a.MyClass() - 9. my_pkg.X.Y.MyClass() - - And that's just with two tokens and two path sections! - -The actual implementation internally is much simpler. When an an abstracted -package is defined/declared and the lookup object is created, the package and -all of its child modules are *recursively imported*. This allows the abstraction -engine to build an internal table of relative paths, their available token -combinations learnt from the import and its corresponding module. This reduced -**relative path + tokens** relationship effectively simplies the above -brute-force search algorithm into an ``O(n)`` lookup, where ``n`` is the number -of tokens. + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` -.. code-block:: text + .. tab:: 2 - Pseudo Lookup Table - =================== + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. - Relative Path Tokens Combos Corresponding Module - ------------- ------------- -------------------- - X.Y a, b X.a.Y.b - X.Y a X.a.Y - X.Y None X.Y + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! - (shown in order of preference, from top down) + .. tab:: 3 -This algorithm limits to only dealing with what's been defined in the user -library, instead of going through all possible permutations of **relative path** -and **tokens**. The system assumes that it is unlikely for users to make -redundant declarations, such as defining both ``from X.a.Y.b import target`` and -``from X.a.b.Y import target`` within the same library. + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. An implementation + of the `show feature` parser exists here. -.. note:: - - The learning process safeguards against these redundant scenarios. + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> `show feature` parser exists! + .. tab:: 4 -.. _token_builder: + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. An implementation + of the `show feature` parser exists here. This implementation is returned. -Token Builder -------------- + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> `show feature` parser exists! + - Return found parser -The token builder is a simple function that implements the token permutation -portion of the :ref:`abstract_search_algorithm`. The default token builder is -available as ``abstract.magic.default_builder()``. +Example 2 +^^^^^^^^^ -.. csv-table:: default_builder Argument List - :header: "Argument", "Description" +.. tabs:: - ``tokens``, "list of tokens to permute" - ``mandatory``, "list of tokens that must be used" + .. tab:: 1 -.. code-block:: python + We want to parse `show feature` with ``Device_B``. + + - ``Device_B`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9700'`` + + .. tab:: 2 + + We want to parse `show feature` with ``Device_B``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9700/show_feature.py``. + + - ``Device_B`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9700'`` + - ``parser/iosxe/cat9k/c9700/show_feature.py`` -> File does not exist! + + .. tab:: 3 + + We want to parse `show feature` with ``Device_B``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9700/show_feature.py``. This file does + not exist so the abstraction mechanism falls back to the next best match, + ``parser/iosxe/cat9k/show_feature.py``. + + - ``Device_B`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9700'`` + - ``parser/iosxe/cat9k/c9700/show_feature.py`` -> File does not exist! -> Fallback! + - ``parser/iosxe/cat9k/show_feature.py`` -> File exists! + + .. tab:: 4 + + We want to parse `show feature` with ``Device_B``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9700/show_feature.py``. This file does + not exist so the abstraction mechanism falls back to the next best match, + ``parser/iosxe/cat9k/show_feature.py``. An implementation of the + `show feature` parser exists here. + + - ``Device_B`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9700'`` + - ``parser/iosxe/cat9k/c9700/show_feature.py`` -> File does not exist! -> Fallback! + - ``parser/iosxe/cat9k/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/show_feature.py`` -> `show feature` parser exists! + + .. tab:: 5 + + We want to parse `show feature` with ``Device_B``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9700/show_feature.py``. This file does + not exist so the abstraction mechanism falls back to the next best match, + ``parser/iosxe/cat9k/show_feature.py``. An implementation of the + `show feature` parser exists here. This implementation is returned. + + - ``Device_B`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9700'`` + - ``parser/iosxe/cat9k/c9700/show_feature.py`` -> File does not exist! -> Fallback! + - ``parser/iosxe/cat9k/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/show_feature.py`` -> `show feature` parser exists! + - Return found parser + +Example 3 +^^^^^^^^^ + +.. tabs:: + + .. tab:: 1 + + We want to parse `show other feature` with ``Device_A``. + + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + + .. tab:: 2 + + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. + + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! + + .. tab:: 3 + + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. This file exists + but does not have an implementation of the `show other feature` parser + that we want. + + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> `show other feature` parser does not exist! + + .. tab:: 4 + + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. This file exists + but does not have an implementation of the `show other feature` parser + that we want. The abstraction mechanism falls back to the next best match + which is ``parser/iosxe/cat9k/show_feature.py``. + + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> `show other feature` parser does not exist! -> Fallback! + - ``parser/iosxe/cat9k/show_feature.py`` -> File exists! + + .. tab:: 5 + + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. This file exists + but does not have an implementation of the `show other feature` parser + that we want. The abstraction mechanism falls back to the next best match + which is ``parser/iosxe/cat9k/show_feature.py``. An implementation of the + `show other feature` parser exists here. + + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> `show other feature` parser does not exist! -> Fallback! + - ``parser/iosxe/cat9k/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/show_feature.py`` -> `show other feature` parser exists! + + .. tab:: 6 + + We want to parse `show feature` with ``Device_A``. The abstraction + mechanism will find the file that matches the most number of tokens, + which is ``parser/iosxe/cat9k/c9300/show_feature.py``. This file exists + but does not have an implementation of the `show other feature` parser + that we want. The abstraction mechanism falls back to the next best match + which is ``parser/iosxe/cat9k/show_feature.py``. An implementation of the + `show other feature` parser exists here. This implementation is returned. + + - ``Device_A`` tokens ``os='iosxe'``, ``platform='cat9k'``, ``model='c9300'`` + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/c9300/show_feature.py`` -> `show other feature` parser does not exist! -> Fallback! + - ``parser/iosxe/cat9k/show_feature.py`` -> File exists! + - ``parser/iosxe/cat9k/show_feature.py`` -> `show other feature` parser exists! + - Return found parser - # Example - # ------- - # - # pseudo code demonstrating the behavior of default token builder - - from abstract.magic import default_builder - - # without any mandatory tokens - default_builder(tokens = ['nxos', 'n7k', 'c7003', 'yang', 'R8_1']) - # [('nxos', 'n7k', 'c7003', 'yang', 'R8_1'), - # ('nxos', 'n7k', 'c7003', 'yang'), - # ('nxos', 'n7k', 'c7003'), - # ('nxos', 'n7k'), - # ('nxos',), - # ()] - - # a mandatory token is one that MUST be used in the search - default_builder(tokens = ['nxos', 'n7k', 'c7003', 'yang', 'R8_1'], - mandatory = ['yang']) - # [('nxos', 'n7k', 'c7003', 'yang', 'R8_1'), - # ('nxos', 'n7k', 'c7003', 'yang'), - # ('nxos', 'n7k', 'yang'), - # ('nxos', 'yang'), - # ('yang',)] - -In essence, the "tokens" input parameter to the builder is a reflection of -the actual, longest possible chain of tokens under any given relative path. If -no target is found at this token/relative path combination, the next, reduced -set of tokens is tried. This reduction mechanism always reduces from the right. - -Use the ``mandatory`` input argument when you absolutely require some tokens to -be present in any token permutations during abstraction. This can be useful when -you do not want the system to automatically fallback using the above logic and -remove it. This ensures the proper "set" of libraries is picked. diff --git a/docs/abstract/contributing.rst b/docs/abstract/contributing.rst new file mode 100644 index 0000000..9f97f68 --- /dev/null +++ b/docs/abstract/contributing.rst @@ -0,0 +1,111 @@ +.. _abstraction_contributing: + +Adding and Modifying +==================== + +The primary benefit of using ``abstract`` module is its ease of use combined +with long-term scalability. With it, abstraction is no longer a day-one, +front-load investment, but rather a "create more specific implementations as +required" dynamic system. While a feature may exist under the token ``os="iosxe"``, +another implementation can be created at a later date to accommodate differences +in behaviour for specific platforms (eg. ``platform="cat9k"``) which exist under +the previous token. + +.. warning:: + + Tokens may *not* carry arbitrary names or values, and must adhere to a + :ref:`specific hierarchy `. They should match the values + defined in the Unicon `PID tokens`_ file which is the source of truth for device + definitions. If the token values for your particular device are not present + please consider contributing a new definition to the file. + +.. _PID tokens: https://github.com/CiscoTestAutomation/unicon.plugins/blob/master/src/unicon/plugins/pid_tokens.csv + +.. tip:: + + Before adding any new features makes sure to check the `Genie Feature Browser`_ + to ensure that it doesn't already exist, or if it already exists for another + type of device. + + +.. _Genie Feature Browser: https://pubhub.devnetcloud.com/media/genie-feature-browser/docs/#/ + + +Library Structure +----------------- + +Always group methods and classes together using some form of system (such as +by functionality) into modules & submodules. This will add depth and structure +to your libraries, simplifying user's understanding & maintenance. + +As abstracted packages and tokens are simply python packages and modules, the +standard python :pythonpackages:`Python Packages ` guidelines apply, +and your directory structure should naturally fall into place. + + +.. code-block:: text + + Library Structure + ----------------- + + library_root/ + ├── __init__.py # declare_package() + ├── module.py # any default (no token) modules + │ + ├── submodule/ # and regular Python submodules. + │ ├── __init__.py + │ └── module.py + │ + ├── / # a token definition + │ ├── __init__.py # declare_token(os='') + │ │ + │ ├── module.py # OS specific modules + │ ├── submodule/ # and submodules + │ │ ├── __init__.py + │ │ └── module.py + │ │ + │ └── / # a token under another token + │ ├── __init__.py # declare_token(platform='') + │ ├── module.py + │ └── submodule/ # platform specific submodules for above OS + │ ├── __init__.py + │ └── module.py + ... + +.. code-block:: text + + Example Library + --------------- + + The following is implemented: + - generic reload, verify, configure ospf & interface + - iosxr default reload, configure interface + - iosxr/crs specific reload, configure interface + + example_abstracted_apis/ + ├── __init__.py # declare_package() + | + ├── reload.py + ├── verify.py + | + ├── configure/ + │ ├── __init__.py + │ ├── ospf.py + │ └── interface.py + | + ├── iosxr/ + | ├── __init__.py # declare_token(os='iosxr') + | | + | ├── reload.py + | ├── configure/ + | │ ├── __init__.py + | │ └── interface.py + | | + | └── asr9k/ + | ├── __init__.py # declare_token(platform='asr9k') + | ├── reload.py + | └── configure/ + | ├── __init__.py + | └── interface.py + ... + diff --git a/docs/abstract/conventions.rst b/docs/abstract/conventions.rst deleted file mode 100644 index f875ab3..0000000 --- a/docs/abstract/conventions.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _abstraction_conventions: - -Conventions -=========== - -The primary benefit of using ``abstract`` module is its ease of use combined -with long-term scalability. With it, abstraction is no longer a day-one, -front-load investment, but rather an "add a token as you need" dynamic system. - -In order to fully reap the benefits offered by abstraction and allowing for -sharing between teams and projects, considering following the conventions -below during your coding. - - -Rules of Thumb --------------- - -- when creating a new library, adding the :ref:`abstraction_pkg` declaration - at the root ``__init__.py`` file is always a good habit to get into. There's - no side effect and/or runtime costs. - -- add your methods, modules as you see fit, but focus on on tackling one - combination (tokenset) at a time (eg, start with writing libraries for your - uut: iosxr/sunstone/latest) - -- always create ``Lookup`` objects for each device as one of the first things in - your script's CommonSetup section. This will ensure the rest of your script - have proper access to abstracted packages & libraries. - -- when needing to call/reference any library functions & classes, always do so - through the ``Lookup`` object created above. - -- when a new set of combination (tokensets) needs to be added, consider their - relationships and group like-levels together. - - .. code-block:: text - - if you started with the following for iosxr/sunstone/latest - /my_lib/__init__.py - /my_lib/file.py - - and now needs to add iosxr/enxr, consider refactoring to: - /my_lib/iosxr/sunstone/__init__.py - /my_lib/iosxr/sunstone/file.py - /my_lib/iosxr/enxr/__init__.py - /my_lib/iosxr/enxr/file.py - - remember - a module is just a folder. You're merely shuffling files. - -- create new tokens with meaningful names that reflects the set of differences - it covers. Eg, os names, relese variants, branches, etc. - -- use python inheritance between your various implementations for... a variety - of OOP-related benefits. - -- if your library requires a set of *defaults*, eg, fallback functions/classes - when no tokens match, define them directly under the root package module. The - engine will auto-fallback to it. - -- consider sharing your code with the rest of the community. - -- Follow PEP8 - :modulenamingconvention:`module naming convention `. - - -Library Structure ------------------ - -Always group methods and classes together using some form of system (such as -by funtionality) into modules & submodules. This will add depth and struture -to your libraries, simplifying user's understanding & maintenance. - -As abstracted packages and tokens are simply python packages and modules, the -standard python :pythonpackages:`Python Packages ` guidelines apply, -and your directory structure should naturally fall into place. - - -.. code-block:: text - - Library Structure - ----------------- - - library_root/ - |-- __init__.py # declare_package() - | - |-- module.py # any default (no token) modules - |-- submodule/ # and submodules. - | |-- __init__.py - | `-- submodule_module.py - | - |-- token_module/ # a new token - | |-- __init__.py # declare_token() - | | - | |-- module.py # token specific modules - | |-- submodule/ # and submodules - | | |-- __init__.py - | | `-- submodule_module.py - | | - | `-- token_subtoken_module/ # a token under another token - | |-- __init__.py # declare_token - | |-- module.py - | `-- submodule/ # ditto - | |-- __init__.py - | `-- submodule_module.py - | - ... - -.. code-block:: text - - Example Library - --------------- - - The following is implemented: - - generic reload, verify, configure ospf & interface - - iosxr default reload, configure interface - - iosxr/crs specific reload, configure interface - - example_abstracted_module/ - |-- __init__.py - | - |-- reload.py - |-- verify.py - | - |-- configure/ - | |-- __init__.py - | |-- ospf.py - | `-- interface.py - | - |-- iosxr/ - | |-- __init__.py - | | - | |-- reload.py - | |-- configure/ - | | |-- __init__.py - | | `-- interface.py - | | - | `-- crs/ - | |-- __init__.py - | |-- reload.py - | `-- configure/ - | |-- __init__.py - | `-- interface.py - | - ... - diff --git a/docs/abstract/index.rst b/docs/abstract/index.rst index ea4d152..71df546 100644 --- a/docs/abstract/index.rst +++ b/docs/abstract/index.rst @@ -9,12 +9,6 @@ scripts without hard-coded imports. For more details, refer to :pyats:`pyATS `. -.. tip:: - - It is **strongly** recommended that all scripts to be written using this - abstraction package. This will vastly reduce future maintenance work if the - script is to be re-used. Wanna know why? Read on... - .. important:: The abstraction package is undergoing a revision to more consistently @@ -32,8 +26,7 @@ For more details, refer to :pyats:`pyATS `. introduction concept - lookup_class - lookup_decorator - conventions + contributing + revisions apidoc diff --git a/docs/abstract/introduction.rst b/docs/abstract/introduction.rst index 1967fbd..fb06a37 100644 --- a/docs/abstract/introduction.rst +++ b/docs/abstract/introduction.rst @@ -4,17 +4,17 @@ Introduction In software engineering and computer science, **abstraction** is a technique for managing complexity of computer systems. It works by establishing a level of complexity on which a person interacts with the system, suppressing the more -complex details below the current level. The programmer works with an -idealized interface (usually well defined) and can add additional levels of +complex details below the current level. The programmer works with an +idealized interface (usually well defined) and can add additional levels of functionality that would otherwise be too complex to handle. (:wikipedia:`Wikipedia `) As Python is a :dynamicallytyped:`dynamically typed `, :objectoriented:`object-oriented ` programming language, function/code -and data abstraction can be achieved easily through +and data abstraction can be achieved easily through :ducktyping:`duck typing ` and :inheritance:`inheritance `: defining classes/objects that behaves similarly in a given context, and hiding details of implementation under its methods and properties. - + *If it flies like a duck & quacks like a duck, it's a duck!* .. code-block:: python @@ -51,141 +51,20 @@ details of implementation under its methods and properties. The above stipulates how classes can be defined to behave similarly (eg, through -similar interfaces and subclassing). However, it doesn't solve how the system -should pick the *correct class* during runtime. Often times, additional logic is -required to sort out which appropriate class should be used, based on given -information. - -.. code-block:: python - - # Example - # ------- - # - # try and pick the class by breed - - if breed == "human": - cls = Person - else: - cls = Duck - - mallard = cls() - - # ... etc - -Eventually, this logic leads us to factory methods. - -.. code-block:: python - - # Example - # ------- - # - # factory method creating various shapes - - class Shape(object): - types = [] - - class Circle(Shape): - def draw(self): print("Circle.draw") - def erase(self): print("Circle.erase") - - class Square(Shape): - def draw(self): print("Square.draw") - def erase(self): print("Square.erase") - - def factory(type): - '''I'm the factory method!''' - - if type == "Circle": return Circle() - if type == "Square": return Square() - raise ValueError("Bad shape creation: " + type) - -Factory Method - a creational pattern that uses factory methods to deal with the problem - of creating objects without having to specify the exact class of the - object that will be created. - -:FactoryMethods:`Factory Methods ` allowers users to create classes using "requirements" as -inputs, returning the corresponding class objects. The main challenge with this -approach is **its dependency on the creator**: factory methods are only as -powerful as its creator's coding. Adding support for more requirements and/or -classes requires modification to the original code. Further, as each person may -choose to implement their own set of logic and requirements... the similarity, -traceability and debuggability of factory methods across the board may be poor. +similar interfaces and subclassing). This is the underlying principle for Genie +Libraries (ie. parsers and APIs). However, it doesn't solve how the system +should pick the *correct class* during runtime. Our Solution ------------ -The ``abstract`` package is intended solve the above issue by **standardizing the -abstraction decision making process**. Through the use of abstraction tokens & -lookup algorithms, the package empowers users to write agnostic libraries and -scripts capable of handling a variety of differences between -os/platform/feature/release/mgmt interface, etc. +The ``abstract`` package is intended solve the above issue by **standardizing the +abstraction decision making process**. Through the use of abstraction tokens +gathered from devices & lookup algorithms, the package empowers users to write +agnostic libraries and scripts capable of handling a variety of differences between +os/platform/model, etc. .. figure:: abstract.png :align: center - Abstraction Concept - -.. code-block:: python - - # Example - # ------- - # - # with and without abstraction - - # typical non-abstracted script - # ----------------------------- - # import the proper function through if statements - if release == 'v2.1': - if context == 'YANG': - from my_library.v2_1.yang import configure_something - else: - from my_library.v2_1.cli import configure_something - elif release == 'v2.2': - if context == 'YANG': - from my_library.v2_2.yang import configure_something - else: - from my_library.v2_2.cli import configure_something - else: - if context == 'YANG': - from my_library.generic import configure_something - else: - from my_library.generic import configure_something - - # get result - result = configure_something() - - # using abstraction & properly abstracted libraries - # ------------------------------------------------- - from genie import abstract - - # build a lookup object and pass the release/context as tokens - lookup = abstract.Lookup(release, context) - - # collect result by looking up the corresponding API - result = lookup.my_library.configure_something() - - -As show above, through the use of ``abstract`` package, users can write -straightforward codes (single-source) that automatically invokes the right set -of library APIs (classes, functions, methods etc) based on given requirements, -without the repeated use of custom ``if..elif..elif..else`` statements -everywhere. This dynamic library referencing can be beneficial in many use -cases, including but not limited to: - - - handling minute release-to-release, image-to-image differences - - running the same tests/scripts across different management interfaces: - CLI, YANG, XML - - running the same tests/scripts across a variety of hardware (controllers, - linecards, interfaces, os/platforms, etc) - - -Support -------- - -Reach out to :mailto:`contact us ` for any questions or issues related to the -``genie.abstract`` package. - -You can also post questions to the :communityforum:`community forum ` - the support team patrols -these forums daily. diff --git a/docs/abstract/lookup_class.rst b/docs/abstract/lookup_class.rst deleted file mode 100644 index b4c0beb..0000000 --- a/docs/abstract/lookup_class.rst +++ /dev/null @@ -1,329 +0,0 @@ -.. _abstract_lookup_cls: - -Lookup Class -============ - -``Lookup`` class is the main feature of ``abstract`` package. It implements -:ref:`Abstraction Concepts ` in a user-friendly fashion, -and allows users to perform dynamic lookups just as if they were accessing -object attributes. - -.. code-block:: text - - .------> TokenX.Y implementation - / - UserScript -> Lookup Target --+--------> Token X implementation - (func/cls/var) \ - `------> Default (no token) implementation - - -Usages ------- - -When instanciated with a list of :ref:`abstraction_tokens`, ``Lookup`` class -allows the user to reference any :ref:`abstraction_pkg` available in the current -namespace scope. This behavior can be generally summarized into the following: - -- at miminum, a list of :ref:`abstraction_tokens` is required in order to - instanciate a new ``Lookup`` object. - -- by default, all :ref:`Abstraction-Enabled Packages ` imported - and available at the scope where ``Lookup()`` is called, gets discovered and - registered internally. - -- if an package is a part of a parent package, it needs to be imported - directly into the current namespace. - - .. code-block:: python - - # instead of - import parent_package.my_abstracted_package - - # you must import it directly - from parent_package import my_abstracted_package - -- users can provide a dictionary of ``name: package`` to ``Lookup()`` and - override the default discovery behavior. ``name`` is the alias to refer to - the given package. - - .. code-block:: python - - import parent.my_package - - lookup = Lookup(*tokens, packages = {'pkg': parent.my_package}) - -- perform library lookups as if you were referencing attributes of an object. - - .. code-block:: python - - import my_abstracted_library - - lookup = Lookup(*tokens) - - # always start with the name of the library you want to search from - lookup.my_abstracted_library.some_module.some_other_module.Target() - -- the default :ref:`token_builder` supports specifying mandatory tokens. This - generator can be overwritten with ``builder`` argument to ``Lookup()`` (very - advanced functionality). - - .. code-block:: python - - from genie import abstract - from my_library import my_builder - - # use your default builder - lookup = Lookup(*tokens, builder = my_builder) - - -- in addition, this global default builder setting can be modified by setting - ``abstract.magic.DEFAULT_BUILDER`` to a builder of your liking. This will - affect **all** newly created ``Lookup()`` object from this point onwards. - - .. code-block:: python - - from genie import abstract - from my_library import my_default_builder - - # overwrite the default builder - abstract.magic.DEFAULT_BUILDER = my_default_builder - - # any lookup object created hereonward will take on your builder - lookup = Lookup(*tokens) - - -.. code-block:: python - - # Example - # ------- - # - # Lookup() class examples & features - - # import the class from abstract - from genie.abstract import Lookup - - # import any abstraction-enabled packages you need - import my_abstracted_library - from xbu_shared import genie, parser - - # create the lookup object and provide it with tokens - # this auto discovers and registers the above imported packages: - # my_abstracted_library, genie, parser - lookup = Lookup('iosxr') - - # now use the lookup object and reference the above imported - # libraries using attribute queries. Eg: - - result = lookup.my_abstracted_library.my_abstracted_function() - # runtime absolute path translation: - # from my_abstracted_library.iosxr import my_abstracted_function - # result = my_abstracted_function() - - ospf = lookup.genie.conf.ospf.Ospf() - # runtime absolute path translation: - # from xbu_shared.genie.conf.ospf.iosxr import Ospf - # ospf = Ospf() - - output = lookup.parser.ShowVersion(device = device) - # runtime absolute path translation: - # from xbu_shared.parser.iosxr import ShowVersion - # output = ShowVersion() - - # -------------------------------------------------------------------------- - - # create new Lookup() instances if tokens requirements change - # you can also change the set of packages available for it, - # as well as its base reference name. - lookup = Lookup('token_a', 'token_b', '...', 'etc', - packages = {'lib_1': my_abstracted_library, - 'lib_2': genie', - 'lib_3': parser}) - - # as new names are tokens are provided, we can now do: - result = lookup.lib_1.my_abstracted_function() - ospf = lookup.lib_2.conf.ospf.Ospf() - output = lookup.lib_3.ShowVersion(device = device) - -.. tip:: - - always use meaningful package names. - -.. csv-table:: Lookup Class Argument List - :header: "Argument", "Description" - - ``*token``, "list of tokens to be used as input requirements for to this - lookup" - ``packages``, "dictionary of name/abstraction package to lookup from - (optional)" - ``builder``, "token permutation builder (optional)" - ``**builder_kwargs``, "any keyword arguments/values to be passed to the - builder (optional)" - - -Integration with Topology -------------------------- - -``Lookup()`` class also features a classmethod constructor that enables it to -understand pyATS topology module's ``Device()`` object, and subsequently, create -lookup objects based on the tokens specified under ``Device.custom.abstraction`` -field. - -.. code-block:: yaml - - # Example - # ------- - # - # example pyATS topology device yaml - - device: - my-example-device: - type: router - os: iosxe - series: asr1k - custom: - abstraction: - order: [os, series, context] - context: yang - -.. code-block:: python - - # Example - # ------- - # - # using the above testbed definition with abstraction - - from pyats import topology - testbed = topology.loader.load('/path/to/above/testbed.yaml') - device = testbed.devices['my-example-device'] - - # create abstraction - from genie.abstract import Lookup - - lookup = Lookup.from_device(device) - # eg, the above is equivalent to: - # os = device.custom.abstraction.get('os', device.os) - # series = device.custom.abstraction.get('series', device.series) - # context = device.custom.abstraction.get('context') - # lookup = Lookup(os, series, context) - -In the above testbed YAML file, we defined a custom abstraction definition, -specifying the expected token list ``[os, series, context]``, and the expected -``context = 'yang'``. - -When ``Lookup.from_device()`` method is called, the tokens associated with that -device is automatically extracted following these rules: - - - ``device.custom.abstraction`` is a dictionary - - ``device.custom.abstraction['tokens']`` specifies the list of attributes - to read from this device object, and converted into token values. - - the code prefers to read the attributes from - ``device.custom.abstraction[attrbute]``, and falls back to - ``device.`` if needed. - -All other arguments to ``Lookup()``, such as ``builder, packages, -builder_kwargs`` also applies to this classmethod. - -If however you would like to not specify the ``device.custom.abstraction`` block -in your testbed YAML file all the time, you can provide ``default_tokens`` as a -list to ``Lookup.from_device()``. Any tokens specified there would be looked-up -from the provided device attribute. - -.. code-block:: python - - # Example - # ------- - # - # Lookup.from_device using defaults - - lookup = Lookup.from_device(device, default_tokens = ['os', 'series']) - # eg, the above is equivalent to: - # os = device.os - # series = device.serie - # lookup = Lookup(os, series) - -.. note:: - - note that when using ``default_tokens``, the lookup from device attribute - is non-strict, eg: if tokens ``a``, ``b``, ``c`` are specified, and only - ``a``, ``c`` exists, it will not error and just use these values instead. - - -Tips & Tricks -------------- - -Typically, abstraction should be used when the end library needs to handle -differences (such as OS/Release/Mgmt Interface) etc. This leads to a per-device -lookup model, where the set of :ref:`abstraction-tokens` per device differs. -The best, pythonic method to tackle this is to follow the natural patterns -of Python/pyATS programming: - -- ``import`` all your packages at the top of your script/code, including all - :ref:`Abstraction-Enabled Packages `. - -- inside AEtest ``CommonSetup`` section, as soon as you have connected to your - testbed devices and learnt about what they are, create your ``Lookup()`` - objects and assign them as an attribute to each ``Device`` instance. - -.. code-block:: python - - # Example - # ------- - # - # an example AEtest script with abstraction enabled - - # import everything at the top - import logging - from genie import abstract - from pyats import aetest - - # eg, these are my abstraction libraries - import my_abstracted_library - from xbu_shared import genie, parser - - logger = logging.getLogger(__name__) - - class CommonSetup(aetest.CommonSetup): - - @aetest.subsection - def connect_to_testbed(self, testbed): - for name, device in testbed.devices.items(): - device.connect() - logger.info('connected to device %s' % device.name) - - @aetest.subsection - def create_abstraction_lookup_objects(self, testbed, context): - '''create_abstraction_lookup_objects - - Subsection to create abstraction Lookup object and assigns it to - each corresponding device object as 'device.lib' attribute. - - In this example, we are using device object's attribute 'os', 'type' - (from testbed YAML file) and script input parameter 'context' as - tokens. - ''' - for device in testbed.devices.values(): - device.lib = Lookup(device.os, device.type, context) - - # ... other subsections - - # from here onwards, you can refer to libraries dynamically. - - class Configure_Ospf(aetest.Testcase): - - @aetest.setup - def setup(self, testbed): - # iterate through all devices and configure device... - for device in testbed.devices.values(): - device.lib.my_abstracted_library.configure_ospf(arg_1 = '...', - arg_2 = '...', - etc = '...') - - @aetest.test - def test(self, testbed): - for device in testbed.devices.values(): - output = device.lib.parser.ShowOspf(device = device) - - # validate values... etc - # ... - - diff --git a/docs/abstract/lookup_decorator.rst b/docs/abstract/lookup_decorator.rst deleted file mode 100644 index e9a70bd..0000000 --- a/docs/abstract/lookup_decorator.rst +++ /dev/null @@ -1,239 +0,0 @@ -.. _abstract_lookup_decorator: - -Lookup Decorator -================ - -``LookupDecorator`` is a feature extension to :ref:`abstract_lookup_cls`. -Whereas the ``Lookup`` class allows users to write **different** classes, -functions and variables in tokenized modules and dynamically reference them, the -lookup decorator operates at the class method level, allowing users to write -a **single class** with different method implementations per each token variance -combination. - -.. code-block:: text - - .--> TokenX.Y class method - / - UserScript -> import cls -> call cls method --+----> TokenX class method - \ - `--> Default (no token) - class method - -.. code-block:: python - - # Example - # ------- - # - # a simple lookup decorator example - - # my_library/config.py - # -------------------- - - # import the decorator - # (note the lowercase 'lookup') - from abstract import lookup - - # define a class using the decorator on its methods - class ConfigureRouting(object) - def __init__(self, os): - self.os = os - - # apply the decorator on methods to be abstracted - @lookup('os') - def apply_config(self): - # ... insert generic/non-os specific code here - - - # my_library/nxos/config.py - # ------------------------- - from ..config import ConfigureRouting as BaseConfigRouting - - # inherit the parent class - class ConfigureRouting(BaseConfigRouting): - - # define the same method specific to this token - def apply_config(self): - # ... insert nxos specific code here - -The main benefit of using ``LookupDecorator`` is that it allows the user to -perform standard python ``import`` and deal with only one class instance. -During runtime, the engine looks up the class's attributes and forms a list of -tokens based on these values, and replaces the decorated methods during with a -"more" appropriate one from a tokenized search -(see :ref:`abstract_search_algorithm`). - -.. code-block:: python - - # Example - # ------- - # - # using the above code - - # import the main entry class directly - from my_library.config import ConfigureRouting - - # use it as you would naturally - obj = ConfigureRouting(os = 'nxos') - - # when a decorated method is called, the lookup occurs and the - # most appriorate method from one of its subclasses is called instead. - result = obj.apply_config() - # lookup information - # ------------------ - # attributes to read: os - # attribute value: os = 'nxos' - # - # thus, the search result equivalence is: - # from my_library.nxos.config import ConfigureRouting - # result = ConfigureRouting.apply_config(obj) - - -Usages ------- - -To use ``LookupDecorator``, start with writing your abstraction-enabled library -as you normally would. When arriving at defining classes that requires methods -level abstraction, simply apply the decorator onto each method that needs to be -abstracted. Behaviors: - -- Lookup decorator can be imported as ``lookup`` (note the lowercase), or as - ``decorator.LookupDecorator``. They are exactly the same, but some may prefer - one name over the other. - - .. code-block:: python - - from abstract import lookup - from abstract.decorator import LookupDecorator - -- The usage of lookup decorator does not mandate a top-level - :ref:`abstraction_pkg` declaration. It only requires :ref:`abstraction_tokens` - definitions under the module where the lookup decorator is used. - - .. code-block:: text - - Example: - if LookupDecorator is used in on class X under module A.B, - tokens should be defined as child modules under A.B. - -- Lookup decorator takes in a list of **attributes names** as arguments. During - runtime, the engine will lookup the given class instance for these attributes - to be used as tokens. This mechanism is called an *attribute getter*. The - default attribute getter looks up both the class instance and - ``instance.device`` (if exists) for the named attribute. - - .. code-block:: python - - class MyClass(object): - - @lookup('attr_1', 'attr_2') - def some_func(self): - # ... - - # equivalent to - # obj = MyClass() - # token_1 = getattr(obj, 'attr_1', getattr(obj.device, 'attr_1')) - # token_2 = getattr(obj, 'attr_2', getattr(obj.device, 'attr_2')) - -- The search for matching token combinations always begins at this class's - module declaration level onwards. It will match for the same **relative path** - as the current module, and the same class name (or names in nested class defs) - and target method. - - .. code-block:: text - - Example: - a search originating from: moduleX.moduleY.classA.classB.some_func() - may match: moduleX.moduleY.tokenJ.tokenK.classA.classB.some_func() - -- the default *attribute getter* can be replaced by providing a new function - through ``attr_getter`` argument. The provided function must take in two - arguments: ``obj`` and ``attr`` for both the object under scrutiny and the - attribute to lookup - -.. code-block:: python - - # Examples - # -------- - # - # lookup decorator usage - - # assuming we had a lookup-decorator enabled library - # my_library.my_module.ConfigureOspf - - # import it regularly - from my_library.my_module import ConfigureOspf - - # instaciate it naturally - # (in this case our class requires argument 'os' and mgmt_context) - routing = ConfigureOspf(os = 'iosxr', mgmt_context = 'yang') - - # if we call a decorated method, say, apply_configuration - # eg, code snippet: - # @lookup('os', 'mgmt_context') - # def apply_configuration(self): - # # ... code - - routing.apply_configuration() - # the engine translates this to: - # token_os = routing.os = 'iosxr' - # token_mgmt_context = routing.mgmt_context = 'yang' - # and the resulting lookup equivalent could be: - # from my_library.my_module.iosxr.yang import ConfigureOspf - # result = ConfigureOspf.apply_configuration(routing) - - # note - # ---- - # after lookup is performed, notice that the found target class's method - # is called directly with the original class instance as first argument. - # This is a python property: class methods can be treated as "functions" - # if you pass in a "similar" class instance as the first argument. - # See: https://docs.python.org/3.4/tutorial/classes.html#method-objects - -.. csv-table:: LookupDecorator Class Argument List - :header: "Argument", "Description" - - ``*attrs``, "list of attributes to be used read as input tokens for lookup" - ``attr_getter``, "class instance attribute getter (optional)" - ``builder``, "token permutation builder (optional)" - ``**builder_kwargs``, "any keyword arguments/values to be passed to the - builder (optional)" - -Lookup From Device Decorator -============================ - -``LookupDecorator.from_device`` is a feature extension to ``LookupDecorator``. -The lookup.from_device decorator operates at the runtime, allowing users to -write a **single class** with different method implementations and dynamically -based on the token variance combination from device's custom abstraction or -pre-defined at class method level. - -.. code-block:: python - - # Example - # ------- - # - # a simple lookup.from_device decorator example - - # my_library/config.py - # -------------------- - - # import the decorator - # (note the lowercase 'lookup') - from abstract import lookup - - # define a class using the decorator on its methods - class ConfigureRouting(object) - def __init__(self, os): - self.os = os - - # apply the decorator on methods to be abstracted dynamically based on - # custom abstraction data - @lookup.from_device - def apply_config(self): - # ... insert generic/non-os specific code here - - # apply the decorator on methods to be abstracted dynamically based on - # custom abstraction data or fallback to token 'os' - @lookup.from_device('os') - def check_config(self): - # ... insert generic/non-os specific code here diff --git a/docs/abstract/revisions.rst b/docs/abstract/revisions.rst new file mode 100644 index 0000000..1057b96 --- /dev/null +++ b/docs/abstract/revisions.rst @@ -0,0 +1,125 @@ +Revisions +========= + +Purpose +^^^^^^^ + +As CLI parsers and APIs continue to evolve, updates may introduce breaking +changes that result in errors within previously functional jobs. To address this +issue, we have implemented a revision system that allows us to update parsers +and APIs, without changing the existing implementations. + +To use these revisions, we now generate a `.abstract` YAML file each +time a pyATS job is run that can be loaded back into a job to ensure parity on +rerun. This file contains a comprehensive collection of all parsers, APIs, ops, +and other elements used throughout the job, allowing for seamless integration +of new parser/api/ops developments without disrupting existing job workflows. + +Usage +^^^^^ + +When new jobs are initiated, the most recent revision of the relevant +parser/api/ops will be automatically identified and used. These functions are +then saved to a file, which can be conveniently loaded as needed for future use. + +Saving +^^^^^^ + +At the completion of each job, abstract revisions are automatically saved to the +`.abstract` file within the corresponding run info folder. This +file can be utilized during future runs to ensure that the same parser/api/ops +versions are utilized for consistency across all devices. + +Loading +^^^^^^^ + +There are two options for loading the `.abstract` file. Placing +the file in the same folder as the job file will automatically load it. +Alternatively, the `--abstract-revisions` argument can be used to indicate the +file path if it is stored elsewhere. + +.. code-block:: sh + + pyats run job --abstract-revisions + + +Legacy +^^^^^^ + +New jobs will automatically leverage the most recent parser/api/ops revision, +which is typically suitable for most use cases. However, in certain situations, +reverting back to the initial parser/api/ops implementation may be necessary. +This can be achieved by passing in the `--abstract-legacy` argument. + +.. code-block:: sh + + pyats run job --abstract-legacy + +The `.abstract` can also specify that a job should always use the +earliest revision of any feature as long as it contains the line: + +.. code-block:: text + + default_revision: earliest + +For large collections of jobs, the `pyats migrate abstract`_ command has an +added option to generate `.abstract` files with this line for every +discovered job + +.. _pyats migrate abstract: https://pubhub.devnetcloud.com/media/pyats/docs/cli/pyats_migrate.html + +Creating a Revision +^^^^^^^^^^^^^^^^^^^ + +To create a revision for a parser/api/ops, a new revision folder must be +established within the existing OS folder, following the naming convention +`rv`. + +.. code-block:: text + + genieparser/ + └── src/ + └── genie/ + └── libs/ + └── parser/ + └── / + ├── rv1/ + ├── rv2/ + └── rv3/ + +Within the revision folder, two steps must be taken: + +1) Create a new `__init__.py` file with the following contents: + ```python + from genie import abstract + abstract.declare_token(revision='') + ``` + The `` should match the number used in the folder name (for + instance, `rv1` would use `'1'` as the revision number). + +2) Create a new file with the same name as the file of the feature you are + revising. For example, if the `"show platform"` command is to be revised, + create a `show_platform.py` in the revision folder. + +3) Once the new file has been created, open it and create the revised version + of whatever feature you would like with **the exact function/class name**. + IE, if you were to create a revision for the `show platform` parser, you + would create a new class `ShowPlatform` and the accompanying schema. + +As an example, assume the first revision is being created for the IOSXE version +of the `"show platform"` command, the resulting file structure would resemble: + +.. code-block:: text + + genieparser/ + └── src/ + └── genie/ + └── libs/ + └── parser/ + └── iosxe/ + └── rv1/ + ├── __init__.py + └── show_platform.py + +Any changes made in `iosxe/rv1/show_platform.py` will then be used instead of +the original `iosxe/show_platform.py` file. \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 350d8d6..dd59bf8 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,6 +39,7 @@ 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.extlinks', + 'sphinx_tabs.tabs', #'sphinxcontrib_robotframework', ] diff --git a/docs/index.rst b/docs/index.rst index 616c1f7..0a367e4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,7 +47,7 @@ libraries and avoiding functional programming. 'slots': 'None'}, ... -.. note:: +.. note:: :ref:`More information on the testbed file` @@ -127,7 +127,7 @@ massive news for automation within and outside of Cisco! .. toctree:: :caption: Developer Docs :maxdepth: 1 - + abstract/index metaparser/index parsergen/index