Added modules to retrieve policy group and security policy objects

- plugins/modules/nsxt_policy_group_info.py
- plugins/modules/nsxt_policy_group_member_info.py
- plugins/modules/nsxt_policy_group_member_types.py
- plugins/modules/nsxt_policy_security_policy_facts.py
- plugins/modules/nsxt_policy_security_policy_rule_stats.py
- plugins/modules/nsxt_policy_security_policy_rules_facts.py

plugins/modules/common_utils.py modified.
Functions used for building query strings in URIs and for handling
retrieval of paged data were added.

The collection as it is now does not implement modules for the
retrieval of objects and focuses on CrUD ( no read ) functionality.
I want to work to flesh out the "R" functionality.

Signed-off-by: Ed McGuigan <[email protected]>

plugins/module_utils/common_utils.py

@@ -164,3 +164,82 @@ def get_upgrade_orchestrator_node(module, mgr_hostname, mgr_username, mgr_passwo
         module.fail_json(changed=True, msg='Error getting ip address of the upgrade'
                         ' orchestrator node. Error: {}'.format(err))
     return resp['service_properties']['enabled_on'];
+
+def build_url_query_string(parm_dict):
+    '''
+    This function just builds up a URL query string of the form:
+    
+    ?parm1=val1&bool1=True&bool2=False
+    
+    '''
+    qstring = ""
+    qlist = list()
+    for dkey in parm_dict.keys():
+        if parm_dict[dkey] is not None:
+            if type(parm_dict[dkey]) is bool:
+                qlist.append("{}={}".format(dkey,str(parm_dict[dkey])))
+            else:
+                qlist.append("{}={}".format(dkey,parm_dict[dkey]))
+    if qlist:
+        qstring = "?{}".format("&".join(qlist))
+    return qstring
+
+def build_url_query_dict(params,query_keys):
+    '''
+    The params dict in many of the modules contains a lot of keys. Some keys pertain
+    to the URL path, some to things like credentials, certificates etc.
+    We only want to process the ones relating to the query section of a URL. So the whole set of params is passed here
+    along with a filter defined as the set of keys pertaining to the query section.
+    The fields are filtered down
+    '''
+    query_params_dict = { k:v for (k,v) in params.items() if k in query_keys }
+    return query_params_dict
+
+def do_objects_get(module,manager_url,module_params,
+                        headers=dict(Accept='application/json'), validate_certs=True, ignore_errors=False):
+
+    mgr_username = module_params["username"]
+    mgr_password = module_params["password"]
+    nsx_cert_path = module_params["nsx_cert_path"]
+    nsx_key_path = module_params["nsx_key_path"]
+    # If a cursor was provided, or a page size then we are making a single call
+    # If we test for a key that doesn't exist and trap
+    mp_keys = module_params.keys()
+    if ('cursor' in mp_keys and module_params['cursor'] is not None ) or ('page_size' in mp_keys  and module_params['page_size'] is not None):
+        try:
+            (rc, resp) = request(manager_url, headers=dict(Accept='application/json'),
+                        url_username=mgr_username, url_password=mgr_password, validate_certs=validate_certs, ignore_errors=True)
+        except Exception as err:
+            module.fail_json(msg='Error retrieving groups. Error [%s]' % (to_native(err)))
+    else:
+        # No cursor parameter was provided so all data is being fetched
+        # This might still require multiple calls if there are more objects than are allowed to be returned in a single call
+        still_more_groups = True
+        cursor = None
+        all_group_data = dict()
+    #        all_group_data["results"] = list()
+        while still_more_groups:
+            if cursor:
+                # Add the cursor to the URL
+                url_with_cursor = "{}&cursor={}".format(manager_url,cursor)
+            else:
+                url_with_cursor = manager_url
+            try:
+                (rc, resp) = request(url_with_cursor, headers=dict(Accept='application/json'),
+                        url_username=mgr_username, url_password=mgr_password, validate_certs=validate_certs, ignore_errors=True)
+            except Exception as err:
+                module.fail_json(msg='Error retrieving groups. Error [%s]' % (to_native(err)))
+            if not "cursor" in resp:
+                still_more_groups = False
+            else:
+                cursor = resp["cursor"]
+            # Add new results to existing results
+            # If this is the first add, all the other data besides the "results" needs to be added
+            if not "results" in all_group_data:
+                all_group_data = resp
+            else:
+                # JUst add the additionally fetched results
+                all_group_data["results"] += resp["results"]
+        resp = all_group_data
+    return resp
+

plugins/modules/nsxt_policy_group_info.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+
+DOCUMENTATION = '''
+---
+module: nsxt_policy_group_facts
+short_description: List Policy Groups
+description: Returns paginated list of policy groups
+             Policy Groups are used in Distributed Firewall Policies in three places:
+             - Source of a network traffic flow - group is resolved to a list of IP addresses
+             - Destination of a network traffic flow - group is resolved to a list of IP addresses
+             - Applied To which determines the VM on which a policy will be applied so the group is resolved
+               to a list of VMs
+               
+            Note that there is the potential for a very large number of groups to be returned.
+
+version_added: "X.Y"
+author: Ed McGuigan <[email protected]>
+options:
+    hostname:
+        description: Deployed NSX manager hostname.
+        required: true
+        type: str
+    username:
+        description: The username to authenticate with the NSX manager.
+        required: true
+        type: str
+    password:
+        description: The password to authenticate with the NSX manager.
+        required: true
+        type: str
+    ca_path:
+        description: Path to the CA bundle to be used to verify host's SSL
+                     certificate
+        type: str
+    nsx_cert_path:
+        description: Path to the certificate created for the Principal
+                     Identity using which the CRUD operations should be
+                     performed
+        type: str
+    nsx_key_path:
+        description:
+            - Path to the certificate key created for the Principal Identity
+              using which the CRUD operations should be performed
+            - Must be specified if nsx_cert_path is specified
+        type: str        
+        
+    global_infra:
+        description: Flag set to True when targeting a Global NSX Manager (Federation)
+        required: false
+        type: bool
+        
+    domain_id:
+        description: The domain string value to be used in the query, usually "default"
+        required: false
+        type: string
+        default: default
+        
+    page_size:
+        description: if there is a desire to fetch the data in chunks rather than all at
+                     once, an integer specifying the maximum number of objects to fetch
+        required: false
+        type: integer
+        
+    cursor:
+        description: when a page_size is specified, the returned data includes a "cursor" that
+                     must be provided in a subsequent call in order to carry on where the prior call
+                     left off. User would need to capture the cursor value from one call and provide it
+                     in the next call
+        required: false
+        type: string
+        
+    sort_ascending:
+        description: Used to reverse sort order by setting it to False
+        required: false
+        type: bool
+        default: True
+        
+    sort_by:
+        description: Field to sort on
+        required: false
+        type: string
+        default: 
+        
+    include_mark_for_delete_objects:
+        description: Show groups marked for deletion
+        required: false
+        type: bool
+        default: False
+'''
+
+EXAMPLES = '''
+- name: List Policy Groups
+  nsxt_policy_group_facts:
+    hostname: "10.192.167.137"
+    username: "admin"
+    password: "Admin!23Admin"
+    validate_certs: False
+'''
+
+RETURN = '''# '''
+import json
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.vmware.ansible_for_nsxt.plugins.module_utils.vmware_nsxt import vmware_argument_spec, request
+from ansible_collections.vmware.ansible_for_nsxt.plugins.module_utils.policy_communicator import PolicyCommunicator
+from ansible_collections.vmware.ansible_for_nsxt.plugins.module_utils.common_utils import build_url_query_dict, build_url_query_string, do_objects_get
+from ansible.module_utils._text import to_native
+
+def main():
+    # Fetch the specification of the absolute basic arguments needed to connect to the NSX Manager
+    argument_spec = PolicyCommunicator.get_vmware_argument_spec()
+    # The URL will need to be specified as being non-global or global and we will need a domain
+    URL_path_spec = dict(
+        global_infra=dict(type='bool', required=False, default=False),
+        domain_id=dict(type='str', required=False, default='default')
+        )
+    '''
+    Now add the arguments relating to query field in the URL for this GET method
+    All the options from the API are offered, including paging. Not sure when a user
+    might want to use paging but the option is provided.
+    If no paging specification is provided, I need to make sure that 
+    all data is retrieved, looking for a returned cursor in the response
+    indicating that there is more data to fetch.
+    
+    NOTE: I suspect that the member_types filter parameter is not actually valid
+    for a Policy Group where membership is described by a series of expressions
+    and this may be an error when converting from MP ( Management Plane ) Groups
+    to Policy Groups
+    '''
+    URL_query_spec = dict(
+                            include_mark_for_delete_objects=dict(type='bool', required=False),
+                            included_fields=dict(type='str', required=False),
+                            member_types=dict(type='str', required=False),
+                            sort_ascending=dict(type='bool', required=False, default=True),
+                            sort_by=dict(type='str', required=False),
+                            page_size=dict(type='int'   , required=False ),
+                            cursor=dict(type='str', required=False )
+                            )
+    # Combine the base URL, URL path spec and URL query argument specs
+    URL_path_spec.update(URL_query_spec)
+    argument_spec.update(URL_path_spec)
+    # Some code to validate the arguments provided with the invocation of the module
+    # in a playbook versus the defined argument spec.
+    module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True)
+
+    mgr_hostname = module.params['hostname']
+    validate_certs = module.params['validate_certs']
+    domain_id = module.params['domain_id']
+    if module.params['global_infra']:
+        infra_string = 'global-infra'
+    else:
+        infra_string = 'infra'
+
+    # Need to build up a query string
+    url_query_string = build_url_query_string( build_url_query_dict(module.params, URL_query_spec.keys() ) )
+    manager_url = 'https://{}/policy/api/v1/{}/domains/{}/groups{}'.format(mgr_hostname,infra_string,domain_id,url_query_string)
+
+    changed = False
+    '''
+    We potentially need to loop to fetch all data the code here will be the same for
+    any object we are doing a GET on, not just Policy Groups, so I have put it into a function and put the function
+    in the common_utils package.
+    '''
+    resp = do_objects_get(module,manager_url,module.params,
+                        headers=dict(Accept='application/json'),validate_certs=validate_certs, ignore_errors=True)     
+
+    module.exit_json(changed=changed, **resp)
+if __name__ == '__main__':
+    main()