1st part of ansible config, adds ansible-config to view/manage configs (ansible#12797)

* Start of ansible config project

moved configuration definitions to external yaml file vs hardcoded
 * updated constants to be a data strcutures that are looped over and also return origin of setting
changed to manager/data scheme for base classes
new cli ansible-config to view/manage ansible configuration settings
 * prints green for default/unchanged and yellow for those that have been overriden
 * added list action to show all configurable settings and their associated ini and env var names
 * allows specifying config file to see what result would look like
 * TBD update, edit and view options

removed test for functions that have been removed

env_Vars are now list of dicts
allows for version_added and deprecation in future
added a couple of descriptions for future doc autogeneration
ensure test does not fail if delete_me exists
normalized 'path expansion'
added yaml config to setup packaging
removed unused imports
better encoding handling

updated as per feedback

* pep8

Author: bcoca

bin/ansible

@@ -127,7 +127,7 @@ if __name__ == '__main__':
         exit_code = 99
     except Exception as e:
         have_cli_options = cli is not None and cli.options is not None
-        display.error("Unexpected Exception: %s" % to_text(e), wrap_text=False)
+        display.error("Unexpected Exception, this is probably a bug: %s" % to_text(e), wrap_text=False)
         if not have_cli_options or have_cli_options and cli.options.verbosity > 2:
             log_only = False
         else:

bin/ansible-config

@@ -0,0 +1 @@
+ansible

hacking/conf2yaml.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+import ast
+import yaml
+import os
+import sys
+from ansible.parsing.yaml.dumper import AnsibleDumper
+
+things = {}
+stuff = {}
+
+op_map = {
+    ast.Add: '+',
+    ast.Sub: '-',
+    ast.Mult: '*',
+    ast.Div: '/',
+}
+
+
+
+def get_values(values):
+    if not isinstance(values, list):
+        return get_value(values)
+    ret = []
+    for value in values:
+        ret.append(get_value(value))
+    return ret
+
+
+def get_value(value):
+    if hasattr(value, 'id'):
+        ret = value.id
+    elif hasattr(value, 's'):
+        ret =  value.s
+    elif hasattr(value, 'n'):
+        ret = value.n
+    elif hasattr(value, 'left'):
+        operator = op_map[type(value.op)]
+        left = get_values(value.left)
+        right = get_values(value.right)
+        return '%s %s %s' % (left, operator, right)
+    elif hasattr(value, 'value'):
+        ret = value.value
+    elif hasattr(value, 'elts'):
+        ret = get_values(value.elts)
+    elif isinstance(value, ast.Call):
+        func, args, kwargs = get_call(value)
+        args[:] = [repr(arg) for arg in args]
+        for k, v in kwargs.items():
+            args.append('%s=%s' % (k, repr(v)))
+        return '%s(%s)' % (func, ', '.join(args))
+    else:
+        return value
+
+    return get_value(ret)
+
+
+def get_call(value):
+    args = []
+    for arg in value.args:
+        v = get_value(arg)
+        try:
+            v = getattr(C, v, v)
+        except:
+            pass
+        args.append(v)
+    kwargs = {}
+    for keyword in value.keywords:
+        v = get_value(keyword.value)
+        try:
+            v = getattr(C, v, v)
+        except:
+            pass
+        kwargs[keyword.arg] = v
+
+    func = get_value(value.func)
+    try:
+        attr = '.%s' % value.func.attr
+    except:
+        attr = ''
+    return '%s%s' % (func, attr), args, kwargs
+
+
+with open(sys.argv[1]) as f:
+    tree = ast.parse(f.read())
+
+for item in tree.body:
+    if hasattr(item, 'value') and isinstance(item.value, ast.Call):
+        try:
+            if item.value.func.id != 'get_config':
+                continue
+        except AttributeError:
+            continue
+
+        _, args, kwargs = get_call(item.value)
+
+        name = get_value(item.targets[0])
+        section = args[1].lower()
+        config = args[2]
+
+        # new form
+        if name not in stuff:
+            stuff[name] = {}
+        stuff[name] = {
+            'desc': 'TODO: write it',
+            'ini': [{'section': section, 'key': config}],
+            'env': [args[3]],
+            'default': args[4] if len(args) == 5 else None,
+            'yaml': {'key': '%s.%s' % (section, config)},
+            'vars': []
+        }
+        stuff[name].update(kwargs)
+
+        ## ini like
+        #if section not in things:
+        #    things[section] = {}
+
+        #things[section][config] = {
+        #    'env_var': args[3],
+        #    'default': args[4] if len(args) == 5 else 'UNKNOWN'
+        #}
+        #things[section][config].update(kwargs)
+print(yaml.dump(stuff, Dumper=AnsibleDumper, indent=2, width=170))
+

lib/ansible/cli/__init__.py

@@ -269,10 +269,8 @@ def validate_conflicts(self, vault_opts=False, runas_opts=False, fork_opts=False
                     (op.su or op.su_user) and (op.become or op.become_user) or
                     (op.sudo or op.sudo_user) and (op.become or op.become_user)):
 
-                self.parser.error("Sudo arguments ('--sudo', '--sudo-user', and '--ask-sudo-pass') "
-                                  "and su arguments ('--su', '--su-user', and '--ask-su-pass') "
-                                  "and become arguments ('--become', '--become-user', and '--ask-become-pass')"
-                                  " are exclusive of each other")
+                self.parser.error("Sudo arguments ('--sudo', '--sudo-user', and '--ask-sudo-pass') and su arguments ('--su', '--su-user', and '--ask-su-pass') "
+                                  "and become arguments ('--become', '--become-user', and '--ask-become-pass') are exclusive of each other")
 
         if fork_opts:
             if op.forks < 1:
@@ -283,20 +281,13 @@ def expand_tilde(option, opt, value, parser):
         setattr(parser.values, option.dest, os.path.expanduser(value))
 
     @staticmethod
-    def unfrack_path(option, opt, value, parser):
-        setattr(parser.values, option.dest, unfrackpath(value))
+    def unfrack_paths(option, opt, value, parser):
+        if isinstance(value, string_types):
+            setattr(parser.values, option.dest, [unfrackpath(x) for x in value.split(os.sep)])
 
     @staticmethod
-    def expand_paths(option, opt, value, parser):
-        """optparse action callback to convert a PATH style string arg to a list of path strings.
-
-        For ex, cli arg of '-p /blip/foo:/foo/bar' would be split on the
-        default os.pathsep and the option value would be set to
-        the list ['/blip/foo', '/foo/bar']. Each path string in the list
-        will also have '~/' values expand via os.path.expanduser()."""
-        path_entries = value.split(os.pathsep)
-        expanded_path_entries = [os.path.expanduser(path_entry) for path_entry in path_entries]
-        setattr(parser.values, option.dest, expanded_path_entries)
+    def unfrack_path(option, opt, value, parser):
+        setattr(parser.values, option.dest, unfrackpath(value))
 
     @staticmethod
     def base_parser(usage="", output_opts=False, runas_opts=False, meta_opts=False, runtask_opts=False, vault_opts=False, module_opts=False,

lib/ansible/cli/config.py

@@ -0,0 +1,181 @@
+# (c) 2017, Ansible by Red Hat, Inc.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+# ansible-vault is a script that encrypts/decrypts YAML files. See
+# http://docs.ansible.com/playbooks_vault.html for more details.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import shlex
+import subprocess
+import sys
+import yaml
+
+from ansible.cli import CLI
+from ansible.config.data import Setting
+from ansible.config.manager import ConfigManager
+from ansible.errors import AnsibleError, AnsibleOptionsError
+from ansible.module_utils._text import to_native, to_text
+from ansible.parsing.yaml.dumper import AnsibleDumper
+from ansible.utils.color import stringc
+from ansible.utils.path import unfrackpath
+
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class ConfigCLI(CLI):
+    """ Config command line class """
+
+    VALID_ACTIONS = ("view", "edit", "update", "dump", "list")
+
+    def __init__(self, args, callback=None):
+
+        self.config_file = None
+        self.config = None
+        super(ConfigCLI, self).__init__(args, callback)
+
+    def parse(self):
+
+        self.parser = CLI.base_parser(
+            usage = "usage: %%prog [%s] [--help] [options] [ansible.cfg]" % "|".join(self.VALID_ACTIONS),
+            epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0])
+        )
+
+        self.parser.add_option('-c', '--config', dest='config_file', help="path to configuration file, defaults to first file found in precedence.")
+
+        self.set_action()
+
+        # options specific to self.actions
+        if self.action == "list":
+            self.parser.set_usage("usage: %prog list [options] ")
+        if self.action == "dump":
+            self.parser.set_usage("usage: %prog dump [options] [-c ansible.cfg]")
+        elif self.action == "view":
+            self.parser.set_usage("usage: %prog view [options] [-c ansible.cfg] ")
+        elif self.action == "edit":
+            self.parser.set_usage("usage: %prog edit [options] [-c ansible.cfg]")
+        elif self.action == "update":
+            self.parser.add_option('-s', '--setting', dest='setting', help="config setting, the section defaults to 'defaults'")
+            self.parser.set_usage("usage: %prog update [options] [-c ansible.cfg] -s '[section.]setting=value'")
+
+        self.options, self.args = self.parser.parse_args()
+        display.verbosity = self.options.verbosity
+
+    def run(self):
+
+        super(ConfigCLI, self).run()
+
+        if self.options.config_file:
+            self.config_file = unfrackpath(self.options.config_file, follow=False)
+            self.config = ConfigManager(self.config_file)
+        else:
+            self.config = ConfigManager()
+            self.config_file = self.config.data.get_setting('ANSIBLE_CONFIG')
+            try:
+                if not os.path.exists(self.config_file):
+                    raise AnsibleOptionsError("%s does not exist or is not accessible" % (self.config_file))
+                elif not os.path.isfile(self.config_file):
+                    raise AnsibleOptionsError("%s is not a valid file" % (self.config_file))
+
+                os.environ['ANSIBLE_CONFIG'] = self.config_file
+            except:
+                if self.action in ['view']:
+                    raise
+                elif self.action in ['edit', 'update']:
+                    display.warning("File does not exist, used empty file: %s" % self.config_file)
+
+        self.execute()
+
+    def execute_update(self):
+        '''
+        Updates a single setting in the specified ansible.cfg
+        '''
+        raise AnsibleError("Option not implemented yet")
+
+        if self.options.setting is None:
+            raise AnsibleOptionsError("update option requries a setting to update")
+
+        (entry, value) = self.options.setting.split('=')
+        if '.' in entry:
+            (section, option) = entry.split('.')
+        else:
+            section = 'defaults'
+            option = entry
+        subprocess.call([
+            'ansible',
+            '-m','ini_file',
+            'localhost',
+            '-c','local',
+            '-a','"dest=%s section=%s option=%s value=%s backup=yes"' % (self.config_file, section, option, value)
+        ])
+
+    def execute_view(self):
+        '''
+        Displays the current config file
+        '''
+        try:
+            with open(self.config_file, 'rb') as f:
+                self.pager(to_text(f.read(), errors='surrogate_or_strict'))
+        except Exception as e:
+            raise AnsibleError("Failed to open config file: %s" % to_native(e))
+
+    def execute_edit(self):
+        '''
+        Opens ansible.cfg in the default EDITOR
+        '''
+        raise AnsibleError("Option not implemented yet")
+        try:
+            editor = shlex.split(os.environ.get('EDITOR','vi'))
+            editor.append(self.config_file)
+            subprocess.call(editor)
+        except Exception as e:
+            raise AnsibleError("Failed to open editor: %s" % to_native(e))
+
+    def execute_list(self):
+        '''
+        list all current configs reading lib/constants.py and shows env and config file setting names
+        '''
+        self.pager(to_text(yaml.dump(self.config.initial_defs, Dumper=AnsibleDumper), errors='surrogate_or_strict'))
+
+    def execute_dump(self):
+        '''
+        Shows the current settings, merges ansible.cfg if specified
+        '''
+        text = []
+        defaults = self.config.initial_defs.copy()
+        for setting in self.config.data.get_settings():
+            if setting.name in defaults:
+                defaults[setting.name] = setting
+
+        for setting in sorted(defaults):
+            if isinstance(defaults[setting], Setting):
+                if defaults[setting].origin == 'default':
+                    color = 'green'
+                else:
+                    color = 'yellow'
+                msg = "%s(%s) = %s" % (setting, defaults[setting].origin, defaults[setting].value)
+            else:
+                color = 'green'
+                msg = "%s(%s) = %s" % (setting, 'default', defaults[setting].get('default'))
+            text.append(stringc(msg, color))
+
+        self.pager(to_text('\n'.join(text), errors='surrogate_or_strict'))

lib/ansible/cli/galaxy.py

@@ -117,11 +117,9 @@ def set_action(self):
         if self.action not in ("delete", "import", "init", "login", "setup"):
             # NOTE: while the option type=str, the default is a list, and the
             # callback will set the value to a list.
-            self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str,
-                                   default=C.DEFAULT_ROLES_PATH,
-                                   help='The path to the directory containing your roles. The default is the roles_path configured in your ansible.cfg '
-                                        'file (/etc/ansible/roles if not configured)')
-
+            self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.unfrack_paths, default=C.DEFAULT_ROLES_PATH,
+                                   help='The path to the directory containing your roles. The default is the roles_path configured in your ansible.cfg'
+                                        'file (/etc/ansible/roles if not configured)', type="string")
         if self.action in ("init", "install"):
             self.parser.add_option('-f', '--force', dest='force', action='store_true', default=False, help='Force overwriting an existing role')
 
@@ -308,16 +306,13 @@ def execute_install(self):
         uses the args list of roles to be installed, unless -f was specified. The list of roles
         can be a name (which will be downloaded via the galaxy API and github), or it can be a local .tar.gz file.
         """
-
         role_file = self.get_opt("role_file", None)
 
         if len(self.args) == 0 and role_file is None:
-            # the user needs to specify one of either --role-file
-            # or specify a single user/role name
+            # the user needs to specify one of either --role-file or specify a single user/role name
             raise AnsibleOptionsError("- you must specify a user/role name or a roles file")
         elif len(self.args) == 1 and role_file is not None:
-            # using a role file is mutually exclusive of specifying
-            # the role name on the command line
+            # using a role file is mutually exclusive of specifying the role name on the command line
             raise AnsibleOptionsError("- please specify a user/role name, or a roles file, but not both")
 
         no_deps = self.get_opt("no_deps", False)