parse functions, canonical spelling, sort files, hashrulers

* Implement canonical command case
* Canonicalize capitalization of keys in cmdspec
* Add README documentation regarding fences and enable/disable
* Statement parsers are now generic functions. Old standard parser remains
  for most statements, but some statements now have custom parsers.
* Implement deeper parse logic for ``install()`` and ``file()`` commands,
  improving the formatting of these statements.
* Implement input/output encoding configuration parameters
* Implement hashruler markup logic and preserve hashrulers if markup is
  disable or if configured to do so.
* Separate cmake-annotate frontend
* Provider a ``Loader=`` to yaml ``load()``
* Fix python3 lint
* Fix bad lexing of make-style variables
* Fix multiple hash chars ``lstrip()ed`` from comments

* Closes `#62`_: Possible improvement on formatting "file"
* Closes `#75`_: configurable positioning of flags
* Closes `#87`_: Hash-rulers are stripped when markup disabled
* Closes `#91`_: Add missing keyword arguments to project command
* Closes `#95`_: added argument --encoding to allow for non-utf8
* Closes `#98`_: Fix kwargs/flag index for non-lowercase functions
* Closes `#100`_: Extra linebreak inserted when '$(' encountered
* Closes `#101`_: Provide a Loader to yaml.load
* Closes `#102`_: fences does not work as expected

.. _#62: https://github.com/cheshirekow/cmake_format/issues/62
.. _#75: https://github.com/cheshirekow/cmake_format/issues/75
.. _#87: https://github.com/cheshirekow/cmake_format/issues/87
.. _#91: #91
.. _#95: #95
.. _#98: #98
.. _#100: https://github.com/cheshirekow/cmake_format/issues/100
.. _#101: #101
.. _#102: https://github.com/cheshirekow/cmake_format/issues/102

.gitignore

@@ -1,4 +1,3 @@
-
 *.pyc
 *.o
 *.so
@@ -21,3 +20,12 @@ dist/
 
 # jupyter autosaves
 .ipynb_checkpoints
+
+# bazel poop
+bazel-*
+
+# vscode poop
+.vscode/ipch
+
+# node package manager
+node_modules

CMakeLists.txt

@@ -34,6 +34,7 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
 
 add_custom_target(format)
 add_custom_target(lint)
+
 add_custom_target(doc)
 
 set(ENV{PYTHONPATH} ${CMAKE_SOURCE_DIR})
@@ -72,4 +73,4 @@ if(EXISTS ${CMAKE_SOURCE_DIR}/doxy.config.in)
 
   add_custom_target(doxygen DEPENDS ${PROJECT_BINARY_DIR}/doxy.stamp)
   add_dependencies(doc doxygen)
-endif()
+endif()

cmake/codestyle.cmake

@@ -65,12 +65,15 @@ function(format_and_lint module)
   set(fmtcmds_)
   set(depfiles_)
   if(cmake_files_)
-    list(APPEND fmtcmds_ COMMAND python -Bm cmake_format -i ${cmake_files_})
+    list(APPEND fmtcmds_ COMMAND env PYTHONPATH=${CMAKE_SOURCE_DIR}
+                                 python -Bm cmake_format -i ${cmake_files_})
     list(APPEND depfiles_ ${cmake_files_}
                           ${CMAKE_SOURCE_DIR}/.cmake-format.py)
   endif()
   if(cc_files_)
-    list(APPEND fmtcmds_ COMMAND clang-format -style file -i ${cc_files_})
+    list(APPEND fmtcmds_ COMMAND clang-format-6.0 -style file -i ${cc_files_})
+    list(APPEND lntcmds_
+         COMMAND clang-tidy-6.0 -p ${CMAKE_BINARY_DIR} ${cc_files_})
     list(APPEND lntcmds_ COMMAND cpplint ${cc_files_})
     list(APPEND depfiles_ ${cc_files_}
                           ${CMAKE_SOURCE_DIR}/.clang-format
@@ -79,7 +82,10 @@ function(format_and_lint module)
   if(py_files_)
     list(APPEND fmtcmds_ COMMAND autopep8 -i ${py_files_})
     list(APPEND lntcmds_ COMMAND pylint ${py_files_})
-    list(APPEND lntcmds_ COMMAND flake8 ${py_files_})
+    # NOTE(josh): flake8 tries to use semaphores which fail in our containers
+    # https://bugs.python.org/issue3770 (probably due to /proc/shmem or
+    # something not being mounted)
+    list(APPEND lntcmds_ COMMAND flake8 --jobs 1 ${py_files_})
     list(APPEND depfiles_ ${py_files_}
                           ${CMAKE_SOURCE_DIR}/.flake8
                           ${CMAKE_SOURCE_DIR}/.pep8

cmake/doctools.cmake

@@ -14,12 +14,12 @@
 function(sphinx module)
   set(stamp_path_ ${CMAKE_CURRENT_BINARY_DIR}/${module}_doc.stamp)
   add_custom_command(OUTPUT ${stamp_path_}
-                     COMMAND sphinx-build -M html ${CMAKE_CURRENT_SOURCE_DIR}
+                     COMMAND env PYTHONPATH=${CMAKE_SOURCE_DIR}
+                             sphinx-build -M html ${CMAKE_CURRENT_SOURCE_DIR}
                              ${CMAKE_CURRENT_BINARY_DIR}
                      COMMAND touch ${stamp_path_}
                      DEPENDS ${ARGN}
                      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
   add_custom_target(${module}_doc DEPENDS ${stamp_path_})
   add_dependencies(doc ${module}_doc)
 endfunction()
-

cmake/pkgconfig.cmake

@@ -176,8 +176,14 @@ function(target_pkg_depends target pkg0)
     endif()
     if(pkg_${pkgname}_libdirs)
       get_target_property(lflags_ ${target} LINK_FLAGS)
-      list(APPEND lflags_ ${pkg_${pkgname}_lflags})
-      set_target_properties(${target} PROPERTIES LINK_FLAGS ${lflags_})
+      if(lflags_)
+        list(APPEND lflags_ ${pkg_${pkgname}_lflags})
+        set_target_properties(${target} PROPERTIES LINK_FLAGS ${lflags_})
+      else()
+        set_target_properties(${target} PROPERTIES LINK_FLAGS
+                              ${pkg_${pkgname}_lflags})
+      endif()
+
     endif()
     if(pkg_${pkgname}_libs)
       # Passthrough options like INTERFACE|PUBLIC|PRIVATE and
@@ -186,4 +192,3 @@ function(target_pkg_depends target pkg0)
     endif()
   endforeach()
 endfunction()
-

cmake_format/CMakeLists.txt

@@ -56,4 +56,3 @@ if(NOT IS_TRAVIS_CI)
 endif()
 
 add_subdirectory(doc)
-add_subdirectory(test)

cmake_format/__init__.py

@@ -3,4 +3,4 @@
 """
 from __future__ import unicode_literals
 
-VERSION = '0.4.5'
+VERSION = '0.5.0'

cmake_format/__main__.py

@@ -26,12 +26,13 @@
 import textwrap
 
 import cmake_format
+from cmake_format import commands
 from cmake_format import configuration
 from cmake_format import formatter
 from cmake_format import lexer
 from cmake_format import markup
 from cmake_format import parser
-from cmake_format import render
+from cmake_format import parse_funs
 
 
 def detect_line_endings(infile_content):
@@ -107,25 +108,18 @@ def process_file(config, infile, outfile, dump=None):
       outfile.write("{}\n".format(token))
     return
   config.first_token = lexer.get_first_non_whitespace_token(tokens)
-  parse_tree = parser.parse(tokens, config.fn_spec)
+  parse_db = parse_funs.get_parse_db()
+  parse_db.update(parse_funs.get_legacy_parse(config.fn_spec).kwargs)
+  parse_tree = parser.parse(tokens, parse_db)
   if dump == "parse":
     parser.dump_tree([parse_tree], outfile)
     return
   if dump == "markup":
     dump_markup([parse_tree], config, outfile)
     return
-  if dump == "html-page":
-    html_content = render.get_html(parse_tree, fullpage=True)
-    outfile.write(html_content)
-    return
-  if dump == "html-stub":
-    html_content = render.get_html(parse_tree, fullpage=False)
-    outfile.write(html_content)
-    return
 
   box_tree = formatter.layout_tree(parse_tree, config)
   if dump == "layout":
-    infile.seek(0)
     formatter.dump_tree([box_tree], outfile)
     return
 
@@ -164,6 +158,18 @@ def find_config_file(infile_path):
   return None
 
 
+def load_yaml(config_file):
+  """
+  Attempt to load yaml configuration from an opened file
+  """
+  import yaml
+  try:
+    from yaml import CLoader as Loader
+  except ImportError:
+    from yaml import Loader
+  return yaml.load(config_file, Loader=Loader)
+
+
 def try_get_configdict(configfile_path):
   """
   Try to read the configuration as yaml first, then json, then python.
@@ -178,9 +184,8 @@ def try_get_configdict(configfile_path):
     pass
 
   try:
-    import yaml
     with io.open(configfile_path, 'r', encoding='utf-8') as config_file:
-      return yaml.load(config_file)
+      return load_yaml(config_file)
   except:  # pylint: disable=bare-except
     pass
 
@@ -212,8 +217,7 @@ def get_config(infile_path, configfile_path):
       if configfile_path.endswith('.json'):
         config_dict = json.load(config_file)
       elif configfile_path.endswith('.yaml'):
-        import yaml
-        config_dict = yaml.load(config_file)
+        config_dict = load_yaml(config_file)
       elif configfile_path.endswith('.py'):
         config_dict = {}
         with io.open(configfile_path, 'r', encoding='utf-8') as infile:
@@ -238,12 +242,16 @@ def dump_config(args, config_dict, outfile):
       config_dict[key] = value
 
   cfg = configuration.Configuration(**config_dict)
+  # Don't dump default per-command configs
+  for key in commands.get_default_config():
+    cfg.per_command.pop(key, None)
+
   if outfmt == 'yaml':
     import yaml
     yaml.dump(cfg.as_dict(), sys.stdout, indent=2,
               default_flow_style=False)
     return
-  elif outfmt == 'json':
+  if outfmt == 'json':
     json.dump(cfg.as_dict(), sys.stdout, indent=2)
     sys.stdout.write('\n')
     return
@@ -269,36 +277,10 @@ def dump_config(args, config_dict, outfile):
 """
 
 
-def setup_argparser(arg_parser):
+def add_config_options(optgroup):
   """
-  Add argparse options to the parser.
+  Add configuration options as flags to the argument parser
   """
-  arg_parser.add_argument('-v', '--version', action='version',
-                          version=cmake_format.VERSION)
-
-  mutex = arg_parser.add_mutually_exclusive_group()
-  mutex.add_argument('--dump-config', choices=['yaml', 'json', 'python'],
-                     default=None, const='python', nargs='?',
-                     help='If specified, print the default configuration to '
-                          'stdout and exit')
-  mutex.add_argument('--dump', choices=['lex', 'parse', 'layout', 'markup',
-                                        'html-page', 'html-stub'],
-                     default=None)
-
-  mutex = arg_parser.add_mutually_exclusive_group()
-  mutex.add_argument('-i', '--in-place', action='store_true')
-  mutex.add_argument('-o', '--outfile-path', default=None,
-                     help='Where to write the formatted file. '
-                          'Default is stdout.')
-
-  arg_parser.add_argument('-c', '--config-file',
-                          help='path to configuration file')
-  arg_parser.add_argument('infilepaths', nargs='*')
-
-  optgroup = arg_parser.add_argument_group(
-      title='Formatter Configuration',
-      description='Override configfile options')
-
   default_config = configuration.Configuration().as_dict()
   if sys.version_info[0] >= 3:
     value_types = (str, int, float)
@@ -330,8 +312,43 @@ def setup_argparser(arg_parser):
     # no arguments then the value will be an empty list. This exactly what we
     # want since we can ignore `None` values.
     elif isinstance(value, (list, tuple)):
+      typearg = None
+      if value:
+        typearg = type(value[0])
       optgroup.add_argument('--' + key.replace('_', '-'), nargs='*',
-                            help=helptext)
+                            type=typearg, help=helptext)
+
+
+def setup_argparser(arg_parser):
+  """
+  Add argparse options to the parser.
+  """
+  arg_parser.add_argument('-v', '--version', action='version',
+                          version=cmake_format.VERSION)
+
+  mutex = arg_parser.add_mutually_exclusive_group()
+  mutex.add_argument('--dump-config', choices=['yaml', 'json', 'python'],
+                     default=None, const='python', nargs='?',
+                     help='If specified, print the default configuration to '
+                          'stdout and exit')
+  mutex.add_argument('--dump', choices=['lex', 'parse', 'layout', 'markup'],
+                     default=None)
+
+  mutex = arg_parser.add_mutually_exclusive_group()
+  mutex.add_argument('-i', '--in-place', action='store_true')
+  mutex.add_argument('-o', '--outfile-path', default=None,
+                     help='Where to write the formatted file. '
+                          'Default is stdout.')
+
+  arg_parser.add_argument('-c', '--config-file',
+                          help='path to configuration file')
+  arg_parser.add_argument('infilepaths', nargs='*')
+
+  optgroup = arg_parser.add_argument_group(
+      title='Formatter Configuration',
+      description='Override configfile options')
+
+  add_config_options(optgroup)
 
 
 def main():
@@ -394,7 +411,8 @@ def main():
     if args.in_place:
       ofd, tempfile_path = tempfile.mkstemp(suffix='.txt', prefix='CMakeLists-')
       os.close(ofd)
-      outfile = io.open(tempfile_path, 'w', encoding='utf-8', newline='')
+      outfile = io.open(tempfile_path, 'w', encoding=cfg.output_encoding,
+                        newline='')
     else:
       if args.outfile_path == '-':
         # NOTE(josh): The behavior or sys.stdout is different in python2 and
@@ -404,17 +422,17 @@ def main():
         # it with byte arrays (assuming it was opened with 'wb'). So we use
         # io.open instead of open in this case
         outfile = io.open(os.dup(sys.stdout.fileno()),
-                          mode='w', encoding='utf-8', newline='')
+                          mode='w', encoding=cfg.output_encoding, newline='')
       else:
-        outfile = io.open(args.outfile_path, 'w', encoding='utf-8',
+        outfile = io.open(args.outfile_path, 'w', encoding=cfg.output_encoding,
                           newline='')
 
     parse_ok = True
     if infile_path == '-':
       infile = io.open(os.dup(sys.stdin.fileno()),
-                       mode='r', encoding='utf-8', newline='')
+                       mode='r', encoding=cfg.input_encoding, newline='')
     else:
-      infile = io.open(infile_path, 'r', encoding='utf-8')
+      infile = io.open(infile_path, 'r', encoding=cfg.input_encoding)
 
     try:
       with infile: