Rebase of chromiumos fork

https://chromium.googlesource.com/chromiumos/third_party/shellcheck/

ShellCheck.cabal

@@ -97,6 +97,7 @@ library
       ShellCheck.Regex
     other-modules:
       Paths_ShellCheck
+      ShellCheck.PortageAutoInternalVariables
     default-language: Haskell98
 
 executable shellcheck

portage/get_vars.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2019 The ChromiumOS Authors
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google LLC nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# Binary License Terms
+
+"""Extract eclass variable names into Haskell list format."""
+from __future__ import print_function
+import datetime
+import os
+import re
+import sys
+import textwrap
+# Matches a line that declares a variable in an eclass.
+VAR_RE = re.compile(r'@(?:ECLASS-)?VARIABLE:\s*(\w+)$')
+# Matches a line that declares inheritance.
+INHERIT_RE = re.compile(r'^[^#]*\binherit((?:\s+[\w-]+)+)$')
+VAR_FILE_HEADER = """module ShellCheck.PortageAutoInternalVariables (
+  portageAutoInternalVariables
+  ) where
+-- This file contains the variables generated by
+-- portage/get_vars.py"""
+PORTAGE_AUTO_VAR_NAME = 'portageAutoInternalVariables'
+class Eclass:
+  """Container for eclass information"""
+  def __init__(self, name, eclass_vars, inheritances):
+    self.name = name
+    self.vars = eclass_vars
+    self.inheritances = inheritances
+  def calculate_eclass_vars(self, eclasses):
+    while self.inheritances:
+      name = self.inheritances.pop()
+      try:
+        sub_eclass = eclasses[name]
+        new_vars = sub_eclass.calculate_eclass_vars(eclasses).vars
+        self.vars = self.vars.union(new_vars)
+      except Exception:
+        pass
+    return self
+def print_var_list(eclass, eclass_vars):
+  var_list = ' '.join(['"%s",' % v for v in sorted(eclass_vars)])
+  print('    -- %s\n%s' %
+        (eclass,
+         textwrap.fill(
+             var_list, 80, initial_indent='    ', subsequent_indent='    ')))
+def process_file(eclass_path):
+  eclass_name = os.path.splitext(os.path.basename(eclass_path))[0]
+  with open(eclass_path, 'r') as f:
+    eclass_vars = set()
+    eclass_inheritances = set()
+    for line in f:
+      line = line.strip()
+      if not line:
+        continue
+      while line[-1] == '\\':
+        line = line[:-1] + next(f).strip()
+      match = VAR_RE.search(line)
+      if match:
+        var_name = match.group(1)
+        eclass_vars.add(var_name.strip())
+      else:
+        match = INHERIT_RE.search(line)
+        if match:
+          for inheritance in re.split(r'\s+', match.group(1)):
+            if inheritance.strip():
+              eclass_inheritances.add(inheritance.strip())
+    return Eclass(eclass_name, eclass_vars, eclass_inheritances)
+def format_eclasses_as_haskell_map(eclasses):
+  map_entries = []
+  join_string = '", "'
+  for value in sorted(eclasses, key=(lambda x: x.name)):
+    if value.vars:
+      var_list_string = f'"{join_string.join(sorted(list(value.vars)))}"'
+      map_entries.append(
+          textwrap.fill(
+              f'("{value.name}", [{var_list_string}])',
+              80,
+              initial_indent='    ',
+              subsequent_indent='    '))
+  return_string = ',\n\n'.join(map_entries)
+  return_string = f"""    Data.Map.fromList
+    [
+{return_string}
+    ]"""
+  return f"""{VAR_FILE_HEADER}\n\n
+-- Last Generated: {datetime.datetime.now().strftime("%x")}
+import qualified Data.Map
+{PORTAGE_AUTO_VAR_NAME} =
+{return_string}"""
+def main(argv):
+  eclasses = {}
+  for path in sorted(argv, key=os.path.basename):
+    if not path.endswith('.eclass'):
+      continue
+    new_eclass = process_file(path)
+    eclasses[new_eclass.name] = new_eclass
+  eclasses_list = [
+      value.calculate_eclass_vars(eclasses) for key, value in eclasses.items()
+  ]
+  print(format_eclasses_as_haskell_map(eclasses_list))
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

portage/get_vars_diff.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# Copyright 2020 The ChromiumOS Authors
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google LLC nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# Binary License Terms
+
+"""Generates diff of vars from get_vars.py and those existing in Data.hs."""
+import itertools
+from pathlib import Path
+import subprocess
+SCRIPT = Path(__file__).resolve()
+THIRD_PARTY = SCRIPT.parent.parent.parent.parent.parent
+# List of relative directories in which to find the eclasses.
+eclass_rel_dirs = (
+    THIRD_PARTY / 'chromiumos-overlay' / 'eclass',
+    THIRD_PARTY / 'portage-stable' / 'eclass',
+    THIRD_PARTY / 'eclass-overlay' / 'eclass',
+)
+# Runs get_vars.py with the eclass paths and store the output.
+cmd = [SCRIPT.with_name('get_vars.py')] + list(
+    itertools.chain(*(x.glob('*') for x in eclass_rel_dirs)))
+new_output = subprocess.check_output(cmd, encoding='utf-8').splitlines()
+new = []
+for line in new_output:
+  if '--' in line:
+    new.append(line.strip())
+  elif not line.strip():
+    continue
+  else:
+    new += (line.replace('"', '').replace('\n', '').split(','))
+# Reads the Data.hs relevant area and store the lines.
+data_hs = THIRD_PARTY / 'shellcheck' / 'src' / 'ShellCheck' / 'Data.hs'
+with data_hs.open('r', encoding='utf-8') as fp:
+  record = False
+  old = []
+  for line in fp:
+    if line.strip() == '-- autotest.eclass declared incorrectly':
+      break
+    if line.strip() == '-- generic ebuilds':
+      record = True
+    if record:
+      if '--' in line:
+        old.append(line.strip())
+      elif not line.strip():
+        continue
+      else:
+        old += line.replace('"', '').replace('\n', '').split(',')
+# Cleans up empty bits as a result of parsing difficulties.
+new = [x.strip() for x in new if x.strip()]
+old = [x.strip() for x in old if x.strip()]
+all_eclasses = set()
+old_vars = {}
+new_vars = {}
+current_eclass = ''
+for item in old:
+  if '--' in item:
+    # It's an eclass comment line.
+    current_eclass = item[3:]
+    all_eclasses.add(current_eclass)
+    continue
+  else:
+    # It's a var, so add it to the dict of the current eclass.
+    old_vars.setdefault(current_eclass, []).append(item)
+for item in new:
+  if '--' in item:
+    # It's an eclass comment line.
+    current_eclass = item[3:]
+    all_eclasses.add(current_eclass)
+    continue
+  else:
+    # It's a var, so add it to the dict of the current eclass.
+    new_vars.setdefault(current_eclass, []).append(item)
+for eclass in sorted(all_eclasses):
+  if eclass in old_vars:
+    if eclass not in new_vars:
+      # Checks if the entire eclass is removed.
+      print(f'{eclass} not present in new variables.')
+      for var in old_vars[eclass]:
+        print(f'\t-{var}')
+      print()
+    else:
+      # Eclass isn't removed, so check for added or removed vars.
+      toprint = '\n'.join(
+          [f'\t-{x}' for x in old_vars[eclass] if x not in new_vars[eclass]] +
+          [f'\t+{x}' for x in new_vars[eclass] if x not in old_vars[eclass]])
+      if toprint:
+        print(eclass)
+        print(toprint)
+  if eclass in new_vars:
+    if eclass not in old_vars:
+      # Checks if entire eclass is new.
+      print(f'{eclass} added in new variables.')
+      for var in new_vars[eclass]:
+        print(f'\t+{var}')
+      print()

shellcheck.1.md

@@ -87,8 +87,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
 
 :   Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*.
     The default is to deduce the shell from the file's `shell` directive,
-    shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to
-    POSIX `sh` (not the system's), and will warn of portability issues.
+    shebang, or `.bash/.bats/.dash/.ksh/.ebuild/.eclass` extension, in that
+    order. *sh* refers to POSIX `sh` (not the system's), and will warn of
+    portability issues.
 
 **-S**\ *SEVERITY*,\ **--severity=***severity*
 

src/ShellCheck/ASTLib.hs

@@ -36,8 +36,6 @@ import Numeric (showHex)
 
 import Test.QuickCheck
 
-arguments (T_SimpleCommand _ _ (cmd:args)) = args
-
 -- Is this a type of loop?
 isLoop t = case t of
         T_WhileExpression {} -> True
@@ -559,11 +557,29 @@ getCommandNameFromExpansion t =
     extract (T_Pipeline _ _ [cmd]) = getCommandName cmd
     extract _ = Nothing
 
+-- If a command substitution is a single command, get its argument Tokens.
+-- Return an empty list if there are no arguments or the token is not a command substitution.
+-- $(date +%s) = ["+%s"]
+getArgumentsFromExpansion :: Token -> [Token]
+getArgumentsFromExpansion t =
+    case t of
+        T_DollarExpansion _ [c] -> extract c
+        T_Backticked _ [c] -> extract c
+        T_DollarBraceCommandExpansion _ [c] -> extract c
+        _ -> []
+  where
+    extract (T_Pipeline _ _ [cmd]) = arguments cmd
+    extract _ = []
+
 -- Get the basename of a token representing a command
 getCommandBasename = fmap basename . getCommandName
 
 basename = reverse . takeWhile (/= '/') . reverse
 
+-- Get the arguments to a command
+arguments (T_SimpleCommand _ _ (cmd:args)) = args
+arguments t = maybe [] arguments (getCommand t)
+
 isAssignment t =
     case t of
         T_Redirecting _ _ w -> isAssignment w