Merge pull request #997 from dkrupp/cross-fix

Improve compiler parameter auto-detection

docs/usage.md

@@ -51,15 +51,55 @@ Once the build is logged successfully (and the `compilation.json`) was created,
 
 Hint:
  You can do the 1st and the 2nd step in one round by executing `check`
- ```
+```
  cd tmux
  make clean
- CodeChecker check -b "make"
+ CodeChecker check -b "make" -o ./reports
+``` 
 or to run on 22 threads
- CodeChecker check -j22 -b "make clean;make -j22"
+
+```
+ CodeChecker check -j22 -b "make clean;make -j22" -o ./reports
 ```
 
-[What to do if the analysis fails (analysis settings for cross-compilation)](/docs/cross-compilation.md)
+
+### Cross-Compilation
+Cross-compilers are auto-detected by CodeChecker, so 
+the `--target` and the compiler pre-configured
+include paths of `gcc/g++` are automatically passed to `clang` when analyzing.
+
+**Make sure that the compilers used for building the project (e.g. `/usr/bin/gcc`) are
+accessible when `CodeChecker analyze` or `check` is invoked.**
+
+### Incremental Analysis
+ The analysis can be run for only the changed files and the `report-directory` will be
+ correctly updated with the new results.
+
+ ```
+ cd tmux
+ make clean
+ CodeChecker check -b "make" -o reports
+ 
+ #Change only 1 file in tmux
+ vi ./cmd-find.c
+ 
+ #Only cmd-find.c will be re-analyzed 
+ CodeChecker check -b "make" -o reports
+```
+Now the `reports` directory contains also the results of the updated `cmd-find.c`.
+
+### Analysis Failures
+
+The `reports/failed` folder contains all build-actions that
+were failed to analyze. For these there will be no results.
+
+Possible reasons for failed analysis:
+* The original `gcc` compiler options were not recognized by `clang`, or not all include paths were
+correctly detected, so Clang analysis was unsuccessful.
+* Clang was more strict when parsing the C/C++ code than the original compiler (e.g.`gcc`).
+ Any non-standard compliant or `gcc` specific code needs to be removed to successfully analyze the file.    
+* Clang crashed during the analysis.
+
 
 ## Step 3: Store analysis results in a CodeChecker DB and visualize results
 You can store the analysis results in a central database and view the results in a web viewer

docs/user_guide.md

@@ -385,6 +385,7 @@ analyzer arguments:
                         Currently supported analyzers are: clangsa, clang-
                         tidy.
   --add-compiler-defaults
+                        DEPRECATED. Always True. 
                         Retrieve compiler-specific configuration from the
                         compilers themselves, and use them with Clang. This is
                         used when the compiler on the system is special, e.g.

libcodechecker/analyze/log_parser.py

@@ -62,7 +62,16 @@ def get_compiler_includes(compiler, lang, compile_opts, extra_opts=None):
             if line.startswith(end_mark):
                 do_append = False
             if do_append:
-                include_paths.append("-isystem " + line)
+                # On OSX there are framework includes,
+                # where we need to strip the "(framework directory)" string.
+                # For instance:
+                # /System/Library/Frameworks (framework directory)
+                fpos = line.find("(framework directory)")
+                if fpos == -1:
+                    include_paths.append("-isystem " + line)
+                else:
+                    include_paths.append("-isystem " + line[0:fpos-1])
+
             if line.startswith(start_mark):
                 do_append = True
 
@@ -75,16 +84,10 @@ def get_compiler_target(compiler):
     """
     Returns the target triple of the given compiler as a string.
 
-    If the compiler is not a version of GCC, an empty string is returned.
-    Compilers other than GCC might have default targets differing from
-    the build target.
     """
     target_label = "Target:"
     target = ""
 
-    gcc_label = "gcc"
-    gcc = False
-
     cmd = compiler + ' -v'
     LOG.debug("Retrieving target platform information via '" + cmd + "'")
 
@@ -99,10 +102,6 @@ def get_compiler_target(compiler):
             line = line.strip().split()
             if line[0] == target_label:
                 target = line[1]
-            if line[0] == gcc_label:
-                gcc = True
-        if not gcc:
-            target = ""
 
     except OSError as oerr:
         LOG.error("Cannot find compiler target: " + oerr.strerror + "\n")
@@ -112,6 +111,9 @@ def get_compiler_target(compiler):
 
 def parse_compile_commands_json(logfile, add_compiler_defaults=False):
     import json
+    # The add-compiler-defaults is a deprecated argument
+    # and we always perform target and include auto-detection.
+    add_compiler_defaults = True
     LOG.debug('parse_compile_commands_json: ' + str(add_compiler_defaults))
 
     actions = []

libcodechecker/libhandlers/analyze.py

@@ -174,7 +174,8 @@ def add_arguments_to_parser(parser):
                                action='store_true',
                                required=False,
                                default=argparse.SUPPRESS,
-                               help="Retrieve compiler-specific configuration "
+                               help="DEPRECATED. Always True. Retrieve "
+                                    "compiler-specific configuration "
                                     "from the compilers themselves, and use "
                                     "them with Clang. This is used when the "
                                     "compiler on the system is special, e.g. "

libcodechecker/libhandlers/check.py

@@ -209,7 +209,8 @@ def add_arguments_to_parser(parser):
                                action='store_true',
                                required=False,
                                default=argparse.SUPPRESS,
-                               help="Retrieve compiler-specific configuration "
+                               help="DEPRECATED. Always True. Retrieve "
+                                    " compiler-specific configuration "
                                     "from the analyzers themselves, and use "
                                     "them with Clang. This is used when the "
                                     "compiler on the system is special, e.g. "

libcodechecker/log/option_parser.py

@@ -177,9 +177,18 @@
     '^-mmultiple$': 0,
     '^-mthumb-interwork$': 0,
     '^-mupdate$': 0,
+
+    # Deprecated ARM specific option
+    # to Generate a stack frame that is compliant
+    # with the ARM Procedure Call Standard.
+    '^-mapcs': 0,
     '^-fno-merge-const-bfstores$': 0,
     '^-fno-ipa-sra$': 0,
     '^-mno-thumb-interwork$': 0,
+    # ARM specific option.
+    # Prevent the reordering of
+    # instructions in the function prologue.
+    '^-mno-sched-prolog': 0,
     # This is not unknown but we want to preserve asserts to improve the
     # quality of analysis.
     '^-DNDEBUG$': 0
@@ -489,6 +498,13 @@ def parse_options(args):
             result_map.compile_opts[idx] = opt.replace('"', r'"\"')
 
     result_map.compiler = shlex.split(args)[0]
+
+    #  If the compiler is C++ (contains ++ in its name)
+    #  we set the language explicitly to c++.
+    cpp_regex = re.compile('.*\+\+.*')
+    if cpp_regex.match(result_map.compiler) is not None:
+        result_map.lang = 'c++'
+
     is_source = False
     for source_file in result_map.files:
         lang = get_language(os.path.splitext(source_file)[1].rstrip('"'))
@@ -500,6 +516,9 @@ def parse_options(args):
                 result_map.lang = lang
             break
 
+    if result_map.lang:
+        LOG.debug("Detected language" + result_map.lang)
+
     # If there are no source files in the compilation argument
     # handle it as a link command.
     if not is_source:

tests/unit/test_log_parser.py

@@ -52,7 +52,7 @@ def test_old_ldlogger(self):
 
         self.assertEqual(' '.join(results.files),
                          r'"-DVARIABLE="some value"" /tmp/a.cpp')
-        self.assertEqual(len(build_action.analyzer_options), 0)
+        self.assertEqual(len(build_action.analyzer_options), 1)
 
     def test_new_ldlogger(self):
         """
@@ -64,12 +64,13 @@ def test_new_ldlogger(self):
         # now properly log the multiword arguments. When these are parsed by
         # the log_parser, the define's value will be passed to the analyzer.
         #
-        # Logfile contains -DVARIABLE="some value".
+        # Logfile contains -DVARIABLE="some value"
+        # and --target=x86_64-linux-gnu.
 
         build_action = log_parser.parse_log(logfile)[0]
 
         self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
-        self.assertEqual(len(build_action.analyzer_options), 1)
+        self.assertEqual(len(build_action.analyzer_options), 2)
         self.assertEqual(build_action.analyzer_options[0],
                          r'-DVARIABLE="\"some value"\"')
 
@@ -88,14 +89,14 @@ def test_old_intercept_build(self):
         logfile = os.path.join(self.__test_files, "intercept-old.json")
 
         # Scan-build-py shipping with clang-5.0 makes a logfile that contains:
-        # -DVARIABLE=\"some value\"
+        # -DVARIABLE=\"some value\" and --target=x86_64-linux-gnu
         #
         # The define is passed to the analyzer properly.
 
         build_action = log_parser.parse_log(logfile)[0]
 
         self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
-        self.assertEqual(len(build_action.analyzer_options), 1)
+        self.assertEqual(len(build_action.analyzer_options), 2)
         self.assertEqual(build_action.analyzer_options[0],
                          r'-DVARIABLE="\"some value"\"')
 
@@ -117,13 +118,15 @@ def test_new_intercept_build(self):
         # command string. This argument vector contains the define as it's
         # element in the following format:
         # -DVARIABLE=\"some value\"
+        # and the target triplet, e.g.:
+        # --target=x86_64-linux-gnu
         #
         # The define is passed to the analyzer properly.
 
         build_action = log_parser.parse_log(logfile)[0]
 
         self.assertEqual(list(build_action.sources)[0], r'/tmp/a.cpp')
-        self.assertEqual(len(build_action.analyzer_options), 1)
+        self.assertEqual(len(build_action.analyzer_options), 2)
         self.assertEqual(build_action.analyzer_options[0],
                          r'-DVARIABLE="\"some value"\"')
 

vendor/build-logger/README.md

@@ -0,0 +1,69 @@
+# Build Logger
+
+This tool can capture the build process and generate a
+[JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html) 
+
+## Compilation
+
+To build the project execute
+~~~~~~~
+cd vendor/build-logger
+make -f Makefile.manual
+~~~~~~~
+
+## Usage
+
+Set the following environment variables:
+~~~~~~~
+export LD_PRELOAD=ldlogger.so
+export LD_LIBRARY_PATH=`pwd`/build/lib:$LD_LIBRARY_PATH
+export CC_LOGGER_GCC_LIKE="gcc:g++:clang"
+#The output compilation JSON file
+export CC_LOGGER_FILE=`pwd`/compilation.json
+~~~~~~~
+
+then when you call `gcc` from a sub-shell (e.g. as a part of a Make build process),
+ `compilation.json` will be created.
+For example:
+`bash -c "gcc -c something.c"`
+will create
+~~~~~~~
+compilation.json:
+[
+	{
+		"directory": "/home/john_doe/",
+		"command": "/usr/bin/gcc-4.8 -c /home/john_doe/something.c",
+		"file": "/home/john_doe/something.c"
+	}
+]
+~~~~~~~
+
+
+
+## Environment Variables
+
+### `CC_LOGGER_GCC_LIKE` 
+You can change the compilers that should be logged. 
+Set `CC_LOGGER_GCC_LIKE` environment variable to a colon separated list.
+
+ For example (default):
+
+ ```export CC_LOGGER_GCC_LIKE="gcc:g++:clang"```
+
+ The logger will match any compilers with `gcc`,`g++` or `clang` in their filenames.
+
+
+### `CC_LOGGER_FILE`
+Output file to generate compilation database into. 
+This can be a relative or absolute path.
+
+### `CC_LOGGER_JAVAC_LIKE` 
+You can specify the `javac` like 
+compilers that should be logged as a colon separated string list.
+
+### `CC_LOGGER_DEF_DIRS` 
+If the environment variable is defined, 
+the logger will extend the  compiler argument list in the compilation 
+database  with the pre-configured include paths of the logged compiler.
+
+