Fix up Binary Ninja UI options (#48)

* fallback fuzzability analysis with warning alert
* fix up exporting to different formats
* minor fix in harness generation
* fix up older settings with newer ones
* unify logging interfaces
* minor fix in top-level call recognition
* more error handling for UI interactions
* other bugfixes and cleanup

__init__.py

@@ -4,20 +4,34 @@
 
     Plugin module used for Binary Ninja
 """
+import dataclasses
+
 from binaryninja.plugin import PluginCommand
 from binaryninja.settings import Settings
 
 from .fuzzable.analysis import binja, DEFAULT_SCORE_WEIGHTS
+from .fuzzable.config import AnalysisKnobs
 
 # TODO register settings from a config of analysis flags
 
 Settings().register_group("fuzzable", "Fuzzable")
+Settings().register_setting(
+    "fuzzable.list_ignored",
+    """
+    {
+        "title"         : "List Ignored Symbols",
+        "description"   : "If set, will also additionally output and/or export ignored symbols.",
+        "type"          : "boolean",
+        "default"       : false
+    }
+""",
+)
 Settings().register_setting(
     "fuzzable.include_sym",
     """
     {
         "title"         : "Symbols to Include",
-        "description"   : "Include symbols that are accidentally ignored to be considered for analysis.",
+        "description"   : "Comma-seperated list of symbols to absolutely be considered for analysis.",
         "type"          : "array",
         "elementType"   : "string",
         "default"       : []
@@ -42,7 +56,7 @@
     """
     {
         "title"         : "Symbols to Exclude",
-        "description"   : "Exclude symbols from being considered for analaysis.",
+        "description"   : "Exclude symbols from being considered for analysis.",
         "type"          : "array",
         "elementType"   : "string",
         "default"       : []
@@ -63,13 +77,13 @@
 )
 
 Settings().register_setting(
-    "fuzzable.list_ignored",
+    "fuzzable.ignore_metrics",
     """
     {
-        "title"         : "List Ignored Symbols",
-        "description"   : "Include the symbols that we've ignored using `recommend` mode.",
+        "title"         : "Ignoring Displaying Metrics",
+        "description"   : "If set, include individual metrics' scores for each function target analyzed.",
         "type"          : "boolean",
-        "default"       : false
+        "default"       : true
     }
 """,
 )
@@ -90,7 +104,7 @@
 )
 
 PluginCommand.register(
-    "Fuzzable\\Analyze & Rank Functions",
+    "Fuzzable\\Analyze and Rank Functions",
     "List out functions we've determined to be the best candidates for fuzzing."
     "This will exclude functions that is determined to not be directly usable for a harness.",
     binja.run_fuzzable,

fuzzable/analysis/__init__.py

@@ -7,8 +7,15 @@
 import abc
 import typing as t
 
-import skcriteria as skc
-from skcriteria.madm import simple
+# skc has dep problems (e.g in numpy). Set a flag that can be reused to warn the user that
+# the module is not used, and a basic fuzzability calculation will be performed instead.
+BASIC_FUZZABLE_ERROR: t.Optional[str] = None
+try:
+    import skcriteria as skc
+    from skcriteria.madm import simple
+except ImportError as err:
+    BASIC_FUZZABLE_ERROR = f"Cannot import scikit-criteria, using basic ranking method instead. Reason: {repr(err)}"
+
 
 from ..metrics import CallScore, METRICS
 from ..config import GLOBAL_IGNORES, INTERESTING_PATTERNS, RISKY_GLIBC_CALL_PATTERNS
@@ -67,8 +74,6 @@ def __str__(self) -> str:
     def run(self) -> Fuzzability:
         """
         Determine the fuzzability of each function in the binary or source targets.
-        If the mode to recommend targets, determine and statically analyze only top-level calls.
-        If the mode is to rank targets, iterate and analyze over all calls and rank.
         """
         ...
 
@@ -90,6 +95,10 @@ def _rank_fuzzability(self, unranked: t.List[CallScore]) -> Fuzzability:
                 "only one function symbol parsed for fuzzability ranking"
             )
 
+        if BASIC_FUZZABLE_ERROR:
+            log.warning(BASIC_FUZZABLE_ERROR)
+            return AnalysisBackend._rank_simple_fuzzability(unranked)
+
         log.debug("Normalizing static analysis metric values")
         nl_normalized = AnalysisBackend._normalize(
             [score.natural_loops for score in unranked]
@@ -116,7 +125,7 @@ def _rank_fuzzability(self, unranked: t.List[CallScore]) -> Fuzzability:
         dec = simple.WeightedSumModel()
         rank = dec.evaluate(decision_matrix)
 
-        # finalize CallScores by setting scores and ranks
+        log.debug("Finalizing CallScores by setting calculated scores and ranks")
         scores = rank.e_.score
         ranks = list(rank.rank_)
         new_unranked = []
@@ -125,13 +134,13 @@ def _rank_fuzzability(self, unranked: t.List[CallScore]) -> Fuzzability:
             entry.score = score
             new_unranked += [entry]
 
-        # can sort our unranked list appropriately now
+        log.debug("Sorting finalized list by ranks")
         sorted_results = [y for _, y in sorted(zip(ranks, new_unranked))]
         return sorted_results
 
     @staticmethod
     def _rank_simple_fuzzability(unranked: t.List[CallScore]) -> Fuzzability:
-        """Deprecated."""
+        """To be deprecated."""
         return sorted(unranked, key=lambda obj: obj.simple_fuzzability, reverse=True)
 
     @staticmethod

fuzzable/analysis/angr.py

@@ -61,7 +61,7 @@ def run(self) -> Fuzzability:
                 self.skipped[name] = addr
                 continue
 
-            log.debug(f"Checking to see if {name} is a top-level call")
+            log.debug(f"Checking if {name} is a top-level call")
             if not self.include_nontop and not self.is_toplevel_call(func):
                 log.warning(
                     f"Skipping {name} (not top-level) from fuzzability analysis."

fuzzable/analysis/ast.py

@@ -111,7 +111,7 @@ def run(self) -> Fuzzability:
                     log.warning(f"Skipping {name} from fuzzability analysis.")
                     continue
 
-                log.debug(f"Checking to see if {name} is a top-level call")
+                log.debug(f"Checking if {name} is a top-level call")
                 self.is_top_level = self.is_toplevel_call(name, node)
                 if not self.include_nontop and not self.is_top_level:
                     log.warning(