extend coverage stats analysis in reporting (#678)

- adds comparison to the current state of OSS-Fuzz. This makes it easy to see how much improvement the experiment has across the OSS-Fuzz landscape. - adds a new metric, "relative coverage gain" which compares the newly achieved coverage per project relatiev to the existing coverage lines. The point is here the metric of coverage gain relative to total line number does not tell all about the achievements. An interesting metric is how much have been improved relative to the existing manually achieved fuzzing. --------- Signed-off-by: David Korczynski <[email protected]>
google · Oct 30, 2024 · 73cd2a8 · 73cd2a8
1 parent af535be
commit 73cd2a8
Show file tree

Hide file tree

Showing 5 changed files with 109 additions and 19 deletions.
diff --git a/data_prep/introspector.py b/data_prep/introspector.py
@@ -70,6 +70,7 @@
 INTROSPECTOR_ALL_FUNC_TYPES = ''
 INTROSPECTOR_TEST_SOURCE = ''
 INTROSPECTOR_HARNESS_SOURCE_AND_EXEC = ''
+INTROSPECTOR_LANGUAGE_STATS = ''
 
 INTROSPECTOR_HEADERS_FOR_FUNC = ''
 INTROSPECTOR_SAMPLE_XREFS = ''
@@ -107,7 +108,7 @@ def set_introspector_endpoints(endpoint):
       INTROSPECTOR_FUNCTION_WITH_MATCHING_RETURN_TYPE, \
       INTROSPECTOR_ORACLE_ALL_TESTS, INTROSPECTOR_JVM_PROPERTIES, \
       INTROSPECTOR_TEST_SOURCE, INTROSPECTOR_HARNESS_SOURCE_AND_EXEC, \
-      INTROSPECTOR_JVM_PUBLIC_CLASSES
+      INTROSPECTOR_JVM_PUBLIC_CLASSES, INTROSPECTOR_LANGUAGE_STATS
 
   INTROSPECTOR_ENDPOINT = endpoint
 
@@ -145,6 +146,8 @@ def set_introspector_endpoints(endpoint):
       f'{INTROSPECTOR_ENDPOINT}/harness-source-and-executable')
   INTROSPECTOR_JVM_PUBLIC_CLASSES = (
       f'{INTROSPECTOR_ENDPOINT}/all-public-classes')
+  INTROSPECTOR_LANGUAGE_STATS = (
+      f'{INTROSPECTOR_ENDPOINT}/database-language-stats')
 
 
 def _construct_url(api: str, params: dict) -> str:
@@ -456,6 +459,13 @@ def query_introspector_cross_references(project: str,
   return xref_source
 
 
+def query_introspector_language_stats() -> dict:
+  """Queries introspector for language stats"""
+
+  resp = _query_introspector(INTROSPECTOR_LANGUAGE_STATS, {})
+  return _get_data(resp, 'stats', {})
+
+
 def query_introspector_type_info(project: str, type_name: str) -> list[dict]:
   """Queries FuzzIntrospector API for information of |type_name|."""
   resp = _query_introspector(INTROSPECTOR_TYPE, {

diff --git a/report/common.py b/report/common.py
@@ -75,6 +75,7 @@ class Project:
   name: str
   count: int = 0
   coverage_gain: float = 0.0
+  coverage_relative_gain: float = 0.0
   coverage_ofg_total_new_covered_lines = 0
   coverage_existing_total_covered_lines = 0
   coverage_existing_total_lines = 0
@@ -466,6 +467,18 @@ def get_macro_insights(self,
           benchmark.result.max_line_coverage_diff)
     return accumulated_results
 
+  def get_coverage_language_gains(self):
+    """Gets report.json created by experiment runners."""
+    summary_path = os.path.join(self._results_dir, 'report.json')
+    if FileSystem(summary_path).exists():
+      with FileSystem(summary_path).open() as f:
+        try:
+          return json.load(f)
+        except ValueError:
+          # Skip if error
+          logging.debug('Failed to decode project_coverage_gain.json')
+    return {}
+
   def get_project_summary(self, benchmarks: list[Benchmark]) -> list[Project]:
     """Returns a list of project summary."""
     project_summary_dict = {}
@@ -491,6 +504,8 @@ def get_project_summary(self, benchmarks: list[Benchmark]) -> list[Project]:
       for project in project_summary_list:
         if project.name in coverage_dict:
           project.coverage_gain = coverage_dict[project.name]['coverage_diff']
+          project.coverage_relative_gain = coverage_dict[
+              project.name]['coverage_relative_gain']
           project.coverage_ofg_total_new_covered_lines = coverage_dict[
               project.name]['coverage_ofg_total_new_covered_lines']
           project.coverage_existing_total_covered_lines = coverage_dict[

diff --git a/report/templates/index.html b/report/templates/index.html
@@ -106,6 +106,7 @@ <h2>Project summary</h2>
             <th data-sorted="asc">Project</th>
             <th data-sort-number>Total generated harnesses</th>
             <th data-sort-number>Total coverage gain</th>
+            <th data-sort-number>Total relative gain</th>
             <th data-sort-number>OSS-Fuzz-gen total covered lines</th>
             <th data-sort-number>OSS-Fuzz-gen new covered lines</th>
             <th data-sort-number>Existing covered lines</th>
@@ -118,7 +119,8 @@ <h2>Project summary</h2>
             <td class="table-index">{{ loop.index }}</td>
             <td data-sort-value="{{ project.name }}">{{ project.name }}</td>
             <td data-sort-value="{{ project.count }}">{{ project.count }}</td>
-            <td data-sort-value="{{ project.coverage_gain|percent }}">{{ project.coverage_gain|percent }}</td>
+            <td data-sort-value="{{ project.coverage_gain|percent }}">{{ project.coverage_gain|percent }}%</td>
+            <td data-sort-value="{{ project.coverage_relative_gain|percent}}">{{ project.coverage_relative_gain | percent }}%</td>
             <td data-sort-value="{{ project.coverage_ofg_total_covered_lines}}">{{project.coverage_ofg_total_covered_lines}}</td>
             <td data-sort-value="{{ project.coverage_ofg_total_new_covered_lines}}">{{ project.coverage_ofg_total_new_covered_lines}}</td>
             <td data-sort-value="{{ project.coverage_existing_total_covered_lines}}">{{ project.coverage_existing_total_covered_lines}}</td>
@@ -128,6 +130,30 @@ <h2>Project summary</h2>
     </tbody>
 </table>
 
+<h2>Language coverage gains</h2>
+<table class="sortable-table" id="language-coverage-gain">
+    <thead>
+        <th>language</th>
+        <th>OSS-Fuzz total lines</th>
+        <th>OSS-Fuzz coverage lines</th>
+        <th>Experiment new coverage lines</th>
+        <th>Increase of total</th>
+        <th>Increase of covered</th>
+    </thead>
+    <tbody>
+        {% for language in coverage_language_gains['coverage_gains_per_language'] %}
+        <tr>
+            <td>{{language}}</td>
+            <td>{{coverage_language_gains['oss_fuzz_language_status'][language]['total']}}</td>
+            <td>{{coverage_language_gains['oss_fuzz_language_status'][language]['covered']}}</td>
+            <td>{{coverage_language_gains['coverage_gains_per_language'][language]}}</td>
+            <td>{{coverage_language_gains['comperative_coverage_gains'][language]['total_coverage_increase']}}%</td>
+            <td>{{coverage_language_gains['comperative_coverage_gains'][language]['relative_coverage_increase']}}%</td>
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+
 <h2>Crashes found by generated fuzz harnesses</h2>
 <table class="sortable-table" id="bug-summary-table">
     <thead>

diff --git a/report/web.py b/report/web.py
@@ -162,11 +162,12 @@ def generate(self):
 
     accumulated_results = self._results.get_macro_insights(benchmarks)
     projects = self._results.get_project_summary(benchmarks)
+    coverage_language_gains = self._results.get_coverage_language_gains()
 
     time_results = self.read_timings()
 
     self._write_index_html(benchmarks, accumulated_results, time_results,
-                           projects, samples_with_bugs)
+                           projects, samples_with_bugs, coverage_language_gains)
     self._write_index_json(benchmarks)
 
   def _write(self, output_path: str, content: str):
@@ -187,14 +188,17 @@ def _write(self, output_path: str, content: str):
   def _write_index_html(self, benchmarks: List[Benchmark],
                         accumulated_results: AccumulatedResult,
                         time_results: dict[str, Any], projects: list[Project],
-                        samples_with_bugs: list[dict[str, Any]]):
+                        samples_with_bugs: list[dict[str, Any]],
+                        coverage_language_gains: dict[str, Any]):
     """Generate the report index.html and write to filesystem."""
-    rendered = self._jinja.render('index.html',
-                                  benchmarks=benchmarks,
-                                  accumulated_results=accumulated_results,
-                                  time_results=time_results,
-                                  projects=projects,
-                                  samples_with_bugs=samples_with_bugs)
+    rendered = self._jinja.render(
+        'index.html',
+        benchmarks=benchmarks,
+        accumulated_results=accumulated_results,
+        time_results=time_results,
+        projects=projects,
+        samples_with_bugs=samples_with_bugs,
+        coverage_language_gains=coverage_language_gains)
     self._write('index.html', rendered)
 
   def _write_index_json(self, benchmarks: List[Benchmark]):

diff --git a/run_all_experiments.py b/run_all_experiments.py
@@ -303,19 +303,46 @@ def parse_args() -> argparse.Namespace:
   return args
 
 
+def extend_report_with_coverage_gains() -> None:
+  """Process total gain from all generated harnesses for each projects and
+  update summary report. This makes it possible to view per-project stats
+  as experiments complete rather than only after all experiments run."""
+  coverage_gain_dict = _process_total_coverage_gain()
+  existing_oss_fuzz_cov = introspector.query_introspector_language_stats()
+
+  total_new_covgains = {}
+  for project_dict in coverage_gain_dict.values():
+    lang_gains = total_new_covgains.get(project_dict['language'], 0)
+    lang_gains += project_dict['coverage_ofg_total_new_covered_lines']
+    total_new_covgains[project_dict['language']] = lang_gains
+
+  comparative_cov_gains = {}
+  for language, lang_cov_gain in total_new_covgains.items():
+    comparative_cov_gains[language] = {
+        'total_coverage_increase':
+            round((lang_cov_gain / existing_oss_fuzz_cov[language]['total']) *
+                  100.0, 10),
+        'relative_coverage_increase':
+            round((lang_cov_gain / existing_oss_fuzz_cov[language]['covered']) *
+                  100.0, 10)
+    }
+  add_to_json_report(WORK_DIR, 'coverage_gains_per_language',
+                     total_new_covgains)
+  add_to_json_report(WORK_DIR, 'project_summary', coverage_gain_dict)
+  add_to_json_report(WORK_DIR, 'oss_fuzz_language_status',
+                     existing_oss_fuzz_cov)
+  add_to_json_report(WORK_DIR, 'comperative_coverage_gains',
+                     comparative_cov_gains)
+
+
 def _print_and_dump_experiment_result(result: Result):
   """Prints the |result| of a single experiment."""
   logger.info('\n**** Finished benchmark %s, %s ****\n%s',
               result.benchmark.project, result.benchmark.function_signature,
               result.result)
 
   EXPERIMENT_RESULTS.append(result)
-
-  # Process total gain from all generated harnesses for each projects and
-  # update summary report. This makes it possible to view per-project stats
-  # as experiments complete rather than only after all experiments run.
-  coverage_gain_dict = _process_total_coverage_gain()
-  add_to_json_report(WORK_DIR, 'project_summary', coverage_gain_dict)
+  extend_report_with_coverage_gains()
 
 
 def _print_experiment_results(results: list[Result],
@@ -441,13 +468,22 @@ def _process_total_coverage_gain() -> dict[str, dict[str, Any]]:
     total_existing_lines = sum(lines)
     total_cov_covered_lines_before_subtraction = total_cov.covered_lines
     total_cov.subtract_covered_lines(existing_textcov)
+    try:
+      cov_relative_gain = (total_cov.covered_lines /
+                           existing_textcov.covered_lines)
+    except ZeroDivisionError:
+      cov_relative_gain = 0.0
 
     total_lines = max(total_cov.total_lines, total_existing_lines)
 
     if total_lines:
       coverage_gain[project] = {
+          'language':
+              oss_fuzz_checkout.get_project_language(project),
           'coverage_diff':
               total_cov.covered_lines / total_lines,
+          'coverage_relative_gain':
+              cov_relative_gain,
           'coverage_ofg_total_covered_lines':
               total_cov_covered_lines_before_subtraction,
           'coverage_ofg_total_new_covered_lines':
@@ -513,9 +549,7 @@ def main():
       # Wait for all workers to complete.
       p.join()
 
-  # Process total gain from all generated harnesses for each projects
-  coverage_gain_dict = _process_total_coverage_gain()
-  add_to_json_report(args.work_dir, 'project_summary', coverage_gain_dict)
+  extend_report_with_coverage_gains()
 
   # Capture time at end
   end = time.time()
@@ -524,6 +558,7 @@ def main():
   add_to_json_report(args.work_dir, 'total_run_time',
                      str(timedelta(seconds=end - start)))
 
+  coverage_gain_dict = _process_total_coverage_gain()
   _print_experiment_results(EXPERIMENT_RESULTS, coverage_gain_dict)