config: runtime: add new coverage report template

This new template is meant to be executed as a post-processing job, once
every test job for a given `kbuild` node are complete. It gathers the
raw coverage data from each of those test jobs and processes it as
follows:
* create a single JSON tracefile using `gcovr`
* extract lines/functions coverage percentages for each job
* create `test` child nodes for each job, reporting those percentages
* merge all tracefiles in a single results file and generate both an
  HTML report and `lcov`-compatible tracefile; the HTML report gives a
  quick overview of the code coverage, while the tracefile can then be
  downloaded by developers for more targeted processing
* create `test` child nodes for the `kbuild` job, reporting global
  lines/functions coverage percentages for this run

Signed-off-by: Arnaud Ferraris <[email protected]>

Author: a-wai

config/runtime/coverage-report.jinja2

@@ -0,0 +1,220 @@
+{# -*- mode: Python -*- -#}
+{# SPDX-License-Identifier: LGPL-2.1-or-later -#}
+
+{%- extends 'base/python.jinja2' %}
+
+{%- block python_imports %}
+{{ super() }}
+import gzip
+import json
+import shutil
+import subprocess
+{%- endblock %}
+
+{%- block python_local_imports %}
+{{ super() }}
+import kernelci.api.helper
+{%- endblock %}
+
+{%- block python_globals %}
+{{ super() }}
+{% endblock %}
+
+{% block python_job -%}
+class Job(BaseJob):
+    def _upload_artifacts(self, local_artifacts):
+        artifacts = {}
+        storage = self._get_storage()
+        if storage and self._node:
+            root_path = '-'.join([JOB_NAME, self._node['id']])
+            print(f"Uploading artifacts to {root_path}")
+            for name, file_path in local_artifacts.items():
+                if os.path.exists(file_path):
+                    file_url = storage.upload_single(
+                        (file_path, os.path.basename(file_path)), root_path
+                    )
+                    print(file_url)
+                    artifacts[name] = file_url
+        return artifacts
+
+    def _extract_coverage(self, summary_file, node=None):
+        if node is None:
+            node = self._node
+
+        child_nodes = []
+
+        with open(summary_file, encoding='utf-8') as summary_json:
+            summary = json.load(summary_json)
+            node_data = node['data']
+
+            func_data = node_data.copy()
+            func_percent = summary.get('function_percent')
+            if func_percent is not None:
+                func_data['misc'] = {}
+                func_data['misc']['measurement'] = func_percent
+                child_nodes += [
+                    {
+                        'node': {
+                            'kind': 'test',
+                            'name': 'coverage.functions',
+                            'result': 'pass',
+                            'state': 'done',
+                            'data': func_data,
+                        },
+                        'child_nodes': [],
+                    },
+                ]
+
+            line_data = node_data.copy()
+            line_percent = summary.get('function_percent')
+            if line_percent is not None:
+                line_data['misc'] = {}
+                line_data['misc']['measurement'] = line_percent
+                child_nodes += [
+                    {
+                        'node': {
+                            'kind': 'test',
+                            'name': 'coverage.lines',
+                            'result': 'pass',
+                            'state': 'done',
+                            'data': line_data,
+                        },
+                        'child_nodes': [],
+                    },
+                ]
+
+        return {
+            'node': {
+                'result': 'pass' if node['id'] == self._node['id'] else node['result'],
+                'artifacts': {},
+            },
+            'child_nodes': child_nodes,
+        }
+
+    def _run(self, src_path):
+        api_helper = kernelci.api.helper.APIHelper(self._api)
+        child_nodes = self._api.node.findfast({'parent': self._parent['id']})
+
+        # Ensure our working dir exists and create the log file there
+        os.makedirs(self._workspace, exist_ok=True)
+        log_path = os.path.join(self._workspace, f"log.txt")
+        log_file = open(log_path, mode='w')
+
+        log_file.write("Getting coverage source...\n")
+        tarball_url = self._get_artifact_url(self._parent, 'coverage_source_tar_xz')
+        self._get_source(tarball_url)
+        # Not getting src_path from _get_source() as it doesn't work in our case
+        # We do know that the top-level folder is named 'linux' however, so let's
+        # just use that
+        src_path = os.path.join(self._workspace, 'linux')
+        log_file.write(f"Coverage source downloaded from {tarball_url}\n")
+
+        base_cmd = ['gcovr', '--root', src_path]
+        tracefiles = []
+
+        # Download and process coverage data for all child nodes
+        for cnode in child_nodes:
+            if cnode['id'] == self._node['id']:
+                log_file.write(f"Skipping self ({cnode['id']})\n")
+                continue
+
+            coverage_dir = os.path.join(self._workspace, f"coverage-{cnode['id']}")
+            json_summary = coverage_dir + '.summary.json'
+            try:
+                data_url = self._get_artifact_url(cnode, 'coverage_data')
+                tracefile = coverage_dir + '.json'
+                self._get_source(data_url, path=coverage_dir)
+                log_file.write(f"Downloaded coverage data from {data_url}\n")
+            except:
+                log_file.write(f"WARNING: Unable to download coverage data for {cnode['id']}\n")
+                continue
+
+            # We now have raw coverage data available, process it
+            log_file.write(f"--- Processing coverage data for {cnode['id']} ---\n")
+            cmd = subprocess.run(base_cmd + [
+                '--gcov-ignore-parse-errors',
+                '--object-directory', coverage_dir,
+                '--json', tracefile,
+                '--json-summary', json_summary,
+            ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+            log_file.write(cmd.stdout)
+
+            try:
+                cmd.check_returncode()
+            except:
+                log_file.write(f"WARNING: Unable to process coverage data for {cnode['id']}")
+                continue
+
+            tracefiles += [tracefile]
+            results = self._extract_coverage(json_summary, node=cnode)
+            # We only want to create child nodes reporting coverage percentages, not actually
+            # update the test node
+            if len(results['child_nodes']) > 0:
+                api_helper.submit_results(results, cnode)
+
+        # Coverage data has been processed for all child nodes, we can now merge the tracefiles
+        args = base_cmd
+        for trace in tracefiles:
+            args += ['--add-tracefile', trace]
+
+        output_base = os.path.join(self._workspace, f"coverage-{self._parent['id']}")
+        json_summary = output_base + '.summary.json'
+        html_report = output_base + '.html'
+        lcov_tracefile = output_base + '.info'
+        args += [
+            '--json-summary', json_summary,
+            '--html', html_report,
+            '--lcov', lcov_tracefile,
+        ]
+
+        log_file.write("--- Merging tracefiles ---\n")
+        cmd = subprocess.run(args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True)
+        log_file.write(cmd.stdout)
+
+        # Ensure job completed successfully or report failure
+        try:
+            cmd.check_returncode()
+        except:
+            log_file.write(f"ERROR: Unable to generate coverage report\n")
+            log_file.close()
+
+            artifacts = self._upload_artifacts({ 'log': log_path })
+            return {
+                'node': {
+                    'result': 'fail',
+                    'artifacts': artifacts,
+                },
+                'child_nodes': [],
+            }
+
+        log_file.write("--- Compressing artifacts ---\n")
+        compressed_lcov = lcov_tracefile + '.gz'
+        with open(lcov_tracefile, 'rb') as f_in:
+            with gzip.open(compressed_lcov, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+
+        # Finish writing the job log and upload it along with other artifacts
+        log_file.write("--- Job successful ---\n")
+        log_file.close()
+
+        artifacts = {
+            'coverage_report': html_report,
+            'tracefile': compressed_lcov,
+            'log': log_path,
+        }
+        artifacts = self._upload_artifacts(artifacts)
+
+        return self._extract_coverage(json_summary)
+
+    def _submit(self, result):
+        # Ensure top-level name is kept the same
+        result = result.copy()
+        result['node']['name'] = self._node['name']
+        result['node']['state'] = 'done'
+        api_helper = kernelci.api.helper.APIHelper(self._api)
+        api_helper.submit_results(result, self._node)
+
+{% endblock %}