From f9fc688c852386ba2c0955947b1bd6bd3bcc216f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Csord=C3=A1s?= Date: Tue, 7 Sep 2021 14:56:20 +0200 Subject: [PATCH] [cli] Handle use case when git is not available It is possible that `git` is not available on the user's system where the `CodeChecker store` command is executed. In this case importing git module will fail. With this patch we will handle this use case and print a warning message to the user. This patch will also print some information to the user whether collecting blame information was successfull or if no blame information is available for source files. --- web/client/codechecker_client/blame_info.py | 111 ++++++++++++++++++++ web/client/codechecker_client/cmd/store.py | 100 +++--------------- 2 files changed, 128 insertions(+), 83 deletions(-) create mode 100644 web/client/codechecker_client/blame_info.py diff --git a/web/client/codechecker_client/blame_info.py b/web/client/codechecker_client/blame_info.py new file mode 100644 index 0000000000..ce3d128d84 --- /dev/null +++ b/web/client/codechecker_client/blame_info.py @@ -0,0 +1,111 @@ +import json +import os +import sys +import zipfile + +from concurrent.futures import ProcessPoolExecutor +from git import Repo +from git.exc import InvalidGitRepositoryError +from typing import Dict, List, Optional + +from codechecker_common.logger import get_logger + +LOG = get_logger('system') + + +FileBlameInfo = Dict[str, Optional[Dict]] + + +def __get_blame_info(file_path: str): + """ Get blame info for the given file. """ + try: + repo = Repo(file_path, search_parent_directories=True) + except InvalidGitRepositoryError: + return + + tracking_branch = None + try: + # If a commit is checked out, accessing the active_branch member will + # throw a type error. In this case we will use the current commit hash. + tracking_branch = str(repo.active_branch.tracking_branch()) + except TypeError: + tracking_branch = repo.head.commit.hexsha + + try: + blame = repo.blame_incremental(repo.head.commit.hexsha, file_path) + + res = { + 'version': 'v1', + 'tracking_branch': tracking_branch, + 'remote_url': next(repo.remote().urls, None), + 'commits': {}, + 'blame': []} + + for b in blame: + commit = b.commit + + if commit.hexsha not in res['commits']: + res['commits'][commit.hexsha] = { + 'author': { + 'name': commit.author.name, + 'email': commit.author.email, + }, + 'summary': commit.summary, + 'message': commit.message, + 'committed_datetime': str(commit.committed_datetime)} + + res['blame'].append({ + 'from': b.linenos[0], + 'to': b.linenos[-1], + 'commit': commit.hexsha}) + + LOG.debug("Collected blame info for %s", file_path) + + return res + except Exception as ex: + LOG.debug("Failed to get blame information for %s: %s", file_path, ex) + + +def __collect_blame_info_for_files( + file_paths: List[str], + zip_iter=map +) -> FileBlameInfo: + """ Collect blame information for the given file paths. """ + file_blame_info = {} + for file_path, blame_info in zip(file_paths, + zip_iter(__get_blame_info, file_paths)): + file_blame_info[file_path] = blame_info + + return file_blame_info + + +def assemble_blame_info( + zip_file: zipfile.ZipFile, + file_paths: List[str] +) -> bool: + """ + Collect and write blame information for the given files to the zip file. + + Returns true if at least one blame information is collected. + """ + # Currently ProcessPoolExecutor fails completely in windows. + # Reason is most likely combination of venv and fork() not + # being present in windows, so stuff like setting up + # PYTHONPATH in parent CodeChecker before store is executed + # are lost. + if sys.platform == "win32": + file_blame_info = __collect_blame_info_for_files(file_paths) + else: + with ProcessPoolExecutor() as executor: + file_blame_info = __collect_blame_info_for_files( + file_paths, executor.map) + + # Add blame information to the zip for the files which will be sent + # to the server if exist. + for f, blame_info in file_blame_info.items(): + if blame_info: + zip_file.writestr( + os.path.join('blame', f.lstrip('/')), + json.dumps(blame_info)) + + return any(v for v in file_blame_info.values()) diff --git a/web/client/codechecker_client/cmd/store.py b/web/client/codechecker_client/cmd/store.py index d8ded1c114..3960256fd3 100644 --- a/web/client/codechecker_client/cmd/store.py +++ b/web/client/codechecker_client/cmd/store.py @@ -13,8 +13,6 @@ import argparse import base64 -from git import Repo -from git.exc import InvalidGitRepositoryError import hashlib import json import os @@ -32,7 +30,6 @@ from codechecker_api_shared.ttypes import RequestFailed, ErrorCode from codechecker_client import client as libclient - from codechecker_common import arg, logger, plist_parser, util, cmd_config from codechecker_common.report import Report from codechecker_common.output import twodim @@ -43,6 +40,11 @@ from codechecker_web.shared import webserver_context, host_check from codechecker_web.shared.env import get_default_workspace +try: + from codechecker_client.blame_info import assemble_blame_info +except ImportError: + pass + LOG = logger.get_logger('system') MAX_UPLOAD_SIZE = 1 * 1024 * 1024 * 1024 # 1GiB @@ -581,66 +583,6 @@ def parse_report_files(report_files: Set[str], zip_iter=map): missing_source_files) -def get_blame_info(file_path: str): - """ Get blame info for the given file. """ - try: - repo = Repo(file_path, search_parent_directories=True) - except InvalidGitRepositoryError: - return - - tracking_branch = None - try: - # If a commit is checked out, accessing the active_branch member will - # throw a type error. In this case we will use the current commit hash. - tracking_branch = str(repo.active_branch.tracking_branch()) - except TypeError: - tracking_branch = repo.head.commit.hexsha - - try: - blame = repo.blame_incremental(repo.head.commit.hexsha, file_path) - - res = { - 'version': 'v1', - 'tracking_branch': tracking_branch, - 'remote_url': next(repo.remote().urls, None), - 'commits': {}, - 'blame': []} - - for b in blame: - commit = b.commit - - if commit.hexsha not in res['commits']: - res['commits'][commit.hexsha] = { - 'author': { - 'name': commit.author.name, - 'email': commit.author.email, - }, - 'summary': commit.summary, - 'message': commit.message, - 'committed_datetime': str(commit.committed_datetime)} - - res['blame'].append({ - 'from': b.linenos[0], - 'to': b.linenos[-1], - 'commit': commit.hexsha}) - - LOG.debug("Collected blame info for %s", file_path) - - return res - except Exception as ex: - LOG.debug("Failed to get blame information for %s: %s", file_path, ex) - - -def collect_blame_info_for_files(file_paths: List[str], zip_iter=map): - """ Collect blame information for the given file paths. """ - file_blame_info = {} - for file_path, blame_info in zip(file_paths, - zip_iter(get_blame_info, file_paths)): - file_blame_info[file_path] = blame_info - - return file_blame_info - - def assemble_zip(inputs, zip_file, client): """Collect and compress report and source files, together with files contanining analysis related information into a zip file which @@ -751,26 +693,18 @@ def assemble_zip(inputs, zip_file, client): except KeyError: zipf.write(f, file_path) - # Currently ProcessPoolExecutor fails completely in windows. - # Reason is most likely combination of venv and fork() not - # being present in windows, so stuff like setting up - # PYTHONPATH in parent CodeChecker before store is executed - # are lost. - if sys.platform == "win32": - file_blame_info = collect_blame_info_for_files( - collected_file_paths) - else: - with ProcessPoolExecutor() as executor: - file_blame_info = collect_blame_info_for_files( - collected_file_paths, executor.map) - - # Add blame information to the zip for the files which will be sent to - # the server if exist. - for f, blame_info in file_blame_info.items(): - if blame_info: - zipf.writestr( - os.path.join('blame', f.lstrip('/')), - json.dumps(blame_info)) + if collected_file_paths: + LOG.info("Collecting blame information for source files...") + try: + if assemble_blame_info(zipf, collected_file_paths): + LOG.info("Collecting blame information done.") + else: + LOG.info("No blame information found for source files.") + except NameError: + LOG.warning( + "Collecting blame information has been failed. Make sure " + "'git' is available on your system to hide this warning " + "message.") zipf.writestr('content_hashes.json', json.dumps(file_to_hash))