From 6775c2300fb5494af8a98467d34340d409390c9f Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Tue, 4 Dec 2018 11:54:31 +0000 Subject: [PATCH] Add --force flag If that flag is set, dirty repositories will fail to aggregate. Set it to preserve old behavior of performing a hard reset on them before aggregating. Additionally, if `--force` is found, untracked files will be cleaned from the repository. --- git_aggregator/config.py | 9 ++++++--- git_aggregator/exception.py | 4 ++++ git_aggregator/main.py | 9 ++++++++- git_aggregator/repo.py | 22 ++++++++++++++++++++-- tests/test_config.py | 2 ++ tests/test_repo.py | 35 +++++++++++++++++++++++++++++++++-- 6 files changed, 73 insertions(+), 8 deletions(-) diff --git a/git_aggregator/config.py b/git_aggregator/config.py index 5445bd5..9e538c4 100644 --- a/git_aggregator/config.py +++ b/git_aggregator/config.py @@ -14,9 +14,10 @@ log = logging.getLogger(__name__) -def get_repos(config): +def get_repos(config, force=False): """Return a :py:obj:`list` list of repos from config file. :param config: the repos config in :py:class:`dict` format. + :param bool force: Force aggregate dirty repos or not. :type config: dict :rtype: list """ @@ -27,6 +28,7 @@ def get_repos(config): repo_dict = { 'cwd': directory, 'defaults': repo_data.get('defaults', dict()), + 'force': force, } remote_names = set() if 'remotes' in repo_data: @@ -121,13 +123,14 @@ def get_repos(config): return repo_list -def load_config(config, expand_env=False): +def load_config(config, expand_env=False, force=False): """Return repos from a directory and fnmatch. Not recursive. :param config: paths to config file :type config: str :param expand_env: True to expand environment varialbes in the config. :type expand_env: bool + :param bool force: True to aggregate even if repo is dirty. :returns: expanded config dict item :rtype: iter(dict) """ @@ -143,4 +146,4 @@ def load_config(config, expand_env=False): config = config.substitute(os.environ) conf.import_config(config) - return get_repos(conf.export('dict') or {}) + return get_repos(conf.export('dict') or {}, force) diff --git a/git_aggregator/exception.py b/git_aggregator/exception.py index 74b3bef..86d6a94 100644 --- a/git_aggregator/exception.py +++ b/git_aggregator/exception.py @@ -13,3 +13,7 @@ class ConfigException(GitAggregatorException): """Malformed config definition """ pass + + +class DirtyException(GitAggregatorException): + """Repo directory is dirty""" diff --git a/git_aggregator/main.py b/git_aggregator/main.py index 0848f37..a3ea684 100644 --- a/git_aggregator/main.py +++ b/git_aggregator/main.py @@ -106,6 +106,13 @@ def get_parser(): action='store_true', help='Expand environment variables in configuration file', ) + main_parser.add_argument( + '-f', '--force', + dest='force', + default=False, + action='store_true', + help='Force cleanup and aggregation on dirty repositories.', + ) main_parser.add_argument( '-j', '--jobs', @@ -206,7 +213,7 @@ def run(args): """Load YAML and JSON configs and run the command specified in args.command""" - repos = load_config(args.config, args.expand_env) + repos = load_config(args.config, args.expand_env, args.force) jobs = max(args.jobs, 1) threads = [] diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index a750c03..dacaf3a 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -11,7 +11,7 @@ import requests -from .exception import GitAggregatorException +from .exception import DirtyException, GitAggregatorException from ._compat import console_to_str FETCH_DEFAULTS = ("depth", "shallow-since", "shallow-exclude") @@ -37,7 +37,8 @@ class Repo(object): _git_version = None def __init__(self, cwd, remotes, merges, target, - shell_command_after=None, fetch_all=False, defaults=None): + shell_command_after=None, fetch_all=False, defaults=None, + force=False): """Initialize a git repository aggregator :param cwd: path to the directory where to initialize the repository @@ -54,6 +55,8 @@ def __init__(self, cwd, remotes, merges, target, for every configured remote. :param defaults: Collection of default parameters to be passed to git. + :param bool force: + When ``False``, it will stop if repo is dirty. """ self.cwd = cwd self.remotes = remotes @@ -65,6 +68,7 @@ def __init__(self, cwd, remotes, merges, target, self.target = target self.shell_command_after = shell_command_after or [] self.defaults = defaults or dict() + self.force = force @property def git_version(self): @@ -203,6 +207,17 @@ def push(self): logger.info("Push %s to %s", branch, remote) self.log_call(['git', 'push', '-f', remote, branch], cwd=self.cwd) + def _check_status(self): + """Check repo status and except if dirty.""" + logger.info('Checking repo status') + status = self.log_call( + ['git', 'status', '--porcelain'], + callwith=subprocess.check_output, + cwd=self.cwd, + ) + if status: + raise DirtyException(status) + def _fetch_options(self, merge): """Get the fetch options from the given merge dict.""" cmd = tuple() @@ -213,6 +228,8 @@ def _fetch_options(self, merge): return cmd def _reset_to(self, remote, ref): + if not self.force: + self._check_status() logger.info('Reset branch to %s %s', remote, ref) rtype, sha = self.query_remote_ref(remote, ref) if rtype is None and not ishex(ref): @@ -223,6 +240,7 @@ def _reset_to(self, remote, ref): if logger.getEffectiveLevel() != logging.DEBUG: cmd.insert(2, '--quiet') self.log_call(cmd, cwd=self.cwd) + self.log_call(['git', 'clean', '-ffd'], cwd=self.cwd) def _switch_to_branch(self, branch_name): # check if the branch already exists diff --git a/tests/test_config.py b/tests/test_config.py index 0a4602f..20200d0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -41,6 +41,7 @@ def test_load(self): repos[0], {'cwd': '/product_attribute', 'fetch_all': False, + 'force': False, 'defaults': {}, 'merges': [{'ref': '8.0', 'remote': 'oca'}, {'ref': 'refs/pull/105/head', 'remote': 'oca'}, @@ -87,6 +88,7 @@ def test_load_defaults(self): repos[0], {'cwd': '/web', 'fetch_all': False, + 'force': False, 'defaults': {'depth': 1}, 'merges': [{'ref': '8.0', 'remote': 'oca', 'depth': 1000}, {'ref': 'refs/pull/105/head', 'remote': 'oca'}, diff --git a/tests/test_repo.py b/tests/test_repo.py index b839900..dddea45 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -24,7 +24,7 @@ from git_aggregator.utils import WorkingDirectoryKeeper,\ working_directory_keeper from git_aggregator.repo import Repo -from git_aggregator import main +from git_aggregator import exception, main def git_get_last_rev(repo_dir): @@ -246,6 +246,35 @@ def test_depth_1(self): # Shallow fetch: just 1 commmit self.assertEqual(len(log_shallow.splitlines()), 1) + def test_force(self): + """Ensure --force works fine.""" + remotes = [{ + 'name': 'r1', + 'url': self.url_remote1 + }] + merges = [{ + 'remote': 'r1', + 'ref': 'tag1' + }] + target = { + 'remote': 'r1', + 'branch': 'agg1' + } + # Aggregate 1st time + repo_noforce = Repo(self.cwd, remotes, merges, target) + repo_noforce.aggregate() + # Create a dummy file to set the repo dirty + dummy_file = os.path.join(self.cwd, "dummy") + with open(dummy_file, "a"): + pass + # Aggregate 2nd time, dirty, which should fail + with self.assertRaises(exception.DirtyException): + repo_noforce.aggregate() + # Aggregate 3rd time, forcing, so it should work and remove that file + repo_force = Repo(self.cwd, remotes, merges, target, force=True) + repo_force.aggregate() + self.assertFalse(os.path.exists(dummy_file)) + def test_depth(self): """Ensure `depth` is used correctly.""" remotes = [{ @@ -321,7 +350,9 @@ def test_multithreading(self): jobs=3, dirmatch=None, do_push=False, - expand_env=False) + expand_env=False, + force=False, + ) with working_directory_keeper: os.chdir(self.sandbox)