From a8d3f05a4470d7542cc70f99aa94af3d81088c41 Mon Sep 17 00:00:00 2001 From: Charlie Liu Date: Sat, 17 Nov 2018 18:43:29 -0500 Subject: [PATCH] Add option to order results by last commit date This adds the ability to clone every repo (which can be used for other purposes later) and determine the date of the last commit, which is then recorded and can be used to sort the export results. Closes https://github.com/ksdme/org-status/issues/7 --- org_status/encoders/gitman.py | 4 +- org_status/org_hosts/__init__.py | 3 +- org_status/org_hosts/github.py | 12 +++++- org_status/org_hosts/gitlab.py | 17 +++++++- org_status/org_status.py | 17 ++++++-- requirements.txt | 2 + .../present_status_output_no_sort.txt | 4 ++ .../fixtures/present_status_output_sorted.txt | 4 ++ tests/test_main.py | 40 ++++++++++++++++++- 9 files changed, 93 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/present_status_output_no_sort.txt create mode 100644 tests/fixtures/present_status_output_sorted.txt diff --git a/org_status/encoders/gitman.py b/org_status/encoders/gitman.py index 12eea2f..23c49a6 100644 --- a/org_status/encoders/gitman.py +++ b/org_status/encoders/gitman.py @@ -14,6 +14,6 @@ def convert_repo_list_to_format(self, repos): name = parse(repo.web_url).repo yml_data['sources'].append({'name': name, 'repo': repo.web_url, - 'rev': 'master'}) - + 'rev': 'master', + }) return yaml.dump(yml_data, default_flow_style=False) diff --git a/org_status/org_hosts/__init__.py b/org_status/org_hosts/__init__.py index 473dfcb..8844cef 100644 --- a/org_status/org_hosts/__init__.py +++ b/org_status/org_hosts/__init__.py @@ -33,9 +33,10 @@ def repositories(self): class RepoStatus: - def __init__(self, repo_url, repo_status): + def __init__(self, repo_url, repo_status, last_commit_date): self.repo_url = repo_url self.repo_status = repo_status + self.last_commit_date = last_commit_date def get_all_supported_hosts(): diff --git a/org_status/org_hosts/github.py b/org_status/org_hosts/github.py index 65a5f1d..15653d3 100644 --- a/org_status/org_hosts/github.py +++ b/org_status/org_hosts/github.py @@ -1,4 +1,5 @@ import json +from types import MethodType import requests from IGitt.GitHub.GitHub import GitHubToken @@ -21,6 +22,9 @@ def __init__(self, token, group, **kargs): self._token = GitHubToken(token) self._org = GitHubOrganization(self._token, self._group) + for repo in self._org.repositories: + repo.get_last_commit_date = MethodType(_get_last_commit_date, repo) + self._status_provider = [] for i in enumerate(self.StatusProvider): self._status_provider.append(self.StatusProvider[i[0]](self._group)) @@ -56,8 +60,14 @@ def process_repository(self, repo, branch='master'): elif Status.ERROR in repo_status: repo_status = Status.ERROR - return RepoStatus(repo.web_url, repo_status) + last_commit_date = repo.get_last_commit_date() + + return RepoStatus(repo.web_url, repo_status, last_commit_date) @property def repositories(self): return self._org.repositories + + +def _get_last_commit_date(self): + return self.data._data['pushed_at'] diff --git a/org_status/org_hosts/gitlab.py b/org_status/org_hosts/gitlab.py index 65ac0e9..200307f 100644 --- a/org_status/org_hosts/gitlab.py +++ b/org_status/org_hosts/gitlab.py @@ -1,4 +1,5 @@ import json +from types import MethodType import requests from IGitt.GitLab.GitLab import GitLabPrivateToken @@ -22,6 +23,9 @@ def __init__(self, token, group, **kargs): self._token = GitLabPrivateToken(token) self._org = GitLabOrganization(self._token, self._group) + for repo in self._org.repositories: + repo.get_last_commit_date = MethodType(_get_last_commit_date, repo) + self._status_provider = self.StatusProvider(self._group) @classmethod @@ -39,8 +43,19 @@ def process_repository(self, repo, branch='master'): self.HostName, branch=branch) - return RepoStatus(repo.web_url, repo_status) + last_commit_date = repo.get_last_commit_date() + + return RepoStatus(repo.web_url, repo_status, last_commit_date) @property def repositories(self): return self._org.repositories + + +def _get_last_commit_date(self): + repo_commit_dates = [''] + for commit in self.commits: + repo_commit_dates.append(commit.data._data['committed_date']) + repo_commit_dates.sort() + + return repo_commit_dates[-1] diff --git a/org_status/org_status.py b/org_status/org_status.py index a9e6f52..1b545a9 100644 --- a/org_status/org_status.py +++ b/org_status/org_status.py @@ -47,10 +47,13 @@ def aggregate_org_status(org_host, threads=2): return pool.map(org_host.process_repository, org_host.repositories) -def present_status(statuses, no_color): +def present_status(statuses, no_color, sort): color = (lambda l, *_: l) if no_color else colored r_pass, r_fail, r_unknown, r_error = 0, 0, 0, 0 + if sort: + statuses.sort(key=lambda status: status.last_commit_date, reverse=True) + for status in statuses: repo_status = status.repo_status or Status.UNDETERMINED status_text = repo_status.value @@ -68,8 +71,13 @@ def present_status(statuses, no_color): r_unknown += 1 continue - print('{repo}: {status}'.format( - repo=status.repo_url, status=status_text)) + date_text = color(status.last_commit_date, 'blue') + + print('{repo}: {status} (last commit: {date})'.format( + repo=status.repo_url, + status=status_text, + date=date_text, + )) print('{} Passing, {} Failing, {} Error, {} Unknown ' 'of {} Repositories'.format( @@ -85,6 +93,7 @@ def get_argument_parser(): parser.add_argument('--hosts-only', '-o', action='store_true') parser.add_argument('--skip-host-checks', action='store_true') parser.add_argument('--export-repos', type=str) + parser.add_argument('--sort-by-last-commit', action='store_true') parser.add_argument('--format', type=str, default='gitman') parser.add_argument('--check-providers-only', action='store_true') @@ -196,7 +205,7 @@ def main(): continue org_status = aggregate_org_status(org_host, threads=args.threads) - present_status(org_status, args.no_color) + present_status(org_status, args.no_color, args.sort_by_last_commit) if args.export_repos: export_data = encode_repo_list(all_repositories, args.format, styled) diff --git a/requirements.txt b/requirements.txt index 5ee0a6b..630f198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ git+https://gitlab.com/gitmate/open-source/IGitt.git#egg=IGitt requests termcolor giturlparse +pyyaml +gitman diff --git a/tests/fixtures/present_status_output_no_sort.txt b/tests/fixtures/present_status_output_no_sort.txt new file mode 100644 index 0000000..d236b21 --- /dev/null +++ b/tests/fixtures/present_status_output_no_sort.txt @@ -0,0 +1,4 @@ +https://github.com/foo/r0: passing (last commit: 2018-11-05T08:20:37Z) +https://github.com/foo/r1: passing (last commit: 2018-11-02T17:31:46Z) +https://github.com/foo/r2: passing (last commit: 2018-11-06T02:27:42Z) +0 Passing, 0 Failing, 0 Error, 0 Unknown of 3 Repositories \ No newline at end of file diff --git a/tests/fixtures/present_status_output_sorted.txt b/tests/fixtures/present_status_output_sorted.txt new file mode 100644 index 0000000..1426b7d --- /dev/null +++ b/tests/fixtures/present_status_output_sorted.txt @@ -0,0 +1,4 @@ +https://github.com/foo/r2: passing (last commit: 2018-11-06T02:27:42Z) +https://github.com/foo/r0: passing (last commit: 2018-11-05T08:20:37Z) +https://github.com/foo/r1: passing (last commit: 2018-11-02T17:31:46Z) +0 Passing, 0 Failing, 0 Error, 0 Unknown of 3 Repositories \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index fb39178..9d49740 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,6 @@ import pytest from unittest import mock +from io import StringIO from argparse import ArgumentParser @@ -7,7 +8,9 @@ generate_fetch_jobs, get_host_token, get_supported_status_providers, - get_status_provider_statuses) + get_status_provider_statuses, + present_status, + ) from org_status.org_hosts import get_all_supported_hosts @@ -56,3 +59,38 @@ def test_get_status_provider_statuses(): expected_status_provider_statuses.add((status_provider, status)) assert actual_status_provider_statuses == expected_status_provider_statuses + + +class MockRepoStatus(): + def __init__(self, value): + self.value = value + + +class MockStatus(): + def __init__(self, repo_url, repo_status, last_commit_date): + self.repo_url = repo_url + self.repo_status = MockRepoStatus(repo_status) + self.last_commit_date = last_commit_date + + +def test_present_status(): + statuses = [MockStatus('https://github.com/foo/r0', + 'passing', '2018-11-05T08:20:37Z'), + MockStatus('https://github.com/foo/r1', + 'passing', '2018-11-02T17:31:46Z'), + MockStatus('https://github.com/foo/r2', + 'passing', '2018-11-06T02:27:42Z')] + + with mock.patch('sys.stdout', new=StringIO()) as output: + present_status(statuses, True, False) + actual = output.getvalue().strip() + with open('tests/fixtures/present_status_output_no_sort.txt', 'r') as f: + expected = f.read() + assert actual == expected + + with mock.patch('sys.stdout', new=StringIO()) as output: + present_status(statuses, True, True) + actual = output.getvalue().strip() + with open('tests/fixtures/present_status_output_sorted.txt', 'r') as f: + expected = f.read() + assert actual == expected