From 893101065e1560579f3c85be0b164715138e2cb3 Mon Sep 17 00:00:00 2001 From: Charlie Liu Date: Sat, 17 Nov 2018 18:43:29 -0500 Subject: [PATCH] Add option to order results by last commit date This adds the ability to clone every repo (which can be used for other purposes later) and determine the date of the last commit, which is then recorded and can be used to sort the export results. Closes https://github.com/ksdme/org-status/issues/7 --- org_status/encoders/gitman.py | 9 +++- org_status/org_hosts/__init__.py | 3 +- org_status/org_hosts/github.py | 12 +++++- org_status/org_hosts/gitlab.py | 17 +++++++- org_status/org_status.py | 17 ++++++-- requirements.txt | 2 + .../present_status_output_no_sort.txt | 4 ++ .../fixtures/present_status_output_sorted.txt | 4 ++ tests/test_encoders.py | 13 ++++++ tests/test_main.py | 41 ++++++++++++++++++- 10 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/present_status_output_no_sort.txt create mode 100644 tests/fixtures/present_status_output_sorted.txt create mode 100644 tests/test_encoders.py diff --git a/org_status/encoders/gitman.py b/org_status/encoders/gitman.py index 12eea2f..dde2ea0 100644 --- a/org_status/encoders/gitman.py +++ b/org_status/encoders/gitman.py @@ -1,3 +1,8 @@ +import os +import subprocess +import tempfile +import datetime + import yaml from giturlparse import parse @@ -14,6 +19,6 @@ def convert_repo_list_to_format(self, repos): name = parse(repo.web_url).repo yml_data['sources'].append({'name': name, 'repo': repo.web_url, - 'rev': 'master'}) - + 'rev': 'master', + }) return yaml.dump(yml_data, default_flow_style=False) diff --git a/org_status/org_hosts/__init__.py b/org_status/org_hosts/__init__.py index 473dfcb..8844cef 100644 --- a/org_status/org_hosts/__init__.py +++ b/org_status/org_hosts/__init__.py @@ -33,9 +33,10 @@ def repositories(self): class RepoStatus: - def __init__(self, repo_url, repo_status): + def __init__(self, repo_url, repo_status, last_commit_date): self.repo_url = repo_url self.repo_status = repo_status + self.last_commit_date = last_commit_date def get_all_supported_hosts(): diff --git a/org_status/org_hosts/github.py b/org_status/org_hosts/github.py index 65a5f1d..15653d3 100644 --- a/org_status/org_hosts/github.py +++ b/org_status/org_hosts/github.py @@ -1,4 +1,5 @@ import json +from types import MethodType import requests from IGitt.GitHub.GitHub import GitHubToken @@ -21,6 +22,9 @@ def __init__(self, token, group, **kargs): self._token = GitHubToken(token) self._org = GitHubOrganization(self._token, self._group) + for repo in self._org.repositories: + repo.get_last_commit_date = MethodType(_get_last_commit_date, repo) + self._status_provider = [] for i in enumerate(self.StatusProvider): self._status_provider.append(self.StatusProvider[i[0]](self._group)) @@ -56,8 +60,14 @@ def process_repository(self, repo, branch='master'): elif Status.ERROR in repo_status: repo_status = Status.ERROR - return RepoStatus(repo.web_url, repo_status) + last_commit_date = repo.get_last_commit_date() + + return RepoStatus(repo.web_url, repo_status, last_commit_date) @property def repositories(self): return self._org.repositories + + +def _get_last_commit_date(self): + return self.data._data['pushed_at'] diff --git a/org_status/org_hosts/gitlab.py b/org_status/org_hosts/gitlab.py index 65ac0e9..200307f 100644 --- a/org_status/org_hosts/gitlab.py +++ b/org_status/org_hosts/gitlab.py @@ -1,4 +1,5 @@ import json +from types import MethodType import requests from IGitt.GitLab.GitLab import GitLabPrivateToken @@ -22,6 +23,9 @@ def __init__(self, token, group, **kargs): self._token = GitLabPrivateToken(token) self._org = GitLabOrganization(self._token, self._group) + for repo in self._org.repositories: + repo.get_last_commit_date = MethodType(_get_last_commit_date, repo) + self._status_provider = self.StatusProvider(self._group) @classmethod @@ -39,8 +43,19 @@ def process_repository(self, repo, branch='master'): self.HostName, branch=branch) - return RepoStatus(repo.web_url, repo_status) + last_commit_date = repo.get_last_commit_date() + + return RepoStatus(repo.web_url, repo_status, last_commit_date) @property def repositories(self): return self._org.repositories + + +def _get_last_commit_date(self): + repo_commit_dates = [''] + for commit in self.commits: + repo_commit_dates.append(commit.data._data['committed_date']) + repo_commit_dates.sort() + + return repo_commit_dates[-1] diff --git a/org_status/org_status.py b/org_status/org_status.py index a9e6f52..1b545a9 100644 --- a/org_status/org_status.py +++ b/org_status/org_status.py @@ -47,10 +47,13 @@ def aggregate_org_status(org_host, threads=2): return pool.map(org_host.process_repository, org_host.repositories) -def present_status(statuses, no_color): +def present_status(statuses, no_color, sort): color = (lambda l, *_: l) if no_color else colored r_pass, r_fail, r_unknown, r_error = 0, 0, 0, 0 + if sort: + statuses.sort(key=lambda status: status.last_commit_date, reverse=True) + for status in statuses: repo_status = status.repo_status or Status.UNDETERMINED status_text = repo_status.value @@ -68,8 +71,13 @@ def present_status(statuses, no_color): r_unknown += 1 continue - print('{repo}: {status}'.format( - repo=status.repo_url, status=status_text)) + date_text = color(status.last_commit_date, 'blue') + + print('{repo}: {status} (last commit: {date})'.format( + repo=status.repo_url, + status=status_text, + date=date_text, + )) print('{} Passing, {} Failing, {} Error, {} Unknown ' 'of {} Repositories'.format( @@ -85,6 +93,7 @@ def get_argument_parser(): parser.add_argument('--hosts-only', '-o', action='store_true') parser.add_argument('--skip-host-checks', action='store_true') parser.add_argument('--export-repos', type=str) + parser.add_argument('--sort-by-last-commit', action='store_true') parser.add_argument('--format', type=str, default='gitman') parser.add_argument('--check-providers-only', action='store_true') @@ -196,7 +205,7 @@ def main(): continue org_status = aggregate_org_status(org_host, threads=args.threads) - present_status(org_status, args.no_color) + present_status(org_status, args.no_color, args.sort_by_last_commit) if args.export_repos: export_data = encode_repo_list(all_repositories, args.format, styled) diff --git a/requirements.txt b/requirements.txt index 5ee0a6b..630f198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ git+https://gitlab.com/gitmate/open-source/IGitt.git#egg=IGitt requests termcolor giturlparse +pyyaml +gitman diff --git a/tests/fixtures/present_status_output_no_sort.txt b/tests/fixtures/present_status_output_no_sort.txt new file mode 100644 index 0000000..d236b21 --- /dev/null +++ b/tests/fixtures/present_status_output_no_sort.txt @@ -0,0 +1,4 @@ +https://github.com/foo/r0: passing (last commit: 2018-11-05T08:20:37Z) +https://github.com/foo/r1: passing (last commit: 2018-11-02T17:31:46Z) +https://github.com/foo/r2: passing (last commit: 2018-11-06T02:27:42Z) +0 Passing, 0 Failing, 0 Error, 0 Unknown of 3 Repositories \ No newline at end of file diff --git a/tests/fixtures/present_status_output_sorted.txt b/tests/fixtures/present_status_output_sorted.txt new file mode 100644 index 0000000..1426b7d --- /dev/null +++ b/tests/fixtures/present_status_output_sorted.txt @@ -0,0 +1,4 @@ +https://github.com/foo/r2: passing (last commit: 2018-11-06T02:27:42Z) +https://github.com/foo/r0: passing (last commit: 2018-11-05T08:20:37Z) +https://github.com/foo/r1: passing (last commit: 2018-11-02T17:31:46Z) +0 Passing, 0 Failing, 0 Error, 0 Unknown of 3 Repositories \ No newline at end of file diff --git a/tests/test_encoders.py b/tests/test_encoders.py new file mode 100644 index 0000000..e48783a --- /dev/null +++ b/tests/test_encoders.py @@ -0,0 +1,13 @@ +from unittest import mock + +from org_status.encoders.gitman import GitManEncoder + + +def test_get_last_commit_date(): + encoder = GitManEncoder() + with mock.patch('org_status.encoders.gitman.clone'): + with mock.patch('org_status.encoders.gitman.os.chdir'): + with mock.patch('subprocess.check_output', + return_value='1542514751'): + output = encoder.get_last_commit_date(mock.Mock(), 'foo') + assert output == 1542514751 diff --git a/tests/test_main.py b/tests/test_main.py index fb39178..b18b72b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,6 @@ import pytest from unittest import mock +from io import StringIO from argparse import ArgumentParser @@ -7,7 +8,9 @@ generate_fetch_jobs, get_host_token, get_supported_status_providers, - get_status_provider_statuses) + get_status_provider_statuses, + present_status, + ) from org_status.org_hosts import get_all_supported_hosts @@ -56,3 +59,39 @@ def test_get_status_provider_statuses(): expected_status_provider_statuses.add((status_provider, status)) assert actual_status_provider_statuses == expected_status_provider_statuses + + +class MockRepoStatus(): + def __init__(self, value): + self.value = value + + +class MockStatus(): + def __init__(self, repo_url, repo_status, last_commit_date): + self.repo_url = repo_url + self.repo_status = MockRepoStatus(repo_status) + self.last_commit_date = last_commit_date + + +def test_present_status(): + statuses = [MockStatus('https://github.com/foo/r0', + 'passing', '2018-11-05T08:20:37Z'), + MockStatus('https://github.com/foo/r1', + 'passing', '2018-11-02T17:31:46Z'), + MockStatus('https://github.com/foo/r2', + 'passing', '2018-11-06T02:27:42Z')] + + with mock.patch('sys.stdout', new=StringIO()) as output: + present_status(statuses, True, False) + actual = output.getvalue().strip() + with open('tests/fixtures/present_status_output_no_sort.txt', 'r') as f: + expected = f.read() + assert actual == expected + + + with mock.patch('sys.stdout', new=StringIO()) as output: + present_status(statuses, True, True) + actual = output.getvalue().strip() + with open('tests/fixtures/present_status_output_sorted.txt', 'r') as f: + expected = f.read() + assert actual == expected