gh_favs

#!/usr/bin/python
"""gh_favs - Github Favorites

A small script to clone and update all your watched GitHub projects

(C) 2011 Manuel Strehl, some rights reserved

    This code is online: https://github.com/Boldewyn/gh_favs


Conflict strategies:

    If you watch several forks of a project, the name of the forks is
    identical. To come around this issue when cloning all of them, there
    are several strategies, controlled by the --strategy flag:

    * subfolders: create one folder for each Github user and clone her
      repositories there
    * prefix: prefix double names with the corresponding user name:
      Abba's charts.git and Zappa's charts.git would end up as
      Abba-charts and Zappa-charts. Note that this is not future-proof.
      If you start watching a fork, the originally cloned repository
      will not be renamed
    * none: first come, first serve. Only the first of identical
      projects will be cloned / updated. All others will be ignored.


License:

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see
    <http://www.gnu.org/licenses/>."""


__version__ = '3.2'


import urllib2
try:
    import json
except ImportError:
    import simplejson as json
import os
import subprocess
import logging
import sys
try:
    import argparse
except ImportError:
    argparse = None


logger = logging.getLogger('github.favs')


def fetch_favs(user, ignore=[], add_own=False, quiet=True, page=1):
    """Fetch the list of watched repos for a user from GitHub

    Own repos are excluded per default. Set add_own to True to add them."""

    if not quiet:
        logger.info('Fetch page %s' % page)
    try:
        r = urllib2.urlopen('https://api.github.com/users/%s/watched?page=%s' %
                            (user, page))
    except urllib2.URLError:
        logger.critical('Request failed. Are you online?')
        raise ValueError('Request failed. Are you online?')
    favs = []

    if r.getcode() != 200:
        logger.critical('Request failed with %s' % r.status_code)
        raise ValueError('Request failed with %s' % r.status_code)

    ls = json.loads(r.read())
    for item in ls:
        if (item['name'] not in ignore and
            '%s/%s' % (item['owner']['login'], item['name']) not in ignore and
            (add_own or item['owner']['login'] != user)):
            favs.append([item['name'], item['clone_url'],
                         item['owner']['login']])

    link = r.headers.getheader('Link')
    if 'rel="next"' in link:
        favs.extend(fetch_favs(user, ignore, add_own, quiet, page+1))
    return favs


def clone_favs(favs, target, quiet=True, with_docs=True):
    """Git-clone a list of repositories

    favs is a list of 2-tuples of name and remote URL
    target is the directory where to clone to
    quiet suppresses most Git status messages"""

    i = 0
    logger.info('Fetch repositories to %s' % target)

    target = os.path.abspath(target)

    qflag = False
    if quiet:
        qflag = '-q'

    def _call(a):
        logger.debug(' '.join(filter(None, a)))
        stdout, stderr = subprocess.Popen(filter(None, a),
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT).communicate()
        if quiet:
            logger.debug(stdout)
        else:
            logger.info(stdout)
        if stderr:
            logger.info(stderr)

    for fav in favs:
        i += 1
        os.chdir(target)

        if os.path.isdir('%s/.git' % fav[0]):
            logger.info('Update %s (%s of %s)' % (fav[0], i, len(favs)))
            os.chdir(fav[0])
            _call(['git', 'checkout', qflag, 'master'])
            _call(['git', 'pull', qflag, 'origin', 'master'])
        elif (os.path.isdir(fav[0]) and len(os.listdir(fav[0])) > 0):
            logger.critical('%s is not a Git repository!' % fav[0])
            continue
        else:
            logger.info('Clone %s (%s of %s)' % (fav[0], i, len(favs)))
            if not os.path.isdir(fav[0]):
                os.makedirs(fav[0])
            os.chdir(fav[0])
            _call(['git', 'clone', qflag, '--recursive', fav[1], '.'])

        if with_docs:
            logger.debug('check out docs / website')
            br = subprocess.check_output(['git', 'branch', '-r'])
            if 'origin/gh-pages' in br:
                br = subprocess.check_output(['git', 'branch'])
                if 'gh-pages' in br:
                    _call(['git', 'checkout', qflag, 'gh-pages'])
                    _call(['git', 'pull', qflag, 'origin', 'gh-pages'])
                else:
                    _call(['git', 'checkout', qflag, '-b', 'gh-pages',
                           'origin/gh-pages'])
                _call(['git', 'checkout', qflag, 'master'])


def resolve_fav_conflicts(favs, strategy=None):
    """resolve name conflicts, e.g., when forks are watched"""
    if strategy == 'prefix':
        doubles = {}
        for fav in favs:
            if fav[0] in doubles:
                doubles[fav[0]] += 1
            else:
                doubles[fav[0]] = 1
        for k, v in doubles.items():
            if v == 1:
                del doubles[k]
        for i, fav in enumerate(favs):
            if fav[0] in doubles:
                favs[i] = (fav[2] + '-' + fav[0], fav[1], fav[2])
    elif strategy == 'subfolders':
        for i, fav in enumerate(favs):
            favs[i] = (fav[2] + '/' + fav[0], fav[1], fav[2])
    return favs


def _get_args(argv=None):
    """Get the CLI args"""
    global __version__, __doc__
    if argparse is not None:
        parser = argparse.ArgumentParser(prog='gh_favs',
                formatter_class=argparse.RawDescriptionHelpFormatter,
                description='Clone and update watched repositories ' +
                            'on Github.',
                epilog='About gh_favs:\n\n    ' +
                       '\n'.join(__doc__.split('\n')[4:]))
        parser.add_argument('user', metavar='USER',
                help='Github user name')
        parser.add_argument('-t', '--target', default='.',
                metavar='DIR',
                help='directory where the clones live, default is ' +
                     '"%(default)s"')
        parser.add_argument('-i', '--ignore', nargs='+', metavar='NAME',
                default=[],
                help='repository names to be ignored, can be followed ' +
                     'by multiple items')
        parser.add_argument('-o', '--add_own', action='store_true',
                help='clone own repositories, too')
        parser.add_argument('-n', '--no_docs', action='store_true',
                help='don\'t fetch origin/gh_pages')
        parser.add_argument('-s', '--strategy', choices=['subfolders',
                'prefix', 'none'], default='subfolders',
                help='how to handle name clashes, default is ' +
                     '"%(default)s"')
        parser.add_argument('-v', '--verbose', action='store_true',
                help='include Git status messages')
        parser.add_argument('--version', action='version',
                version='%(prog)s ' + __version__)
        args = parser.parse_args(argv)
    else:
        # argparse not installed: let's fall back to minimal
        # functionality
        class Dummy(object):
            pass
        args = Dummy()
        if len(sys.argv) not in [2,4]:
            print 'usage: gh_favs [-t DIR] USER'
            exit()
        if len(sys.argv) == 2 and sys.argv[1][0] != '-':
            args.user = sys.argv[1]
            args.target = '.'
        elif len(sys.argv) == 4 and sys.argv[1] in ['--target', '-t']:
            args.user = sys.argv[3]
            args.target = sys.argv[2]
        else:
            print 'usage: gh_favs [-t DIR] USER'
            exit()
        args.ignore = []
        args.verbose = False
        args.add_own = False
        args.no_docs = False
        args.strategy = 'subfolders'

    if not os.path.isdir(args.target):
        logger.debug('Create directory %s' % args.target)
        os.makedirs(args.target)
    return (args.target, args.user, args.ignore, args.verbose, args.add_own,
            args.no_docs, args.strategy)


if __name__ == '__main__':
    logging.basicConfig(format='%(message)s')
    logger.setLevel(logging.INFO)
    argv = sys.argv[1:]
    if os.path.isfile(os.path.expanduser('~/.gh_favsrc')):
        addf = open(os.path.expanduser('~/.gh_favsrc'))
        add = filter(None, addf.read().split("\n"))
        addf.close()
        argv = add + argv
    target, user, ignore, verbose, add_own, no_docs, strategy = _get_args(argv)
    try:
        clone_favs(
            resolve_fav_conflicts(
                fetch_favs(user, ignore, add_own=add_own, quiet=(not verbose)),
                strategy),
            target, quiet=(not verbose), with_docs=(not no_docs))
    except ValueError, e:
        sys.stderr.write('An error occured: %s\n' % e)
        sys.stderr.flush()