Added all kinds of stuff but probably broke github auth

app.yaml

@@ -6,6 +6,8 @@ skip_files:
 - ^.*\.json
 - ^dev/.*
 - ^\..*
+- ^lib/simplejson/tests/.*
+- ^.*\.pyc
 
 handlers:
 - url: /.*

config.py

@@ -0,0 +1,8 @@
+import yaml, os, logging
+
+# Load config
+config = yaml.load(open('config.yaml','r'))
+
+# Enforce some defaults
+config['webapp2_extras.auth'] = { 'session_backend': 'memcache' }
+

github.py

@@ -1,157 +1,121 @@
+import webapp2
+import sys
+import urllib, urlparse
 from urllib2 import Request, urlopen, URLError, HTTPError
 import base64, json, logging, datetime, time
 from google.appengine.ext import deferred
 from google.appengine.ext import vendor
 vendor.add('lib') # uritemplate is vendored
 import uritemplate
-
-from model import Issue
-
-githubUrl = 'https://api.github.com/repos'
-zenhubUrl = 'https://api.zenhub.io/p1/repositories'
-
-def getZenhub(repo, url):
-    if not repo.auth.zenhubToken: return
-    if zenhubUrl not in url:
-        url = '%s/%s%s' % (zenhubUrl, repo.data['id'], url)
-    logging.info('zenhub GET ' + url)
-    request = Request(url)
-    request.add_header('X-Authentication-Token', repo.auth.zenhubToken)
-    try:
+import random
+from model import Issue, Repo
+from config import config
+
+githubUrl = 'https://api.github.com'
+
+class Github:
+    def __init__(self, me):
+        self.me = me
+
+    def get(self, url):
+        logging.info('github GET ' + url)
+        request = Request(url)
+        # Add credentials as we have them
+        if 'paToken' in self.me:
+            #logging.info("Adding personal access token from session")
+            encoded = base64.b64encode(':%s' % self.me['paToken'])
+            request.add_header('Authorization', 'Basic ' + encoded)
+        elif 'aToken' in self.me:
+            request.add_header('Authorization', 'token ' + self.me['aToken'])
         return json.load(urlopen(request))
-    except HTTPError, err:
-        logging.info('Uh oh: %s', err)
-
-def getGithub(repo, url): 
-    if githubUrl not in url:
-        url = githubUrl + '/' + repo.name + url
-    logging.info('github GET ' + url)
-    request = Request(url)
-    encoded = base64.b64encode('%s:%s' % (repo.auth.githubUser, repo.auth.githubToken))
-    request.add_header('Authorization', 'Basic ' + encoded)
-    return json.load(urlopen(request))
-
-def getGithubAll(repo, url):
-    pageSize = 50
-    pageNumber = 0
-    items = []
-    more = True
-    if '?' not in url: 
-       url += '?'
-    else:
-       url += '&'
-    while more:
-        fetchUrl = url + 'per_page=%d&page=%d' % (pageSize, pageNumber)
-        page = getGithub(repo, fetchUrl)
-        items.extend(page)
-        pageNumber += 1
-        more = len(page) == pageSize
-    return items
-
-def getIssueEvents(repo, issue):
-    """Get events for this issue from Github"""
-    events = getGithubAll(repo, issue.github['events_url'])
-    logging.info('Found %d events for #%s' % (len(events),issue.number))
-    issue.github['events'] = events
-
-def getAllIssues(repo):
-    # Update the repo with content
-    repo.data = getGithub(repo, '')
-    repo.put()
-
-    issueUrl = uritemplate.expand(repo.data['issues_url'], {})
-    issueUrl += '?state=all'
-    issues = getGithubAll(repo, issueUrl)
-    for issueData in issues:
-        issue = Issue(repo = repo.key, github = issueData, number = issueData['number'])
-        getIssueEvents(repo, issue)
-        issue.upsert()
-
-    # Also load all zenhub issues. This may take a while
-    getZenhubIssues(repo.key)
 
-def getZenhubIssues(repoKey, keys = None):
-    repo = repoKey.get()   
-    if not repo or not repo.auth.zenhubToken: return
-    if keys is None:
-        keys = map(lambda i: i.key, repo.issues())
-    logging.info("Getting zenhub issues from list (%s)" % len(keys))
-    while keys:
-        key = keys[0]
-        issue = key.get()
-        if getZenhubIssue(repo, issue):
-            keys.pop(0)
+    def getAll(self, url):
+        """Fetches pages of items"""
+        pageSize = 50
+        pageNumber = 0
+        items = []
+        more = True
+        if '?' not in url: 
+           url += '?'
         else:
-            logging.info("Deferring at %s" % issue.number)
-            deferred.defer(getZenhubIssues, repoKey, keys, _countdown=20)
-            break
-    return True
-
-def getZenhubIssue(repo, issue):
-    now = datetime.datetime.now()
-    if hasattr(issue, 'zenhubUpdate') and issue.zenhubUpdate and (now - issue.zenhubUpdate).total_seconds() < 60:
-        return True
-    zenhub = getZenhub(repo, '/issues/%s' % issue.number)
-    if not zenhub: return False
-    zenhub['events'] = getZenhub(repo, '/issues/%s/events' % issue.number)
-    if zenhub['events'] is None: return False
-    issue.zenhub = zenhub
-    issue.zenhubUpdate = now
-    issue.upsert()
-    logging.info("Updated #%s with %d zenhub events" % (issue.number, len(zenhub['events'])))
-    return True
-
-def getUpdatedIssues(repo, recent):
-    """Return issue numbers for issues updated in github since 'recent'"""
-    refresh = set()
-    latestEventTime = None
-    for page in range(0, 10):
-        url = repo.data['events_url'] + '?page=%d' % page
-        events = getGithub(repo, url)        
-        for event in events:
-            latestEventTime = max(latestEventTime, event['created_at'])
-            if event['created_at'] <= recent:
-                # We are done, fetch no more
-                return refresh, latestEventTime
-            elif 'issue' in event['payload']:
-                number = event['payload']['issue']['number']
-                refresh.add(number)
-    return refresh, latestEventTime
-
-def findIssue(issues, number):
-    """Return the first issue in issues with the same number"""
-    for issue in issues:
-        if issue.number == number:
-            return issue
-
-def syncIssues(repo):
-    # Load all issues from store and find the most recent update time
-    recent = None
-    for issue in repo.issues():
-        logging.info("Comparing %s" % issue.githubUpdate)
-        recent = max(recent, issue.githubUpdate)
-
-    logging.info("Getting updated issues since %s" % recent)
-    toRefresh, recent = getUpdatedIssues(repo, recent)
-    if not toRefresh:
-        logging.info("No items to refresh")
-        return
-
-    # For each issue that potentially changed, refresh it
-    keysToRefresh = []
-    for number in toRefresh:
-        # if the repo suddenly disappears, quit
-        if not repo.key.get(): return
-        issue = repo.issue(number)
-        if not issue:
-            issue = Issue(repo = repo.key, number = number)
+           url += '&'
+        while more:
+            page = self.get(url + 'per_page=%d&page=%d' % (pageSize, pageNumber))
+            items.extend(page)
+            pageNumber += 1
+            more = len(page) == pageSize
+        return items
+
+    def repos(self):
+        repos = self.get(self.me['repos_url'])
+        subs = self.get(self.me['subscriptions_url'])
+        repoNames = map(lambda r: r['full_name'], repos)
+        for sub in subs:
+            if sub['full_name'] not in repoNames:
+                repos.append(sub)
+        return repos
+
+    def repo(self, name):
+        return self.get(githubUrl + '/repos/' + name)
+
+    def user(self):
+        """Get user info from github and store it in the session"""
+        return self.get('%s/user' % githubUrl)
+
+    def issue(self, repo, number, oldData):
+        """Return github issue and event data"""
         url = uritemplate.expand(repo.data['issues_url'], { 'number': number })
-        issue.github = getGithub(repo, url) 
-        getIssueEvents(repo, issue)
-        issue.githubUpdate = recent
-        logging.info("Upserting issue %s" % issue.number)
-        issue.upsert()
-        keysToRefresh.append(issue.key)
-
-    # Go get zenhub updates (separately so it can be metered safely)
-    getZenhubIssues(repo.key, keysToRefresh)
+        issue = self.get(url)
+        # TODO: only get events up to the most recent in oldData?
+        issue['events'] = self.getAll(issue['events_url'])
+        logging.info('Found %d events for #%s' % (len(issue['events']), number))
+        return issue
+
+    def syncIssues(self, repoKey):
+        repo = repoKey.get()
+
+        issues = repo.issues()
+        if issues:
+            issueTime = max(map(lambda x: x.githubUpdate, issues))
+        else:
+            issueTime = None
+
+        logging.info("Getting updated issues since %s" % issueTime)
+        newNumbers, updateTime = self.newIssueNumbers(repo, issueTime)
+
+        # Refresh each new issue
+        for number in newNumbers:
+            # if the repo suddenly disappears, quit
+            if not repoKey.get(): return
+            issue = repo.issue(number)
+            if not issue:
+                issue = Issue(repo = repo.key, number = number)
+            issue.github = self.issue(repo, number, issue.github)
+            issue.githubUpdate = updateTime
+            issue.zenhubUpdate = None  # github update means zenhub update
+            logging.info("Upserting issue %s" % issue.number)
+            issue.upsert()
+
+        #TODO: Go get zenhub updates (separately so it can be metered safely)?
+        #getZenhubIssues(repo.key, keysToRefresh)
+
+    def newIssueNumbers(self, repo, until):
+        """Return issues that changed after issueTime, along with most recent update time"""
+        numbers = set()
+        latestTime = None
+        for page in range(0, 10):
+            url = repo.data['events_url'] + '?page=%d' % page
+            for event in self.get(url):
+                latestTime = max(latestTime, event['created_at'])
+                if event['created_at'] <= until:
+                    # We are done, fetch no more
+                    return numbers, latestTime
+                elif 'issue' in event['payload']:
+                    numbers.add(event['payload']['issue']['number'])
+        return list(numbers), latestTime
+
+def syncIssues(user, repoKey):
+    try:
+        Github(user).syncIssues(repoKey)
+    except:
+        logging.exception('Failed to sync Github issues')

html/base.html

@@ -1,6 +1,14 @@
 <!DOCTYPE html>
 <html>
 <body>
+{% if user %}
+<p>Welcome {{ user.name }} (<a href="/logout">Logout</a>)</p>
+{% else %}
+<form method="get">
+  <div>Github Personal Access Token: <input type="text" name="paToken" placeholder="github token"></div>
+  <div><input type="submit" value="Login"></div>
+</form>
+{% endif %}
 {% block body %}{% endblock %}
 </body>
 </html>

html/index.html

@@ -1,23 +1,14 @@
-<html>
-<body>
-{% if repos %}
+{% extends "base.html" %}
+{% block body %}
 <h1>Repositories</h1>
+{% if repos %}
 <ul>
   {% for repo in repos %}
 <li><a href="{{repo.url()}}">{{repo.name}}</a></li>
   {% endfor %}
 </ul>
 {% else %}
-<p>No repos found</p>
+<p>No repos added</p>
 {% endif %}
-
-<h1>Add a repo</h1>
-<form action="/addRepo" method="post">
-  <div>Github Repository Name: <input type="text" name="repoName" placeholder="org/repo"></div>
-  <div>Github User Name: <input type="text" name="githubUser" placeholder="username"></div>
-  <div>Github Personal Access Token: <input type="text" name="githubToken" placeholder="github token"></div>
-  <div>Zenhub API Token: <input type="text" name="zenhubToken" placeholder="zenhub token"></div>
-  <div><input type="submit" value="Add"></div>
-</form>
-</body>
-</html>
+<a href='/repo/add'>Add repository</a>
+{% endblock %}

html/repo.html

@@ -14,7 +14,8 @@ <h1><a href="/">Home</a> &rarr; {{ repo.name }}</h1>
 {% else %}
 <p>No issues</p>
 {% endif %}
-<form action="{{ repo.url() + '/sync' }}" method="post">
+<form method="get">
+  <input type="hidden" name="s" value="1"/>
   <div><input type="submit" value="Sync this repo"></div>
 </form>
 

html/repoAdd.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block body %}
+<h1><a href="/">Home</a> &rarr; Add Repository</h1>
+<form action="/repo/add" method="post">
+<div>Github Repository: <select name="repo">
+{% for repo in repos %}
+<option value='{{repo.full_name}}'>{{repo.full_name}} ({{repo.open_issues}} open issues)</option>
+{% endfor %}
+  </select>
+  </div>
+<div>Zenhub API Token: <input type="text" name="zenhubToken" placeholder="zenhub token"></div>
+<div><input type="submit" value="Add Repository"/>
+</form>
+{% endblock %}