Fixes #15 and #5 (use ?f=1 to recommence full sync)

gladed · Mar 28, 2016 · 5a43ca5 · 5a43ca5
1 parent 873e49d
commit 5a43ca5
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 82 deletions.
diff --git a/github.py b/github.py
@@ -59,7 +59,7 @@ def user(self):
         """Get user info from github and store it in the session"""
         return self.get('%s/user' % githubUrl)
 
-    def issue(self, repo, number, oldData):
+    def issue(self, repo, number, oldData=None):
         """Return github issue and event data"""
         url = uritemplate.expand(repo.data['issues_url'], { 'number': number })
         issue = self.get(url)
@@ -68,32 +68,44 @@ def issue(self, repo, number, oldData):
         logging.info('Found %d events for #%s' % (len(issue['events']), number))
         return issue
 
-    def syncIssues(self, repoKey):
-        repo = repoKey.get()
+    def storeIssue(self, repo, issueData, updateTime = None):
+        """Update database with issue data"""
+        number = issueData['number']
+        issue = repo.issue(number)
+        if 'modified_at' in issueData and not updateTime:
+            updateTime = issueData['modified_at']
+        if not issue:
+            issue = Issue(repo = repo.key, number = number)
+        issue.github = self.issue(repo, number, issue.github)
+        issue.githubUpdate = updateTime
+        issue.zenhubUpdate = None  # mark for Zenhub update
+        logging.info("Upserting issue %s" % issue.number)
+        issue.upsert()
+
+    def issues(self, repo):
+        """Fetch all issues from github"""
+
+    def syncIssues(self, repo, deep=False):
 
         issues = repo.issues()
         if issues:
             issueTime = max(map(lambda x: x.githubUpdate, issues))
         else:
             issueTime = None
 
-        logging.info("Getting updated issues since %s" % issueTime)
-        newNumbers, updateTime = self.newIssueNumbers(repo, issueTime)
-
-        # Refresh each new issue
-        for number in newNumbers:
-            # if the repo suddenly disappears, quit
-            if not repoKey.get(): return
-            issue = repo.issue(number)
-            if not issue:
-                issue = Issue(repo = repo.key, number = number)
-            issue.github = self.issue(repo, number, issue.github)
-            issue.githubUpdate = updateTime
-            issue.zenhubUpdate = None  # mark for Zenhub update
-            logging.info("Upserting issue %s" % issue.number)
-            issue.upsert()
+        if deep:
+            logging.info("Getting all issues")
+            url = uritemplate.expand(repo.data['issues_url'], { }) + '?state=all'
+            for issue in self.getAll(url):
+                self.storeIssue(repo, issue)
+        else:
+            logging.info("Getting updated issues since %s" % issueTime)
+            numbers, updateTime = self.newIssueNumbers(repo, issueTime)
+            for number in numbers:
+                repo.key.get() # Make sure repo is there or throw
+                self.storeIssue(repo, self.issue(repo, number), updateTime)
 
-        # After a github sync, start a zenhub sync
+        # When github sync is successful, launch a follow-up zenhub sync
         zenhub.syncIssues(repo.key)
 
     def newIssueNumbers(self, repo, until):
@@ -111,8 +123,18 @@ def newIssueNumbers(self, repo, until):
                     numbers.add(event['payload']['issue']['number'])
         return list(numbers), latestTime
 
-def syncIssues(user, repoKey):
+def syncIssues(user, repoKey, deep=False):
     try:
-        Github(user).syncIssues(repoKey)
+        repo = repoKey.get()
+        repo.syncing = True
+        repo.put()
+        Github(user).syncIssues(repo, deep)
+        # Do not turn off sync bit, zenhub will do that.
     except:
         logging.exception('Failed to sync Github issues')
+        try:
+            repo = repoKey.get()
+            repo.syncing = False
+            repo.put()
+        except:
+            pass
diff --git a/html/base.html b/html/base.html
@@ -2,7 +2,7 @@
 <html>
 <body>
 {% if user %}
-<p>Welcome {{ user.name }} (<a href="/logout">Logout</a>)</p>
+<p>Welcome {{ user.name }} ({{ user.login }}). <a href="/logout">Logout</a></p>
 {% else %}
 <form action="/login" method="get">
   <!--<div>Github Personal Access Token: <input type="text" name="paToken" placeholder="github token"></div>-->

diff --git a/html/repo.html b/html/repo.html
@@ -16,7 +16,7 @@ <h1><a href="/">Home</a> &rarr; {{ repo.name }}</h1>
 {% endif %}
 <form method="get">
   <input type="hidden" name="s" value="1"/>
-  <div><input type="submit" value="Sync this repo"></div>
+  <div><input type="submit" {% if repo.syncing %}disabled {% endif %}value="Sync this repo"></div>
 </form>
 
 <form action="{{ repo.url() + '/delete' }}" method="post">

diff --git a/model.py b/model.py
@@ -7,21 +7,9 @@ class Auth(ndb.Model):
     zenhubToken = ndb.StringProperty()
 
 class Repo(ndb.Model):
-    @classmethod
-    def create(cls, user):
-        repo = Repo(parent = userKey(user))
-        return repo
-
-    @classmethod
-    def get(cls, user, id):
-        return super(Repo, cls).get_by_id(int(id), parent=userKey(user))
-
-    @classmethod
-    def repos(cls, user):
-        return Repo.query(ancestor = userKey(user)).fetch()
-
     auth = ndb.StructuredProperty(Auth)
     name = ndb.StringProperty(indexed=True)
+    syncing = ndb.BooleanProperty()
     data = ndb.JsonProperty()
     updated = ndb.DateTimeProperty(auto_now=True)
 
@@ -43,6 +31,19 @@ def delete(self):
             issue.key.delete()
         self.key.delete()
 
+    @classmethod
+    def create(cls, user):
+        repo = Repo(parent = userKey(user))
+        return repo
+
+    @classmethod
+    def get(cls, user, id):
+        return super(Repo, cls).get_by_id(int(id), parent=userKey(user))
+
+    @classmethod
+    def repos(cls, user):
+        return Repo.query(ancestor = userKey(user)).fetch()
+
 class Issue(ndb.Model):
     repo = ndb.KeyProperty(kind=Repo)
     number = ndb.IntegerProperty(indexed=True)

diff --git a/zenchart.py b/zenchart.py
@@ -95,7 +95,7 @@ def get(self, id):
             deferred.defer(github.syncIssues, user, repo.key)
             self.redirect(repo.url())
         elif self.request.get('f'):
-            deferred.defer(github.getAllIssues, user, repo.key)
+            deferred.defer(github.syncIssues, user, repo.key, True)
             self.redirect(repo.url())
         else:
             issues = sorted(repo.issues(), key=lambda x: x.number, reverse=True)
@@ -114,31 +114,11 @@ def post(self, id):
         else:
             self.response.set_status(404)
 
-class RepoSyncPage(webapp2.RequestHandler):
-    def post(self, id):
-        repo = Repo.get(id)
-        if not repo:
-            self.response.set_status(404)
-        else:
-            deferred.defer(github.syncIssues, repo)
-            #deferred.defer(github.getAllIssues, repo)
-            #taskqueue.add(url=repo.url() + '/task/sync')
-            self.redirect(repo.url())
-
-class AddRepo(webapp2.RequestHandler):
-    def post(self):
-        name = self.request.get('repoName')
-        repo = Repo()
-        repo.name = name
-        repo.auth = Auth()
-        repo.auth.zenhubToken = self.request.get('zenhubToken')
-	repo.put()
-        deferred.defer(github.getAllIssues, repo)
-        self.redirect(repo.url())
 
-class IssuePage(webapp2.RequestHandler):
+class IssuePage(BaseHandler):
     def get(self, id, number):
-        repo = Repo.get(id)
+        user = self.user()
+        repo = Repo.get(user, id)
         if not repo:
             self.response.set_status(404)
         else:
@@ -150,23 +130,6 @@ def get(self, id, number):
                 self.response.headers['Content-Type'] = 'application/json'
                 self.response.out.write(json.dumps(full, indent=2))
 
-DEBUG = True
-
-class RepoTaskSync(webapp2.RequestHandler):
-    def post(self, id):
-        if not DEBUG and 'X-AppEngine-QueueName' not in self.request.headers:
-            self.response.set_status(400)
-        else:
-            repo = Repo.get(id)
-            if not repo:
-                self.response.set_status(404)
-            else:
-                logging.info("Sync Beginning")
-                #github.syncIssues(repo)
-                #deferred.defer(github.getAllIssues, repo)
-                deferred.defer(github.getZenhubIssues, repo.key)
-                self.response.write("OK")
-
 class LoginPage(BaseHandler):
     def get(self):
         self.session['return'] = self.request.get('r')
@@ -219,13 +182,10 @@ def get(self):
 app = webapp2.WSGIApplication([
     ('/', MainPage),
     ('/repo/add', RepoAddPage),
-    ('/addRepo', AddRepo),
     ('/login', LoginPage),
     ('/logout', LogoutPage),
     webapp2.Route(r'/repo/<id:\d+>', RepoPage),
     webapp2.Route(r'/repo/<id:\d+>/delete', RepoDeletePage),
-    webapp2.Route(r'/repo/<id:\d+>/sync', RepoSyncPage),
-    webapp2.Route(r'/repo/<id:\d+>/task/sync', RepoTaskSync),
     webapp2.Route(r'/repo/<id:\d+>/issue/<number:\d+>', IssuePage),
     ('/auth/github', GithubAuthPage),
 ],debug=True,config=config)
diff --git a/zenhub.py b/zenhub.py
@@ -17,8 +17,6 @@ def get(self, url):
         request = Request(url)
         request.add_header('X-Authentication-Token', self.repo.auth.zenhubToken)
         return json.load(urlopen(request))
-#        except HTTPError, err:
-#            logging.info('Uh oh: %s', err)
 
     def issue(self, number):
         """Load an issue with events from zenhub"""
@@ -37,6 +35,9 @@ def syncIssues(self):
 def syncIssues(repoKey):
     try:
         Zenhub(repoKey.get()).syncIssues()
+        repo = repoKey.get()
+        repo.syncing = False
+        repo.put()
     except HTTPError as e:
         if e.code == 403:
             logging.info('Zenhub returned 403, trying again after a while...')