Scorecard: Use Sanic, cache fully-shaped GitHub response

scorecard/Dockerfile

@@ -2,6 +2,7 @@ FROM continuumio/miniconda
 MAINTAINER Hail Team <[email protected]>
 
 COPY environment.yml .
+RUN apt-get update && apt-get install -y linux-headers-amd64 build-essential
 RUN conda env create scorecard -f environment.yml && \
     rm -f environment.yml && \
     rm -rf /home/root/.conda/pkgs/*

scorecard/environment.yml

@@ -1,9 +1,11 @@
 name: scorecard
 dependencies:
 - python=3.7
-- flask
-- flask-cors
 - humanize
 - pip
+- jinja2
 - pip:
     - PyGithub
+    - sanic
+    - sanic-cors
+    - ujson

scorecard/scorecard/scorecard.py

@@ -3,13 +3,22 @@
 import datetime
 import os
 import sys
-from flask import Flask, render_template, request, jsonify, abort, url_for
-from flask_cors import CORS
 from github import Github
 import random
 import threading
 import humanize
 import logging
+from sanic import Sanic
+from sanic.response import text, json, html
+from sanic_cors import CORS
+from jinja2 import Environment, PackageLoader, select_autoescape
+import ujson
+
+env = Environment(loader=PackageLoader('scorecard', 'templates/'),
+                  autoescape=select_autoescape(['html', 'xml', 'tpl']), enable_async=True)
+
+users_template = env.get_template('index.html')
+one_user_templ = env.get_template('user.html')
 
 fmt = logging.Formatter(
     # NB: no space after levename because WARNING is so long
@@ -55,50 +64,66 @@
     'cloudtools': 'Nealelab/cloudtools'
 }
 
-app = Flask('scorecard')
+app = Sanic(__name__)
 CORS(app, resources={r'/json/*': {'origins': '*'}})
 
-data = None
-timsetamp = None
+fav_path = os.path.join(os.path.dirname(__file__), 'static', 'favicon.ico')
+app.static('/favicon.ico', fav_path)
+
+########### Global variables that are modified in a separate thread ############
+# Must be only read, never written in parent thread, else need to use Lock()
+# http://effbot.org/zone/thread-synchronization.htm#synchronizing-access-to-shared-resources
+data=None
+users_data=None
+users_json=None
+timsetamp=None
+################################################################################
+
 
 @app.route('/')
-def index():
-    user_data, unassigned, urgent_issues, updated = get_users()
+async def index(request):
+    user_data, unassigned, urgent_issues=users_data
 
-    random_user = random.choice(users)
+    # Read timestamp as quickly as possible in case timestamp gets modified
+    # by forever_poll thread
+    cur_timestamp=timestamp
+    updated=humanize.naturaltime(
+        datetime.datetime.now() - datetime.timedelta(seconds=time.time() - cur_timestamp))
+
+    random_user=random.choice(users)
+
+    tmpl=await users_template.render_async(unassigned = unassigned,
+                                             user_data = user_data, urgent_issues = urgent_issues, random_user = random_user, updated = updated)
+    return html(tmpl)
 
-    return render_template('index.html', unassigned=unassigned,
-                           user_data=user_data, urgent_issues=urgent_issues, random_user=random_user, updated=updated)
 
 @app.route('/users/<user>')
-def html_get_user(user):
-    user_data, updated = get_user(user)
-    return render_template('user.html', user=user, user_data=user_data, updated=updated)
+async def html_get_user(request, user):
+    user_data, updated=get_user(user)
 
-@app.route('/json')
-def json_all_users():
-    user_data, unassigned, urgent_issues, updated = get_users()
+    tmpl=await one_user_templ.render_async(user = user, user_data = user_data, updated = updated)
+    return html(tmpl)
 
-    for issue in urgent_issues:
-        issue['timedelta'] = humanize.naturaltime(issue['timedelta'])
 
-    return jsonify(updated=updated, user_data=user_data, unassigned=unassigned, urgent_issues=urgent_issues)
+@app.route('/json')
+async def json_all_users(request):
+    return text(users_json)
+
 
 @app.route('/json/users/<user>')
-def json_user(user):
-    user_data, updated = get_user(user)
-    return jsonify(updated=updated, data=user_data)
+async def json_user(request, user):
+    user_data, updated=get_user(user)
+    return json({"updated": updated, "user_data": user_data})
+
 
 @app.route('/json/random')
-def json_random_user():
-    return jsonify(random.choice(users))
+async def json_random_user(request):
+    return text(random.choice(users))
 
-def get_users():
-    cur_data = data
-    cur_timestamp = timestamp
 
-    unassigned = []
-    user_data = collections.defaultdict(
+def get_and_cache_users(github_data):
+    unassigned=[]
+    user_data=collections.defaultdict(
         lambda: {'CHANGES_REQUESTED': [],
                  'NEEDS_REVIEW': [],
                  'ISSUES': []})
@@ -131,7 +156,7 @@ def add_issue(repo_name, issue):
             else:
                 d['ISSUES'].append(issue)
 
-    for repo_name, repo_data in cur_data.items():
+    for repo_name, repo_data in github_data.items():
         for pr in repo_data['prs']:
             if len(pr['assignees']) == 0:
                 unassigned.append(pr)
@@ -142,20 +167,22 @@ def add_issue(repo_name, issue):
         for issue in repo_data['issues']:
             add_issue(repo_name, issue)
 
-    list.sort(urgent_issues, key=lambda issue: issue['timedelta'], reverse=True)
+    list.sort(urgent_issues,
+              key = lambda issue: issue['timedelta'], reverse=True)
 
-    updated = humanize.naturaltime(
-        datetime.datetime.now() - datetime.timedelta(seconds = time.time() - cur_timestamp))
+    return (user_data, unassigned, urgent_issues)
 
-    return (user_data, unassigned, urgent_issues, updated)
 
 def get_user(user):
-    global data, timestamp
+    global data
 
     cur_data = data
     cur_timestamp = timestamp
 
-    user_data = {
+    updated = humanize.naturaltime(
+        datetime.datetime.now() - datetime.timedelta(seconds=time.time() - cur_timestamp))
+
+    user_data={
         'CHANGES_REQUESTED': [],
         'NEEDS_REVIEW': [],
         'FAILING': [],
@@ -181,8 +208,6 @@ def get_user(user):
             if user in issue['assignees']:
                 user_data['ISSUES'].append(issue)
 
-    updated = humanize.naturaltime(
-        datetime.datetime.now() - datetime.timedelta(seconds=time.time() - cur_timestamp))
     return (user_data, updated)
 
 
@@ -192,6 +217,7 @@ def get_id(repo_name, number):
     else:
         return f'{repo_name}/{number}'
 
+
 def get_pr_data(repo, repo_name, pr):
     assignees = [a.login for a in pr.assignees]
 
@@ -207,7 +233,8 @@ def get_pr_data(repo, repo_name, pr):
             break
         else:
             if review.state != 'COMMENTED':
-                log.warning(f'unknown review state {review.state} on review {review} in pr {pr}')
+                log.warning(
+                    f'unknown review state {review.state} on review {review} in pr {pr}')
 
     sha = pr.head.sha
     status = repo.get_commit(sha=sha).get_combined_status().state
@@ -223,6 +250,7 @@ def get_pr_data(repo, repo_name, pr):
         'status': status
     }
 
+
 def get_issue_data(repo_name, issue):
     assignees = [a.login for a in issue.assignees]
     return {
@@ -235,8 +263,9 @@ def get_issue_data(repo_name, issue):
         'created_at': issue.created_at
     }
 
+
 def update_data():
-    global data, timestamp
+    global data, timestamp, users_data, users_json
 
     log.info(f'rate_limit {github.get_rate_limit()}')
     log.info('start updating_data')
@@ -261,44 +290,52 @@ def update_data():
                 issue_data = get_issue_data(repo_name, issue)
                 new_data[repo_name]['issues'].append(issue_data)
 
-
     log.info('updating_data done')
 
-    now = time.time()
-
     data = new_data
-    timestamp = now
+    timestamp = time.time()
+    users_data = get_and_cache_users(new_data)
+    users_json = ujson.dumps(
+        {"user_data": users_data[0], "unassigned": users_data[1], "urgent_issues": users_data[2], "timestamp": timestamp})
+
 
 def poll():
     while True:
         time.sleep(180)
         update_data()
 
-update_data()
 
 def run_forever(target, *args, **kwargs):
-    # target should be a function
     target_name = target.__name__
+    expected_retry_interval_ms = 15 * 1000  # 15s
 
-    expected_retry_interval_ms = 15 * 1000 # 15s
     while True:
         start = time.time()
         try:
             log.info(f'run target {target_name}')
             target(*args, **kwargs)
             log.info(f'target {target_name} returned')
         except:
-            log.error(f'target {target_name} threw exception', exc_info=sys.exc_info())
+            log.error(f'target {target_name} threw exception',
+                      exc_info=sys.exc_info())
         end = time.time()
 
         run_time_ms = int((end - start) * 1000 + 0.5)
+
         t = random.randrange(expected_retry_interval_ms * 2) - run_time_ms
         if t > 0:
             log.debug(f'{target_name}: sleep {t}ms')
             time.sleep(t / 1000.0)
 
-poll_thread = threading.Thread(target=run_forever, args=(poll,), daemon=True)
-poll_thread.start()
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0')
+    # Any code that is run before main gets executed twice, run here
+
+    update_data()
+
+    poll_thread = threading.Thread(
+        target=run_forever, args=(poll,), daemon=True)
+
+    poll_thread.start()
+
+    app.run(host='0.0.0.0', port=5000, debug=False)

scorecard/scorecard/templates/index.html

@@ -1,6 +1,7 @@
 <html lang="en">
 <head>
     <title>Scorecard</title>
+
     <style type="text/css">
       td {
           padding: 3px 10px 3px 10px;