v2.0.0 (#64)

api/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM registry.osiris.services/anubis/api:v2.0.0-alpha
+FROM registry.osiris.services/anubis/api:v2.0.0
 
 ENV DISABLE_ELK=1
 CMD ./docker-entrypoint-dev.sh

api/anubis/models/__init__.py

@@ -158,6 +158,7 @@ class AssignmentRepo(db.Model):
 
     # id
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    github_username = db.Column(db.String(256), nullable=False)
 
     # Foreign Keys
     owner_id = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)

api/anubis/routes/private.py

@@ -18,15 +18,11 @@
 from anubis.utils.elastic import log_endpoint
 from anubis.utils.redis_queue import enqueue_webhook_rpc
 from anubis.utils.logger import logger
+from anubis.utils.data import fix_dangling
 
 private = Blueprint('private', __name__, url_prefix='/private')
 
 
-def fix_dangling():
-    # TODO rewite this
-    raise
-
-
 @cache.memoize(timeout=30)
 def stats_for(student_id, assignment_id):
     # TODO rewrite this

api/anubis/routes/public.py

@@ -1,4 +1,3 @@
-import json
 import string
 from datetime import datetime, timedelta
 
@@ -11,6 +10,7 @@
 from anubis.utils.auth import get_token
 from anubis.utils.cache import cache
 from anubis.utils.data import error_response, success_response, is_debug
+from anubis.utils.data import fix_dangling
 from anubis.utils.data import get_classes, get_assignments, get_submissions
 from anubis.utils.data import regrade_submission, enqueue_webhook_rpc
 from anubis.utils.decorators import json_response, require_user
@@ -41,7 +41,7 @@ def public_logout():
 @public.route('/oauth')
 @log_endpoint('public-oauth', lambda: 'oauth')
 def public_oauth():
-    next_url = request.args.get('next') or '/'
+    next_url = request.args.get('next') or '/courses'
     resp = provider.authorized_response()
     if resp is None or 'access_token' not in resp:
         return 'Access Denied'
@@ -60,6 +60,8 @@ def public_oauth():
     if u.github_username is None:
         next_url = '/set-github-username'
 
+    fix_dangling()
+
     r = make_response(redirect(next_url))
     r.set_cookie('token', get_token(u.netid))
 
@@ -86,6 +88,9 @@ def public_set_github_username():
         return error_response('Github usernames may only contain alphanumeric characters '
                               'or single hyphens, and cannot begin or end with a hyphen.')
 
+    logger.info(str(u.last_updated))
+    logger.info(str(u.last_updated + timedelta(hours=1)) + ' - ' + str(datetime.now()))
+
     if u.github_username is not None and u.last_updated + timedelta(hours=1) < datetime.now():
         return error_response('Github usernames can only be '
                               'changed one hour after first setting. '
@@ -273,20 +278,24 @@ def public_webhook():
     github_username = webhook['pusher']['name']
     commit = webhook['after']
     assignment_name = webhook['repository']['name'][:-(len(github_username) + 1)]
-    unique_code = assignment_name.split('-')[-1]
-
-    logger.debug('github_username: {} unique_code: {}'.format(github_username, unique_code))
 
     # Attempt to find records for the relevant models
-    assignment = Assignment.query.filter_by(unique_code=unique_code).first()
-    user = User.query.filter(User.github_username == github_username).first()
-    repo = AssignmentRepo.query.join(Assignment).join(Class_).join(InClass).join(User).filter(
-        User.github_username == github_username,
-        Assignment.name == assignment_name,
+    assignment = Assignment.query.filter(
+        Assignment.unique_code.in_(webhook['repository']['name'].split('-'))
     ).first()
+    user = User.query.filter(User.github_username == github_username).first()
 
-    logger.debug('github_username: {} unique_code: {}'.format(github_username, unique_code))
-    logger.debug('assignment: {} user: {} repo: {}'.format(assignment, user, repo))
+    # The before Hash will be all 0s on for the first hash.
+    # We will want to ignore both this first push (the initialization of the repo)
+    # and all branches that are not master.
+    if webhook['before'] == '0000000000000000000000000000000000000000':
+        # Record that a new repo was created (and therefore, someone just started their assignment)
+        logger.info('new student repo ', extra={
+            'repo_url': repo_url, 'github_username': github_username,
+            'assignment_name': assignment_name, 'commit': commit,
+        })
+        esindex('new-repo', repo_url=repo_url, assignment=str(assignment))
+        return success_response('initial commit')
 
     # Verify that we can match this push to an assignment
     if assignment is None:
@@ -296,6 +305,16 @@ def public_webhook():
         })
         return error_response('assignment not found'), 406
 
+    repo = AssignmentRepo.query.join(Assignment).join(Class_).join(InClass).join(User).filter(
+        User.github_username == github_username,
+        Assignment.unique_code == assignment.unique_code,
+    ).first()
+
+    logger.debug('webhook data', extra={
+        'github_username': github_username, 'assignment': assignment.name,
+        'repo_url': repo_url, 'commit': commit, 'unique_code': assignment.unique_code
+    })
+
     if not is_debug():
         # Make sure that the repo we're about to process actually belongs to our organization
         if not webhook['repository']['full_name'].startswith('os3224/'):
@@ -307,22 +326,10 @@ def public_webhook():
 
     # if we dont have a record of the repo, then add it
     if repo is None:
-        repo = AssignmentRepo(owner=user, assignment=assignment, repo_url=repo_url)
+        repo = AssignmentRepo(owner=user, assignment=assignment, repo_url=repo_url, github_username=github_username)
         db.session.add(repo)
         db.session.commit()
 
-    # The before Hash will be all 0s on for the first hash.
-    # We will want to ignore both this first push (the initialization of the repo)
-    # and all branches that are not master.
-    if webhook['before'] == '0000000000000000000000000000000000000000':
-        # Record that a new repo was created (and therefore, someone just started their assignment)
-        logger.info('new student repo ', extra={
-            'repo_url': repo_url, 'github_username': github_username,
-            'assignment_name': assignment_name, 'commit': commit,
-        })
-        esindex('new-repo', github_username=github_username, repo_url=repo_url, assignment=str(assignment))
-        return success_response('initial commit')
-
     if webhook['ref'] != 'refs/heads/master':
         logger.warning('not push to master', extra={
             'repo_url': repo_url, 'github_username': github_username,

api/anubis/utils/data.py

@@ -8,7 +8,7 @@
 from sqlalchemy.exc import IntegrityError
 
 from anubis.config import config
-from anubis.models import User, Class_, InClass, Assignment, Submission
+from anubis.models import User, Class_, InClass, Assignment, Submission, AssignmentRepo
 from anubis.models import db
 from anubis.utils.auth import load_user
 from anubis.utils.cache import cache
@@ -216,4 +216,27 @@ def notify(user: User, message: str, subject: str):
     :return:
     """
     recipient = '{netid}@nyu.edu'.format(netid=user.netid)
-    send_noreply_email(message, subject, recipient)
+    send_noreply_email(message, subject, recipient)
+
+
+def fix_dangling():
+    """
+    Try to connect repos that do not have an owner.
+
+    :return:
+    """
+    dangling_repos = AssignmentRepo.query.filter(
+        AssignmentRepo.owner_id == None
+    ).all()
+
+    for dr in dangling_repos:
+        owner = User.query.filter(
+            User.github_username == dr.github_username
+        ).first()
+
+        if owner is not None:
+            dr.owner_id = owner.id
+            db.session.add_all((dr, owner))
+
+            for s in dr.submissions:
+                enqueue_webhook_rpc(s.id)

api/anubis/utils/elastic.py

@@ -1,5 +1,5 @@
-import traceback
 import json
+import traceback
 from datetime import datetime
 from functools import wraps
 
@@ -8,8 +8,8 @@
 from geoip import geolite2
 from werkzeug import exceptions
 
-from anubis.utils.http import get_request_ip
 from anubis.config import config
+from anubis.utils.http import get_request_ip
 from anubis.utils.logger import logger
 
 es = Elasticsearch(['http://elasticsearch:9200'])
@@ -44,7 +44,11 @@ def wrapper(*args, **kwargs):
                     date=datetime.now(),
                     method=request.method,
                     path=request.path
-                ))
+                ), extra={
+                    'ip': get_request_ip(),
+                    'method': request.method,
+                    'path': request.path,
+                })
                 es.index(index='request', body={
                     'type': log_type.lower(),
                     'path': request.path,

api/anubis/utils/oauth.py

@@ -9,8 +9,9 @@
     request_token_url=None,
     request_token_params={'scope': 'openid'},
     access_token_url='https://auth.nyu.edu/oauth2/token',
-    # access_token_method='POST',
+    #access_token_method='POST',
     access_token_params={'client_id': config.OAUTH_CONSUMER_KEY},
     consumer_key=config.OAUTH_CONSUMER_KEY,
     consumer_secret=config.OAUTH_CONSUMER_SECRET,
+    #content_type='application/json',
 )

cli/anubis/cli.py

@@ -152,6 +152,7 @@ def assignment():
 def sync():
     assignment_meta = yaml.safe_load(open('assignment.yml').read())
     click.echo(json.dumps(assignment_meta, indent=2))
+    import assignment
     import utils
     assignment_meta['tests'] = list(utils.registered_tests.keys())
     r = post_json('/private/assignment/sync', assignment_meta)

docker-compose.yml

@@ -29,7 +29,7 @@ services:
       - "traefik.enable=false"
 
   api:
-    image: registry.osiris.services/anubis/api:v2.0.0-alpha
+    image: registry.osiris.services/anubis/api:v2.0.0
     build: ./api
     # restart: on-failure
     command: sh -c "while true; do python3 dev.py; sleep 1; done"
@@ -64,7 +64,7 @@ services:
       - "traefik.http.routers.api-public.middlewares=strip-api@docker"
 
   api-dev:
-    image: registry.osiris.services/anubis/api-dev:v2.0.0-alpha
+    image: registry.osiris.services/anubis/api-dev:v2.0.0
     build:
       context: ./api
       dockerfile: Dockerfile.dev
@@ -83,7 +83,7 @@ services:
 
   pipeline-api:
     build: ./api
-    image: registry.osiris.services/anubis/api:v2.0.0-alpha
+    image: registry.osiris.services/anubis/api:v2.0.0
     # restart: on-failure
     environment:
       - "DEBUG=1"
@@ -110,7 +110,7 @@ services:
       - "traefik.enable=false"
 
   web:
-    image: registry.osiris.services/anubis/web:v2.0.0-alpha
+    image: registry.osiris.services/anubis/web:v2.0.0
     build: ./web
     command: sh -c 'yarn && yarn run start'
     environment:

kube/helm/templates/api.yml

@@ -27,7 +27,7 @@ spec:
       labels:
         app: anubis-api
     spec:
-      dnsPolicy: ClusterFirst
+      # dnsPolicy: Default
       nodeSelector:
         datacenter: mtc
       containers:
@@ -38,7 +38,7 @@ spec:
         - name: web
           containerPort: 5000
         resources:
-          {{- toYaml .Values.api.resources | nindent 10 }}
+         {{- toYaml .Values.api.resources | nindent 10 }}
         env:
         - name: "DEBUG"
           value: {{- if .Values.debug }} "1"{{- else }} "0"{{- end }}
@@ -48,7 +48,7 @@ spec:
         - name: "WORKERS"
           value: {{ .Values.api.workers | quote }}
         - name: "GUNICORN_OPTIONS"
-          value: "--preload --capture-output --enable-stdio-inheritance"
+          value: "--capture-output --enable-stdio-inheritance --preload --timeout 30"
         - name: "SECRET_KEY"
           valueFrom:
             secretKeyRef:
@@ -109,7 +109,7 @@ spec:
       dnsPolicy: ClusterFirst
       containers:
       - name: jupyter
-        image: registry.osiris.services/anubis/api-dev:v2.0.0-alpha
+        image: registry.osiris.services/anubis/api-dev:v2.0.0
         imagePullPolicy: Always
         ports:
         - name: web

kube/helm/templates/pipeline-api.yml

@@ -32,8 +32,8 @@ spec:
       - name: anubis-pipeline-api
         image: {{ .Values.api.image }}:{{ .Values.api.tag }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
-        resources:
-          {{- toYaml .Values.api.resources | nindent 10 }}
+        #resources:
+        #  { {- toYaml .Values.api.resources | nindent 10 } }
         ports:
         - name: web
           containerPort: 5000

kube/helm/values.yaml

@@ -11,12 +11,10 @@ api:
   workers: 4
 
   image: "registry.osiris.services/anubis/api"
-  tag: "v2.0.0-alpha"
+  tag: "v2.0.0"
   resources:
-    limits:
-      memory: "200Mi"
     requests:
-      memory: "100Mi"
+      memory: "500Mi"
 
 
 pipeline_api:
@@ -25,14 +23,14 @@ pipeline_api:
 
   resources:
     limits:
-      memory: "200Mi"
+      memory: "500Mi"
     requests:
-      memory: "100Mi"
+      memory: "500Mi"
 
 web:
   replicas: 2
   image: "registry.osiris.services/anubis/web"
-  tag: "v2.0.0-alpha"
+  tag: "v2.0.0"
 
 logstash:
   image: "registry.osiris.services/anubis/logstash"

web/src/App.jsx

@@ -238,7 +238,7 @@ function App() {
                   <Route exact path={'/courses/assignments/submissions/info'}>
                     <SubmissionInfo/>
                   </Route>
-                  <Route exact path={'/github-username'}>
+                  <Route exact path={'/set-github-username'}>
                     <GetGithubUsername/>
                   </Route>
                   <Route exact path={"/about"}>