Release/1.6.0 (#36)

* track playbook (#31) this assumes the url is /playbook/joplin/..... and will record the playbook in a field called playbook, that is tracked as part of influxdb and can be shown on grafana. * fix build * Sync user (#33) * update packages * add function to sync user+groups * add user to datawolf fix requirements file cleanup dockerfile * remove output --------- Co-authored-by: Yong Wook Kim <[email protected]>
IN-CORE · Mar 15, 2023 · f713b30 · f713b30
1 parent 02b2869
commit f713b30
Show file tree

Hide file tree

Showing 6 changed files with 151 additions and 49 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -13,7 +13,8 @@ env:
   MAIN_REPO: IN-CORE/incore-auth
   DOCKERHUB_ORG: incore
   NCSAHUB: hub.ncsa.illinois.edu/incore
-  PLATFORM: "linux/amd64,linux/arm64"
+  #PLATFORM: "linux/amd64,linux/arm64" missing pre-compiled packages
+  PLATFORM: "linux/amd64"
 
 jobs:
   docker:
@@ -22,7 +23,7 @@ jobs:
       packages: write
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # calculate some variables that are used later
       - name: version information
@@ -145,7 +146,7 @@ jobs:
 
       # build the docker images
       - name: Build and push docker
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           push: true
           platforms: ${{ env.PLATFORM }}
@@ -161,9 +162,9 @@ jobs:
       # this will update the README of the dockerhub repo
       - name: Docker Hub Description
         if: env.DOCKERHUB_README == 'true'
-        uses: peter-evans/dockerhub-description@v2
-        env:
-          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
-          DOCKERHUB_REPOSITORY: ${{ env.DOCKERHUB_ORG }}/${{ github.event.repository.name }}
-          README_FILEPATH: README.md
+        uses: peter-evans/dockerhub-description@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          repository: ${{ env.DOCKERHUB_ORG }}/${{ github.event.repository.name }}
+          readme-filepath: README.md
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
@@ -27,8 +27,8 @@ jobs:
           changelog="${changelog//'%'/'%25'}"
           changelog="${changelog//$'\n'/'%0A'}"
           changelog="${changelog//$'\r'/'%0D'}"
-          echo "::set-output name=version::$version"
-          echo "::set-output name=changelog::$changelog"
+          echo "version=$version" >> $GITHUB_OUTPUT
+          echo "changelog=$changelog" >> $GITHUB_OUTPUT
 
       - name: create release
         if: github.event_name != 'pull_request' && github.repository == env.MAIN_REPO

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
 From version 1.2.0 the file IP2LOCATION-LITE-DB5.BIN is no longer part of the docker image and will need to be downloaded (after registration) from [ip2location](https://lite.ip2location.com/database/ip-country?lang=en_US) and be placed in /srv/incore_auth.
 
+# [1.6.0] - 2023-03-14
+
+## Added
+- information about user and groups is synced every 30 minutes back to the database and datawolf
+
+## Changed
+- Added playbook and its sub directories to tracking resource [#32](https://github.com/IN-CORE/incore-auth/issues/32)
+- updated all packages used
+
 # [1.5.0] - 2022-09-24
 
 ## Changed

diff --git a/Dockerfile b/Dockerfile
@@ -1,29 +1,14 @@
-FROM alpine:3.7
+FROM python:3.7-alpine
 
 MAINTAINER Incore <[email protected]>
 LABEL PROJECT_REPO_URL         = "" \
       PROJECT_REPO_BROWSER_URL = "" \
-      DESCRIPTION              = ")" 
-
-RUN apk add --no-cache \
-    gcc \
-    g++ \
-    libffi-dev \
-    make \
-    python3 \
-    python3-dev \
-    openssl-dev && \
-  python3 -m ensurepip && \
-  rm -r /usr/lib/python*/ensurepip && \
-  pip3 install --upgrade pip setuptools && \
-  if [[ ! -e /usr/bin/pip ]]; then ln -s pip3 /usr/bin/pip; fi && \
-  if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \
-  rm -r /root/.cache
+      DESCRIPTION              = ")"
 
 WORKDIR /srv
 
 COPY incore_auth/requirements.txt incore_auth/
-RUN pip install -Ur incore_auth/requirements.txt
+RUN pip3 install -Ur incore_auth/requirements.txt
 
 COPY incore_auth incore_auth
 
@@ -32,19 +17,11 @@ WORKDIR /srv/incore_auth
 ENV FLASK_APP="app.py" \
     KEYCLOAK_PUBLIC_KEY="" \
     KEYCLOAK_AUDIENCE="" \
+    DATAWOLF_URL="http://incore-datawolf:8888/datawolf" \
+    MONGODB_URI="" \
     INFLUXDB_V2_URL="" \
     INFLUXDB_V2_ORG="" \
     INFLUXDB_V2_TOKEN="" \
     INFLUXDB_V2_FILE_LOCATION="data/IP2LOCATION-LITE-DB5.BIN"
 
-#CMD ["python", "-m", "flask", "run", "--host", "0.0.0.0"]
 CMD ["gunicorn", "app:app", "--config", "/srv/incore_auth/gunicorn.config.py"]
-
-#ENTRYPOINT ["gunicorn", \
-#    "--access-logfile=-", \
-#    "--log-level=info", \
-#    "--workers=3", \
-#    "--bind=0.0.0.0:5000", \
-#    "incore_auth.app:app" \
-#    ]
-
diff --git a/incore_auth/app.py b/incore_auth/app.py
@@ -2,11 +2,15 @@
 import json
 import os
 import time
+import threading
 import urllib.request
 
 import IP2Location
 import geohash2
 import influxdb_client
+import pymongo
+
+from cachetools import cached, TTLCache
 
 from flask import Flask, request, Response, make_response, json
 from jose import jwt
@@ -25,6 +29,10 @@
 geoserver = {}
 geoserver_delta = 2
 
+cache_size = 1024
+# timeout in seconds, in this case 30 minutes
+cache_timeout = 30*60
+
 # setup database for geolocation
 try:
     geolocation = IP2Location.IP2Location(CONTRIBUTION_DB_NAME)
@@ -40,6 +48,87 @@
     app.logger.setLevel(gunicorn_logger.level)
 
 
+def cache_key(request_info):
+    """Return username from request_info to be used as cache key"""
+    return request_info["username"]
+
+
+def update_services_thread(request_info):
+    """When a user does any action, it will check to update the groups in mongo, as well as make sure
+    the user has access to datawolf. The function is cached to make sure only every 30 minutes we do
+    the checks, since this can be expensive."""
+
+    # get information from request
+    username = request_info["username"]
+    if not username:
+        return username
+    groups = request_info['groups']
+
+    # call datawolf to add user
+    datawolf_url = config["datawolf_url"]
+    if datawolf_url:
+        query = urllib.parse.urlencode({
+            "firstname": request_info["firstname"],
+            "lastname": request_info["lastname"],
+            "email": username
+        })
+        datawolf_url = "%s/persons?%s" % (datawolf_url.rstrip("/"), query)
+        req = urllib.request.Request(datawolf_url, method='POST')
+        response = urllib.request.urlopen(req)
+        if response.code == 200:
+            app.logger.info(f"Added user to datawolf {username}")
+        elif response.code == 204:
+            app.logger.debug(f"User already exists in datawolf {username}")
+        else:
+            app.logger.info(f"Did not add user to datawolf {username}")
+
+    # update database with user quota
+    mongo_client = config["mongo_client"]
+    if mongo_client:
+        mongo_user = mongo_client["spacedb"]["UserGroups"].find_one({"username": username})
+        if not mongo_user:
+            # INSERT
+            mongo_client["spacedb"]["UserGroups"].insert_one({
+                "username": username,
+                "className": "edu.illinois.ncsa.incore.common.models.UserGroups",
+                "groups": groups
+            })
+            app.logger.info(f"Inserted groups document for {username}")
+        elif set(groups) != set(mongo_user["groups"]):
+            # UPDATE
+            mongo_client["spacedb"]["UserGroups"].update_one(
+                {"username": username}, {"$set": {"groups": groups}}
+            )
+            app.logger.info(f"Synced groups for {username} - {groups}")
+        else:
+            # NOTHING
+            app.logger.debug(f"No sync needed for {username}")
+
+        mongo_space = mongo_client["spacedb"]["Space"].find_one({"metadata.name": username})
+        if not mongo_space:
+            mongo_client["spacedb"]["Space"].insert_one({
+                "className": "edu.illinois.ncsa.incore.common.models.Space",
+                "metadata": {
+                    "className": "edu.illinois.ncsa.incore.common.models.SpaceMetadata",
+                    "name": username
+                },
+                "privileges": {
+                    "className": "edu.illinois.ncsa.incore.common.auth.Privileges",
+                    "userPrivileges": {
+                        username: "ADMIN"
+                    }
+                },
+                "members": [
+                ]
+            })
+            app.logger.info(f"Inserted space document for {username}")
+
+
+@cached(cache=TTLCache(maxsize=cache_size, ttl=cache_timeout), key=cache_key)
+def update_services(request_info):
+    threading.Thread(target=update_services_thread, args=(request_info,), daemon=True).start()
+
+
 def record_request(request_info):
     if 'X-Forwarded-For' not in request.headers:
         return
@@ -173,6 +262,11 @@ def request_userinfo(request_info):
         request_info['error'] = 'JWT Error: invalid token'
         return
 
+    # get name of user
+    request_info["firstname"] = access_token["given_name"]
+    request_info["lastname"] = access_token["family_name"]
+    request_info["fullname"] = access_token["name"]
+
     # retrieve the groups the user belongs to from access token
     request_info['username'] = access_token["preferred_username"]
     request_info['groups'] = access_token.get("groups", [])
@@ -200,6 +294,8 @@ def request_resource(request_info):
             request_info['resource'] = pieces[1]
             if request_info['resource'] == "doc" and len(pieces) > 2:
                 request_info['fields']['manual'] = pieces[2]
+            if request_info['resource'] == "playbook" and len(pieces) > 2:
+                request_info['fields']['playbook'] = pieces[2]
             if request_info['resource'] == "data" and len(pieces) > 4 and uri.endswith('blob'):
                 request_info['fields']['dataset'] = pieces[4]
             if request_info['resource'] == "dfr3" and len(pieces) > 4:
@@ -238,6 +334,9 @@ def verify_token():
     # dict to hold all information
     request_info = {
         "username": "",
+        "firstname": "",
+        "lastname": "",
+        "fullname": "",
         "method": request.method,
         "url": request.path,
         "resource": "",
@@ -253,6 +352,9 @@ def verify_token():
     request_resource(request_info)
     request_userinfo(request_info)
 
+    # update backend services
+    update_services(request_info)
+
     # record request
     record_request(request_info)
 
@@ -340,6 +442,17 @@ def setup():
     else:
         config['audience'] = None
 
+    # store datawolf url
+    config["datawolf_url"] = os.environ.get('DATAWOLF_URL', None)
+
+    # setup mongodb
+    mongodb_uri = os.environ.get('MONGODB_URI', None)
+    if mongodb_uri:
+        mongo_client = pymongo.MongoClient(mongodb_uri)
+        config["mongo_client"] = mongo_client
+    else:
+        config["mongo_client"] = None
+
     # setup influxdb
     try:
         client = influxdb_client.InfluxDBClient.from_env_properties()

diff --git a/incore_auth/requirements.txt b/incore_auth/requirements.txt
@@ -1,9 +1,11 @@
-bcrypt==3.1.0
-flask==0.12.2
-gunicorn==19.7.1
-gevent==1.4.0
-python-jose==3.1.0
-influxdb-client==1.14.0
-IP2Location==8.5.1
-geohash2==1.1
-python-dotenv==0.10.3
+bcrypt==4.*
+flask==2.*
+gunicorn==20.*
+gevent==22.*
+python-jose==3.*
+influxdb-client==1.*
+IP2Location==8.*
+geohash2==1.*
+python-dotenv==0.*
+pymongo==4.*
+cachetools==4.*