diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e71b162..28c0c87 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,7 +13,8 @@ env: MAIN_REPO: IN-CORE/incore-auth DOCKERHUB_ORG: incore NCSAHUB: hub.ncsa.illinois.edu/incore - PLATFORM: "linux/amd64,linux/arm64" + #PLATFORM: "linux/amd64,linux/arm64" missing pre-compiled packages + PLATFORM: "linux/amd64" jobs: docker: @@ -22,7 +23,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # calculate some variables that are used later - name: version information @@ -145,7 +146,7 @@ jobs: # build the docker images - name: Build and push docker - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: push: true platforms: ${{ env.PLATFORM }} @@ -161,9 +162,9 @@ jobs: # this will update the README of the dockerhub repo - name: Docker Hub Description if: env.DOCKERHUB_README == 'true' - uses: peter-evans/dockerhub-description@v2 - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_REPOSITORY: ${{ env.DOCKERHUB_ORG }}/${{ github.event.repository.name }} - README_FILEPATH: README.md + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKERHUB_ORG }}/${{ github.event.repository.name }} + readme-filepath: README.md diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f3426a2..0fda39b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -27,8 +27,8 @@ jobs: changelog="${changelog//'%'/'%25'}" changelog="${changelog//$'\n'/'%0A'}" changelog="${changelog//$'\r'/'%0D'}" - echo "::set-output name=version::$version" - echo "::set-output name=changelog::$changelog" + echo "version=$version" >> $GITHUB_OUTPUT + echo "changelog=$changelog" >> $GITHUB_OUTPUT - name: create release if: github.event_name != 'pull_request' && github.repository == env.MAIN_REPO diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a27945..25a3006 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) From version 1.2.0 the file IP2LOCATION-LITE-DB5.BIN is no longer part of the docker image and will need to be downloaded (after registration) from [ip2location](https://lite.ip2location.com/database/ip-country?lang=en_US) and be placed in /srv/incore_auth. +# [1.6.0] - 2023-03-14 + +## Added +- information about user and groups is synced every 30 minutes back to the database and datawolf + +## Changed +- Added playbook and its sub directories to tracking resource [#32](https://github.com/IN-CORE/incore-auth/issues/32) +- updated all packages used + # [1.5.0] - 2022-09-24 ## Changed diff --git a/Dockerfile b/Dockerfile index e7a7631..68e2d2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,14 @@ -FROM alpine:3.7 +FROM python:3.7-alpine MAINTAINER Incore LABEL PROJECT_REPO_URL = "" \ PROJECT_REPO_BROWSER_URL = "" \ - DESCRIPTION = ")" - -RUN apk add --no-cache \ - gcc \ - g++ \ - libffi-dev \ - make \ - python3 \ - python3-dev \ - openssl-dev && \ - python3 -m ensurepip && \ - rm -r /usr/lib/python*/ensurepip && \ - pip3 install --upgrade pip setuptools && \ - if [[ ! -e /usr/bin/pip ]]; then ln -s pip3 /usr/bin/pip; fi && \ - if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \ - rm -r /root/.cache + DESCRIPTION = ")" WORKDIR /srv COPY incore_auth/requirements.txt incore_auth/ -RUN pip install -Ur incore_auth/requirements.txt +RUN pip3 install -Ur incore_auth/requirements.txt COPY incore_auth incore_auth @@ -32,19 +17,11 @@ WORKDIR /srv/incore_auth ENV FLASK_APP="app.py" \ KEYCLOAK_PUBLIC_KEY="" \ KEYCLOAK_AUDIENCE="" \ + DATAWOLF_URL="http://incore-datawolf:8888/datawolf" \ + MONGODB_URI="" \ INFLUXDB_V2_URL="" \ INFLUXDB_V2_ORG="" \ INFLUXDB_V2_TOKEN="" \ INFLUXDB_V2_FILE_LOCATION="data/IP2LOCATION-LITE-DB5.BIN" -#CMD ["python", "-m", "flask", "run", "--host", "0.0.0.0"] CMD ["gunicorn", "app:app", "--config", "/srv/incore_auth/gunicorn.config.py"] - -#ENTRYPOINT ["gunicorn", \ -# "--access-logfile=-", \ -# "--log-level=info", \ -# "--workers=3", \ -# "--bind=0.0.0.0:5000", \ -# "incore_auth.app:app" \ -# ] - diff --git a/incore_auth/app.py b/incore_auth/app.py index d5d5ba0..9ae3309 100644 --- a/incore_auth/app.py +++ b/incore_auth/app.py @@ -2,11 +2,15 @@ import json import os import time +import threading import urllib.request import IP2Location import geohash2 import influxdb_client +import pymongo + +from cachetools import cached, TTLCache from flask import Flask, request, Response, make_response, json from jose import jwt @@ -25,6 +29,10 @@ geoserver = {} geoserver_delta = 2 +cache_size = 1024 +# timeout in seconds, in this case 30 minutes +cache_timeout = 30*60 + # setup database for geolocation try: geolocation = IP2Location.IP2Location(CONTRIBUTION_DB_NAME) @@ -40,6 +48,87 @@ app.logger.setLevel(gunicorn_logger.level) +def cache_key(request_info): + """Return username from request_info to be used as cache key""" + return request_info["username"] + + +def update_services_thread(request_info): + """When a user does any action, it will check to update the groups in mongo, as well as make sure + the user has access to datawolf. The function is cached to make sure only every 30 minutes we do + the checks, since this can be expensive.""" + + # get information from request + username = request_info["username"] + if not username: + return username + groups = request_info['groups'] + + # call datawolf to add user + datawolf_url = config["datawolf_url"] + if datawolf_url: + query = urllib.parse.urlencode({ + "firstname": request_info["firstname"], + "lastname": request_info["lastname"], + "email": username + }) + datawolf_url = "%s/persons?%s" % (datawolf_url.rstrip("/"), query) + req = urllib.request.Request(datawolf_url, method='POST') + response = urllib.request.urlopen(req) + if response.code == 200: + app.logger.info(f"Added user to datawolf {username}") + elif response.code == 204: + app.logger.debug(f"User already exists in datawolf {username}") + else: + app.logger.info(f"Did not add user to datawolf {username}") + + # update database with user quota + mongo_client = config["mongo_client"] + if mongo_client: + mongo_user = mongo_client["spacedb"]["UserGroups"].find_one({"username": username}) + if not mongo_user: + # INSERT + mongo_client["spacedb"]["UserGroups"].insert_one({ + "username": username, + "className": "edu.illinois.ncsa.incore.common.models.UserGroups", + "groups": groups + }) + app.logger.info(f"Inserted groups document for {username}") + elif set(groups) != set(mongo_user["groups"]): + # UPDATE + mongo_client["spacedb"]["UserGroups"].update_one( + {"username": username}, {"$set": {"groups": groups}} + ) + app.logger.info(f"Synced groups for {username} - {groups}") + else: + # NOTHING + app.logger.debug(f"No sync needed for {username}") + + mongo_space = mongo_client["spacedb"]["Space"].find_one({"metadata.name": username}) + if not mongo_space: + mongo_client["spacedb"]["Space"].insert_one({ + "className": "edu.illinois.ncsa.incore.common.models.Space", + "metadata": { + "className": "edu.illinois.ncsa.incore.common.models.SpaceMetadata", + "name": username + }, + "privileges": { + "className": "edu.illinois.ncsa.incore.common.auth.Privileges", + "userPrivileges": { + username: "ADMIN" + } + }, + "members": [ + ] + }) + app.logger.info(f"Inserted space document for {username}") + + +@cached(cache=TTLCache(maxsize=cache_size, ttl=cache_timeout), key=cache_key) +def update_services(request_info): + threading.Thread(target=update_services_thread, args=(request_info,), daemon=True).start() + + def record_request(request_info): if 'X-Forwarded-For' not in request.headers: return @@ -173,6 +262,11 @@ def request_userinfo(request_info): request_info['error'] = 'JWT Error: invalid token' return + # get name of user + request_info["firstname"] = access_token["given_name"] + request_info["lastname"] = access_token["family_name"] + request_info["fullname"] = access_token["name"] + # retrieve the groups the user belongs to from access token request_info['username'] = access_token["preferred_username"] request_info['groups'] = access_token.get("groups", []) @@ -200,6 +294,8 @@ def request_resource(request_info): request_info['resource'] = pieces[1] if request_info['resource'] == "doc" and len(pieces) > 2: request_info['fields']['manual'] = pieces[2] + if request_info['resource'] == "playbook" and len(pieces) > 2: + request_info['fields']['playbook'] = pieces[2] if request_info['resource'] == "data" and len(pieces) > 4 and uri.endswith('blob'): request_info['fields']['dataset'] = pieces[4] if request_info['resource'] == "dfr3" and len(pieces) > 4: @@ -238,6 +334,9 @@ def verify_token(): # dict to hold all information request_info = { "username": "", + "firstname": "", + "lastname": "", + "fullname": "", "method": request.method, "url": request.path, "resource": "", @@ -253,6 +352,9 @@ def verify_token(): request_resource(request_info) request_userinfo(request_info) + # update backend services + update_services(request_info) + # record request record_request(request_info) @@ -340,6 +442,17 @@ def setup(): else: config['audience'] = None + # store datawolf url + config["datawolf_url"] = os.environ.get('DATAWOLF_URL', None) + + # setup mongodb + mongodb_uri = os.environ.get('MONGODB_URI', None) + if mongodb_uri: + mongo_client = pymongo.MongoClient(mongodb_uri) + config["mongo_client"] = mongo_client + else: + config["mongo_client"] = None + # setup influxdb try: client = influxdb_client.InfluxDBClient.from_env_properties() diff --git a/incore_auth/requirements.txt b/incore_auth/requirements.txt index f55e797..73e68bc 100644 --- a/incore_auth/requirements.txt +++ b/incore_auth/requirements.txt @@ -1,9 +1,11 @@ -bcrypt==3.1.0 -flask==0.12.2 -gunicorn==19.7.1 -gevent==1.4.0 -python-jose==3.1.0 -influxdb-client==1.14.0 -IP2Location==8.5.1 -geohash2==1.1 -python-dotenv==0.10.3 \ No newline at end of file +bcrypt==4.* +flask==2.* +gunicorn==20.* +gevent==22.* +python-jose==3.* +influxdb-client==1.* +IP2Location==8.* +geohash2==1.* +python-dotenv==0.* +pymongo==4.* +cachetools==4.*