From 208693b6743200ef85bad385f224584f00cdb9ff Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 11 Mar 2019 16:02:55 -0500 Subject: [PATCH] chore: run black code formatter --- Dockerfile | 12 +- deployment/uwsgi/uwsgi.ini | 19 -- deployment/uwsgi/wsgi.py | 1 + manifestservice/api.py | 39 ++- manifestservice/dev_settings.py | 3 - manifestservice/manifests/__init__.py | 137 +++++++---- setup.py | 1 - tests/app_test.py | 339 +++++++++++++++----------- tests/conftest.py | 4 +- 9 files changed, 328 insertions(+), 227 deletions(-) delete mode 100644 manifestservice/dev_settings.py diff --git a/Dockerfile b/Dockerfile index f39fd34..5349e25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,8 @@ # To run: docker run -v /path/to/wsgi.py:/var/www/manifestservice/wsgi.py --name=manifestservice -p 81:80 manifestservice # To check running container: docker exec -it manifestservice /bin/bash - FROM quay.io/cdis/python-nginx:master -RUN echo "7" && ls ENV appname=manifestservice # number of uwsgi worker processes @@ -16,18 +14,16 @@ RUN apk update \ && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \ && apk add linux-headers musl-dev gcc \ && apk add curl bash git vim -RUN echo "17" && ls + COPY . /$appname COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py COPY ./deployment/nginx/nginx.conf /etc/nginx/ COPY ./deployment/nginx/uwsgi.conf /etc/nginx/conf.d/nginx.conf WORKDIR /$appname -RUN echo "24" && ls RUN python -m pip install --upgrade pip && pip install pipenv && pipenv install -RUN echo "26" && ls RUN mkdir -p /var/www/$appname \ && mkdir -p /var/www/.cache/Python-Eggs/ \ && mkdir /run/nginx/ \ @@ -35,13 +31,13 @@ RUN mkdir -p /var/www/$appname \ && ln -sf /dev/stderr /var/log/nginx/error.log \ && chown nginx -R /var/www/.cache/Python-Eggs/ \ && chown nginx /var/www/$appname -RUN echo "34" && ls + EXPOSE 80 -RUN echo "36" + RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \ && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \ && python setup.py install WORKDIR /var/www/$appname -RUN echo "42" && ls + CMD /$appname/dockerrun.bash diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini index 0c469a5..530bce9 100644 --- a/deployment/uwsgi/uwsgi.ini +++ b/deployment/uwsgi/uwsgi.ini @@ -12,29 +12,10 @@ vacuum = true chdir = /manifestservice/ uid = nginx gid = nginx -# pythonpath = /usr/local/lib/site-packages/ virtualenv = $(VENV) pythonpath = /usr/local/lib/python3.6/site-packages/ -# [uwsgi] -# protocol = uwsgi -# socket = /var/www/wts/uwsgi.sock -# buffer-size = 32768 -# chmod-socket = 666 -# master = true -# harakiri-verbose = 20 -# disable-logging = true -# wsgi-file=/wts/wsgi.py -# plugins = python3 -# vacuum = true -# chdir = /wts/ -# uid = nginx -# gid = nginx -# virtualenv = $(VENV) -# pythonpath = /usr/local/lib/python3.6/site-packages/ - - # Use this to initialize application in worker processes, not master. This # prevents the workers from all trying to open the same database # connections at startup: diff --git a/deployment/uwsgi/wsgi.py b/deployment/uwsgi/wsgi.py index 88f9105..c8ff944 100644 --- a/deployment/uwsgi/wsgi.py +++ b/deployment/uwsgi/wsgi.py @@ -1,2 +1,3 @@ from manifestservice.api import app + application = app diff --git a/manifestservice/api.py b/manifestservice/api.py index bb404f8..2125dcd 100644 --- a/manifestservice/api.py +++ b/manifestservice/api.py @@ -6,6 +6,7 @@ import os import json + def create_app(): app = flask.Flask(__name__) app.register_blueprint(manifests_bp, url_prefix="") @@ -19,30 +20,42 @@ def create_app(): config_dict = json.loads(config_str) except Exception as e: print(e) - raise ValueError("Unable to parse the provided config file at {}".format(config_path)) - + raise ValueError( + "Unable to parse the provided config file at {}".format(config_path) + ) + for key in config_dict: app.config[key] = config_dict[key] - app.config['OIDC_ISSUER'] = 'https://%s/user' % config_dict['hostname'] - app.config['MANIFEST_BUCKET_NAME'] = config_dict['manifest_bucket_name'] + app.config["OIDC_ISSUER"] = "https://%s/user" % config_dict["hostname"] + app.config["MANIFEST_BUCKET_NAME"] = config_dict["manifest_bucket_name"] - app.config['AWS_ACCESS_KEY_ID'] = config_dict['aws_access_key_id'].strip() - app.config['AWS_SECRET_ACCESS_KEY'] = config_dict['aws_secret_access_key'].strip() + app.config["AWS_ACCESS_KEY_ID"] = config_dict["aws_access_key_id"].strip() + app.config["AWS_SECRET_ACCESS_KEY"] = config_dict["aws_secret_access_key"].strip() - os.environ['AWS_ACCESS_KEY_ID'] = config_dict['aws_access_key_id'].strip() - os.environ['AWS_SECRET_ACCESS_KEY'] = config_dict['aws_secret_access_key'].strip() + os.environ["AWS_ACCESS_KEY_ID"] = config_dict["aws_access_key_id"].strip() + os.environ["AWS_SECRET_ACCESS_KEY"] = config_dict["aws_secret_access_key"].strip() - required_config_variables = ['AWS_SECRET_ACCESS_KEY', 'AWS_ACCESS_KEY_ID', 'OIDC_ISSUER', 'MANIFEST_BUCKET_NAME'] + required_config_variables = [ + "AWS_SECRET_ACCESS_KEY", + "AWS_ACCESS_KEY_ID", + "OIDC_ISSUER", + "MANIFEST_BUCKET_NAME", + ] if not set(required_config_variables).issubset(set(app.config.keys())): - raise ValueError("Not all required config variables were provided in {}. Missing: {}".format(config_path, str( - set(required_config_variables).difference(set(app.config.keys())))) + raise ValueError( + "Not all required config variables were provided in {}. Missing: {}".format( + config_path, + str(set(required_config_variables).difference(set(app.config.keys()))), + ) ) return app + app = create_app() + @app.route("/_status", methods=["GET"]) def health_check(): """ @@ -61,5 +74,5 @@ def health_check(): def run_for_development(**kwargs): app.logger.setLevel(logging.INFO) - - app.run(**kwargs) \ No newline at end of file + + app.run(**kwargs) diff --git a/manifestservice/dev_settings.py b/manifestservice/dev_settings.py deleted file mode 100644 index 1b95a06..0000000 --- a/manifestservice/dev_settings.py +++ /dev/null @@ -1,3 +0,0 @@ -# To run locally, fill these values out -OIDC_ISSUER = "https://fence-service/user/" -MANIFEST_BUCKET_NAME = "the_name_of_the_s3_bucket_where_manifests_are_stored" \ No newline at end of file diff --git a/manifestservice/manifests/__init__.py b/manifestservice/manifests/__init__.py index 37440cb..31b0be2 100644 --- a/manifestservice/manifests/__init__.py +++ b/manifestservice/manifests/__init__.py @@ -4,15 +4,12 @@ import requests import ntpath from datetime import date, datetime -from authutils.token.validate import ( - current_token, - validate_request, - set_current_token -) +from authutils.token.validate import current_token, validate_request, set_current_token from authutils import user as authutils_user blueprint = flask.Blueprint("manifests", __name__) + @blueprint.route("/", methods=["GET"]) def get_manifests(): """ @@ -26,24 +23,25 @@ def get_manifests(): 403: description: Unauthorized """ - + err, code = _authenticate_user() if err is not None: return err, code folder_name = _get_folder_name_from_token(current_token) - result, ok = _list_files_in_bucket(flask.current_app.config.get("MANIFEST_BUCKET_NAME"), folder_name) + result, ok = _list_files_in_bucket( + flask.current_app.config.get("MANIFEST_BUCKET_NAME"), folder_name + ) if not ok: - json_to_return = {"error" : "Currently unable to connect to s3."} + json_to_return = {"error": "Currently unable to connect to s3."} return flask.jsonify(json_to_return), 500 - json_to_return = { - "manifests" : result - } + json_to_return = {"manifests": result} return flask.jsonify(json_to_return), 200 + @blueprint.route("/file/", methods=["GET"]) def get_manifest_file(file_name): """ @@ -64,20 +62,24 @@ def get_manifest_file(file_name): err, code = _authenticate_user() if err is not None: return err, code - + if not file_name.endswith("json"): - json_to_return = { "error" : "Incorrect usage. You can only use this pathway to request files of type JSON." } + json_to_return = { + "error": "Incorrect usage. You can only use this pathway to request files of type JSON." + } return flask.jsonify(json_to_return), 400 - + folder_name = _get_folder_name_from_token(current_token) - json_to_return = { - "body" : _get_file_contents(flask.current_app.config.get("MANIFEST_BUCKET_NAME"), folder_name, file_name) + "body": _get_file_contents( + flask.current_app.config.get("MANIFEST_BUCKET_NAME"), folder_name, file_name + ) } return flask.jsonify(json_to_return), 200 + @blueprint.route("/", methods=["PUT", "POST"]) def put_manifest(): """ @@ -97,51 +99,69 @@ def put_manifest(): return err, code if not flask.request.json: - return flask.jsonify({"error" : "Please provide valid JSON."}), 400 - + return flask.jsonify({"error": "Please provide valid JSON."}), 400 + manifest_json = flask.request.json required_keys = ["object_id"] is_valid = is_valid_manifest(manifest_json, required_keys) if not is_valid: - return flask.jsonify({"error" : "Manifest format is invalid. Please POST a list of key-value pairs, like [{'k' : v}, ...] Required keys are: " + " ".join(required_keys)}), 400 + return ( + flask.jsonify( + { + "error": "Manifest format is invalid. Please POST a list of key-value pairs, like [{'k' : v}, ...] Required keys are: " + + " ".join(required_keys) + } + ), + 400, + ) result, ok = _add_manifest_to_bucket(current_token, manifest_json) if not ok: - json_to_return = {"error" : "Currently unable to connect to s3."} + json_to_return = {"error": "Currently unable to connect to s3."} return flask.jsonify(json_to_return), 500 - ret = { - "filename": result, - } + ret = {"filename": result} return flask.jsonify(ret), 200 + def _add_manifest_to_bucket(current_token, manifest_json): """ Puts the manifest_json string into a file and uploads it to s3. Generates and returns the name of the new file. """ - session = boto3.Session(region_name="us-east-1", aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'], aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY']) + session = boto3.Session( + region_name="us-east-1", + aws_access_key_id=app.config["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=app.config["AWS_SECRET_ACCESS_KEY"], + ) s3 = session.resource("s3") folder_name = _get_folder_name_from_token(current_token) - result, ok = _list_files_in_bucket(flask.current_app.config.get("MANIFEST_BUCKET_NAME"), folder_name) + result, ok = _list_files_in_bucket( + flask.current_app.config.get("MANIFEST_BUCKET_NAME"), folder_name + ) if not ok: return result, False - filename = _generate_unique_manifest_filename(folder_name, flask.current_app.config.get("MANIFEST_BUCKET_NAME"), result) + filename = _generate_unique_manifest_filename( + folder_name, flask.current_app.config.get("MANIFEST_BUCKET_NAME"), result + ) manifest_as_bytes = str.encode(str(flask.request.json)) filepath_in_bucket = folder_name + "/" + filename try: - obj = s3.Object(flask.current_app.config.get("MANIFEST_BUCKET_NAME"), filepath_in_bucket) + obj = s3.Object( + flask.current_app.config.get("MANIFEST_BUCKET_NAME"), filepath_in_bucket + ) response = obj.put(Body=manifest_as_bytes) except Exception as e: return str(e), False return filename, True + def _get_folder_name_from_token(user_info): """ Returns the name of the user's manifest folder (their "prefix"). @@ -153,24 +173,26 @@ def _get_folder_name_from_token(user_info): """ return "user-" + str(user_info["sub"]) + def _does_the_user_have_read_access_on_at_least_one_project(project_access_dict): """ Returns True if the user has both read and read-storage access on at least one project, False otherwise. """ privileges = list(project_access_dict.values()) - + for auth_set in privileges: if "read" in auth_set and "read-storage" in auth_set: return True return False + def is_valid_manifest(manifest_json, required_keys): """ - Returns (True, "") if the manifest.json is a list of the form [{'k' : v}, ...], - where valid keys are object_id and subject_id. - Otherwise, returns (False, error_msg) + Returns True if the manifest.json is a list of the form [{'k' : v}, ...], + where each member dictionary contains an object_id key. + Otherwise, returns False """ for record in manifest_json: record_keys = record.keys() @@ -179,17 +201,25 @@ def is_valid_manifest(manifest_json, required_keys): return True -def _generate_unique_manifest_filename(folder_name, manifest_bucket_name, users_existing_manifest_files): + +def _generate_unique_manifest_filename( + folder_name, manifest_bucket_name, users_existing_manifest_files +): """ Returns a filename of the form manifest--.json that is unique among the files in the user's manifest folder. """ timestamp = datetime.now().isoformat() - existing_filenames = map(lambda x: x['filename'], users_existing_manifest_files) - filename = _generate_unique_filename_with_timestamp_and_increment(timestamp, existing_filenames) + existing_filenames = map(lambda x: x["filename"], users_existing_manifest_files) + filename = _generate_unique_filename_with_timestamp_and_increment( + timestamp, existing_filenames + ) return filename -def _generate_unique_filename_with_timestamp_and_increment(timestamp, users_existing_manifest_files): + +def _generate_unique_filename_with_timestamp_and_increment( + timestamp, users_existing_manifest_files +): """ A helper function for _generate_unique_manifest_filename(), which facilitates unit testing. Adds an increment to the filename if there happens to be another timestamped file with the same name @@ -208,11 +238,16 @@ def _generate_unique_filename_with_timestamp_and_increment(timestamp, users_exis return filename + def _list_files_in_bucket(bucket_name, folder): """ Lists the files in an s3 bucket. Returns a list of filenames. """ - session = boto3.Session(region_name="us-east-1", aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'], aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY']) + session = boto3.Session( + region_name="us-east-1", + aws_access_key_id=app.config["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=app.config["AWS_SECRET_ACCESS_KEY"], + ) s3 = session.resource("s3") rv = [] @@ -222,25 +257,33 @@ def _list_files_in_bucket(bucket_name, folder): bucket_objects = bucket.objects.filter(Prefix=folder + "/") for object_summary in bucket_objects: manifest_summary = { - "filename" : ntpath.basename(object_summary.key), - "last_modified" : object_summary.last_modified.strftime("%Y-%m-%d %H:%M:%S") + "filename": ntpath.basename(object_summary.key), + "last_modified": object_summary.last_modified.strftime( + "%Y-%m-%d %H:%M:%S" + ), } rv.append(manifest_summary) except Exception as e: print(e) return str(e), False - + return rv, True + def _get_file_contents(bucket_name, folder, filename): """ Returns the body of a requested file as a string. """ - client = boto3.client("s3", aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'], aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY']) + client = boto3.client( + "s3", + aws_access_key_id=app.config["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=app.config["AWS_SECRET_ACCESS_KEY"], + ) obj = client.get_object(Bucket=bucket_name, Key=folder + "/" + filename) as_bytes = obj["Body"].read() as_string = as_bytes.decode("utf-8") - return as_string.replace("'", "\"") + return as_string.replace("'", '"') + def _authenticate_user(): """ @@ -251,14 +294,18 @@ def _authenticate_user(): set_current_token(validate_request(aud={"user"})) except Exception as e: print(e) - json_to_return = { "error" : "Please log in." } + json_to_return = {"error": "Please log in."} return flask.jsonify(json_to_return), 403 try: - auth_successful = _does_the_user_have_read_access_on_at_least_one_project(authutils_user.current_user.projects) + auth_successful = _does_the_user_have_read_access_on_at_least_one_project( + authutils_user.current_user.projects + ) except Exception as e: print(e) - json_to_return = { "error" : "You must have read access on at least one project in order to use this feature." } + json_to_return = { + "error": "You must have read access on at least one project in order to use this feature." + } return flask.jsonify(json_to_return), 403 - return None, None \ No newline at end of file + return None, None diff --git a/setup.py b/setup.py index 0c2c717..f21da33 100644 --- a/setup.py +++ b/setup.py @@ -15,5 +15,4 @@ url="https://github.com/uc-cdis/manifestservice", license="Apache", packages=find_packages(), - ) diff --git a/tests/app_test.py b/tests/app_test.py index 8bf2c5e..46f655b 100644 --- a/tests/app_test.py +++ b/tests/app_test.py @@ -8,179 +8,246 @@ mocks = {} + @pytest.fixture def app(mocker): - test_user = { - 'context': { - 'user': {'policies': ['data_upload', 'programs.test-read-storage', 'programs.test-read'], - 'google': {'proxy_group': None}, - 'is_admin': True, - 'name': 'example@uchicago.edu', - 'projects': - {'test': - ['read-storage', 'read', 'create', 'write-storage', 'upload', 'update', 'delete'] - } - } - }, - 'aud': ['data', 'user', 'fence', 'openid'], - 'sub': '18' - } - - mocks['current_token'] = mocker.patch("manifestservice.manifests.current_token", return_value=test_user) - - mocks['_authenticate_user'] = mocker.patch("manifestservice.manifests._authenticate_user", return_value=(None, 200)) - - mocks['_list_files_in_bucket'] = mocker.patch("manifestservice.manifests._list_files_in_bucket", return_value=([{ 'filename':'manifest-a-b-c.json' } ], True)) - - mocks['_add_manifest_to_bucket'] = mocker.patch("manifestservice.manifests._add_manifest_to_bucket", return_value=("manifest-xxx.json", True)) + test_user = { + "context": { + "user": { + "policies": [ + "data_upload", + "programs.test-read-storage", + "programs.test-read", + ], + "google": {"proxy_group": None}, + "is_admin": True, + "name": "example@uchicago.edu", + "projects": { + "test": [ + "read-storage", + "read", + "create", + "write-storage", + "upload", + "update", + "delete", + ] + }, + } + }, + "aud": ["data", "user", "fence", "openid"], + "sub": "18", + } + + mocks["current_token"] = mocker.patch( + "manifestservice.manifests.current_token", return_value=test_user + ) + + mocks["_authenticate_user"] = mocker.patch( + "manifestservice.manifests._authenticate_user", return_value=(None, 200) + ) + + mocks["_list_files_in_bucket"] = mocker.patch( + "manifestservice.manifests._list_files_in_bucket", + return_value=([{"filename": "manifest-a-b-c.json"}], True), + ) + + mocks["_add_manifest_to_bucket"] = mocker.patch( + "manifestservice.manifests._add_manifest_to_bucket", + return_value=("manifest-xxx.json", True), + ) + + mocks["_get_file_contents"] = mocker.patch( + "manifestservice.manifests._get_file_contents", return_value="" + ) + + app = create_app() + return app - mocks['_get_file_contents'] = mocker.patch("manifestservice.manifests._get_file_contents", return_value='') - - app = create_app() - return app def test_generate_unique_manifest_filename_basic_date_generation(): - """ + """ Tests that the _generate_unique_filename_with_timestamp_and_increment() function generates a unique filename containing the given timestamp, based on the files in the user's bucket. """ - timestamp = "a-b-c" - users_existing_manifest_files = [] - filename = manifests._generate_unique_filename_with_timestamp_and_increment(timestamp, users_existing_manifest_files) - assert filename == "manifest-a-b-c.json" - - timestamp = "a-b-c" - users_existing_manifest_files = ["some-other-file.txt", "another-file.json"] - filename = manifests._generate_unique_filename_with_timestamp_and_increment(timestamp, users_existing_manifest_files) - assert filename == "manifest-a-b-c.json" - - # Case 1: One collision - timestamp = "a-b-c" - users_existing_manifest_files = ["manifest-a-b-c.json"] - filename = manifests._generate_unique_filename_with_timestamp_and_increment(timestamp, users_existing_manifest_files) - assert filename == "manifest-a-b-c-1.json" - - # Case 2: Two collisions - timestamp = "a-b-c" - users_existing_manifest_files = ["manifest-a-b-c.json", "manifest-a-b-c-1.json"] - filename = manifests._generate_unique_filename_with_timestamp_and_increment(timestamp, users_existing_manifest_files) - assert filename == "manifest-a-b-c-2.json" - - # Case 3: Three collisions. This should never ever happen but eh might as well test it. - timestamp = "a-b-c" - users_existing_manifest_files = ["manifest-a-b-c.json", "manifest-a-b-c-1.json", "manifest-a-b-c-2.json"] - filename = manifests._generate_unique_filename_with_timestamp_and_increment(timestamp, users_existing_manifest_files) - assert filename == "manifest-a-b-c-3.json" + timestamp = "a-b-c" + users_existing_manifest_files = [] + filename = manifests._generate_unique_filename_with_timestamp_and_increment( + timestamp, users_existing_manifest_files + ) + assert filename == "manifest-a-b-c.json" + + timestamp = "a-b-c" + users_existing_manifest_files = ["some-other-file.txt", "another-file.json"] + filename = manifests._generate_unique_filename_with_timestamp_and_increment( + timestamp, users_existing_manifest_files + ) + assert filename == "manifest-a-b-c.json" + + # Case 1: One collision + timestamp = "a-b-c" + users_existing_manifest_files = ["manifest-a-b-c.json"] + filename = manifests._generate_unique_filename_with_timestamp_and_increment( + timestamp, users_existing_manifest_files + ) + assert filename == "manifest-a-b-c-1.json" + + # Case 2: Two collisions + timestamp = "a-b-c" + users_existing_manifest_files = ["manifest-a-b-c.json", "manifest-a-b-c-1.json"] + filename = manifests._generate_unique_filename_with_timestamp_and_increment( + timestamp, users_existing_manifest_files + ) + assert filename == "manifest-a-b-c-2.json" + + # Case 3: Three collisions. This should never ever happen but eh might as well test it. + timestamp = "a-b-c" + users_existing_manifest_files = [ + "manifest-a-b-c.json", + "manifest-a-b-c-1.json", + "manifest-a-b-c-2.json", + ] + filename = manifests._generate_unique_filename_with_timestamp_and_increment( + timestamp, users_existing_manifest_files + ) + assert filename == "manifest-a-b-c-3.json" + def test__does_the_user_have_read_access_on_at_least_one_project(): - """ + """ Tests that the function _does_the_user_have_read_access_on_at_least_one_project() provides the correct value for different arborist user_info inputs. """ - project_access_dict = { } - rv = manifests._does_the_user_have_read_access_on_at_least_one_project(project_access_dict) - assert rv is False + project_access_dict = {} + rv = manifests._does_the_user_have_read_access_on_at_least_one_project( + project_access_dict + ) + assert rv is False + + project_access_dict = {"test": ["read-storage", "write-storage", "read"], "DEV": []} + rv = manifests._does_the_user_have_read_access_on_at_least_one_project( + project_access_dict + ) + assert rv is True + + project_access_dict = { + "test": ["write-storage", "read"], + "abc123": ["something", "something-else"], + } + rv = manifests._does_the_user_have_read_access_on_at_least_one_project( + project_access_dict + ) + assert rv is False + + # You need both read and read-storage to use this service. + project_access_dict = { + "jenkins": ["read"], + "abc123": ["something", "something-else"], + } + rv = manifests._does_the_user_have_read_access_on_at_least_one_project( + project_access_dict + ) + assert rv is False - project_access_dict = {'test' : [ 'read-storage' , 'write-storage', 'read' ], 'DEV' : [] } - rv = manifests._does_the_user_have_read_access_on_at_least_one_project(project_access_dict) - assert rv is True - - project_access_dict = {'test' : [ 'write-storage', 'read' ] , 'abc123' : ['something', 'something-else'] } - rv = manifests._does_the_user_have_read_access_on_at_least_one_project(project_access_dict) - assert rv is False - - # You need both read and read-storage to use this service. - project_access_dict ={'jenkins' : [ 'read' ] , 'abc123' : ['something', 'something-else'] } - rv = manifests._does_the_user_have_read_access_on_at_least_one_project(project_access_dict) - assert rv is False def test_is_valid_manifest(): - """ + """ Tests that the function is_valid_manifest() correctly determines if the input manifest string is valid. """ - required_keys = ["object_id"] - test_manifest = [{ "foo" : 44 }] - is_valid = manifests.is_valid_manifest(test_manifest, required_keys) - assert is_valid is False + required_keys = ["object_id"] + test_manifest = [{"foo": 44}] + is_valid = manifests.is_valid_manifest(test_manifest, required_keys) + assert is_valid is False - test_manifest = [{ "foo" : 44 , "bar" : 88 }] - is_valid = manifests.is_valid_manifest(test_manifest, required_keys) - assert is_valid is False + test_manifest = [{"foo": 44, "bar": 88}] + is_valid = manifests.is_valid_manifest(test_manifest, required_keys) + assert is_valid is False - test_manifest = [{ "foo" : 44 , "object_id" : 88 }] - is_valid = manifests.is_valid_manifest(test_manifest, required_keys) - assert is_valid is True + test_manifest = [{"foo": 44, "object_id": 88}] + is_valid = manifests.is_valid_manifest(test_manifest, required_keys) + assert is_valid is True - test_manifest = [{ "subject_id" : 44 , "object_id" : 88 }] - is_valid = manifests.is_valid_manifest(test_manifest, required_keys) - assert is_valid is True + test_manifest = [{"subject_id": 44, "object_id": 88}] + is_valid = manifests.is_valid_manifest(test_manifest, required_keys) + assert is_valid is True + + test_manifest = [{"object_id": 88}] + is_valid = manifests.is_valid_manifest(test_manifest, required_keys) + assert is_valid is True - test_manifest = [{ "object_id" : 88 }] - is_valid = manifests.is_valid_manifest(test_manifest, required_keys) - assert is_valid is True def test_POST_handles_invalid_json(client): - """ + """ Test that we get a 400 if flask.request.json is not filled in. """ - r = client.post("/", data={'a':1}) - assert r.status_code == 400 + r = client.post("/", data={"a": 1}) + assert r.status_code == 400 + def test_POST_handles_invalid_manifest_keys(client): - """ + """ Test that we get a 400 if the manifest is missing the required key -- object_id. """ - test_manifest = [{ 'foo' : 44 , "bar" : 88 }] - headers = {'Content-Type': 'application/json', 'Accept':'application/json'} - r = client.post("/", json=test_manifest, headers=headers) - assert r.status_code == 400 + test_manifest = [{"foo": 44, "bar": 88}] + headers = {"Content-Type": "application/json", "Accept": "application/json"} + r = client.post("/", json=test_manifest, headers=headers) + assert r.status_code == 400 + + test_manifest = [{"obj__id": 44, "subject_id": 88}] + r = client.post("/", json=test_manifest, headers=headers) + assert r.status_code == 400 - test_manifest = [{ 'obj__id' : 44 , "subject_id" : 88 }] - r = client.post("/", json=test_manifest, headers=headers) - assert r.status_code == 400 def test_POST_successful_manifest_upload(client): - """ + """ Test the full user pathway: a manifest is created, listed, and then downloaded. Unfortunately, we cannot verify here that the manifest is present in the listed files, nor that the filebody is correct, as that would require a real s3 connection. Instead, s3 is mocked and we assert that the correct functions are called. """ - import random - - random_nums = [ random.randint(1,101) , random.randint(1,101) , random.randint(1,101) , random.randint(1,101) ] - test_manifest = [{ "subject_id" : random_nums[0] , "object_id" : random_nums[1] }, { "subject_id" : random_nums[2] , "object_id" : random_nums[3] }] - - headers = {'Content-Type': 'application/json', 'Accept':'application/json'} - r = client.post("/", data=json_utils.dumps(test_manifest), headers=headers) - - assert r.status_code == 200 - assert mocks['_authenticate_user'].call_count == 1 - assert mocks['_add_manifest_to_bucket'].call_count == 1 - assert mocks['_get_file_contents'].call_count == 0 - - json = r.json - new_filename = json['filename'] - - assert new_filename is not None - assert type(new_filename) is str - - r = client.get("/", headers=headers) - assert r.status_code == 200 - assert mocks['_authenticate_user'].call_count == 2 - assert mocks['_add_manifest_to_bucket'].call_count == 1 - assert mocks['_list_files_in_bucket'].call_count == 1 - assert mocks['_get_file_contents'].call_count == 0 - - json = r.json - manifest_files = json['manifests'] - assert type(manifest_files) is list - - r = client.get("/file/" + new_filename, headers=headers) - assert r.status_code == 200 - assert mocks['_authenticate_user'].call_count == 3 - assert mocks['_add_manifest_to_bucket'].call_count == 1 - assert mocks['_list_files_in_bucket'].call_count == 1 - assert mocks['_get_file_contents'].call_count == 1 \ No newline at end of file + import random + + random_nums = [ + random.randint(1, 101), + random.randint(1, 101), + random.randint(1, 101), + random.randint(1, 101), + ] + test_manifest = [ + {"subject_id": random_nums[0], "object_id": random_nums[1]}, + {"subject_id": random_nums[2], "object_id": random_nums[3]}, + ] + + headers = {"Content-Type": "application/json", "Accept": "application/json"} + r = client.post("/", data=json_utils.dumps(test_manifest), headers=headers) + + assert r.status_code == 200 + assert mocks["_authenticate_user"].call_count == 1 + assert mocks["_add_manifest_to_bucket"].call_count == 1 + assert mocks["_get_file_contents"].call_count == 0 + + json = r.json + new_filename = json["filename"] + + assert new_filename is not None + assert type(new_filename) is str + + r = client.get("/", headers=headers) + assert r.status_code == 200 + assert mocks["_authenticate_user"].call_count == 2 + assert mocks["_add_manifest_to_bucket"].call_count == 1 + assert mocks["_list_files_in_bucket"].call_count == 1 + assert mocks["_get_file_contents"].call_count == 0 + + json = r.json + manifest_files = json["manifests"] + assert type(manifest_files) is list + + r = client.get("/file/" + new_filename, headers=headers) + assert r.status_code == 200 + assert mocks["_authenticate_user"].call_count == 3 + assert mocks["_add_manifest_to_bucket"].call_count == 1 + assert mocks["_list_files_in_bucket"].call_count == 1 + assert mocks["_get_file_contents"].call_count == 1 diff --git a/tests/conftest.py b/tests/conftest.py index d2f2312..efd8b2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,10 +2,10 @@ from manifestservice.api import app as service_app + @pytest.fixture(scope="session") def app(): # load configuration # service_app.config.from_object('manifestservice.test_settings') - #app_init(service_app) + # app_init(service_app) return service_app -