flask to 1.1.2 (#15)

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+-   repo: [email protected]:Yelp/detect-secrets
+    rev: v0.13.1
+    hooks:
+    -   id: detect-secrets
+        args: ['--baseline', '.secrets.baseline']
+        exclude: Pipfile.lock
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: no-commit-to-branch
+        args: [--branch, develop, --branch, master, --pattern, release/.*]
+-   repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+    -   id: black

.secrets.baseline

@@ -0,0 +1,75 @@
+{
+  "exclude": {
+    "files": "Pipfile.lock",
+    "lines": null
+  },
+  "generated_at": "2020-10-19T22:23:04Z",
+  "plugins_used": [
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "base64_limit": 4.5,
+      "name": "Base64HighEntropyString"
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "hex_limit": 3,
+      "name": "HexHighEntropyString"
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "keyword_exclude": null,
+      "name": "KeywordDetector"
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "results": {
+    "config.json": [
+      {
+        "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "is_verified": false,
+        "line_number": 3,
+        "type": "Secret Keyword"
+      }
+    ]
+  },
+  "version": "0.13.1",
+  "word_list": {
+    "file": null,
+    "hash": null
+  }
+}

.travis.yml

@@ -20,4 +20,4 @@ script:
   - python -m pytest -v
 
 after_script:
-  - pipenv run python-codacy-coverage -r coverage.xml
+  - pipenv run python-codacy-coverage -r coverage.xml

Jenkinsfile

@@ -2,5 +2,5 @@
 
 @Library('cdis-jenkins-lib@master') _
 
-testPipeline { 
+testPipeline {
 }

Pipfile

@@ -10,7 +10,7 @@ pytest-mock="~=1.10"
 pytest-flask="~=0.15.0"
 
 [packages]
-flask="~=0.12.4"
+flask="~=1.1.2"
 authutils="~=3.1.0"
 boto3="~=1.9"
 cryptography="==2.8"

README.md

@@ -2,8 +2,8 @@
 ### Overview
 This service handles reading from and writing to a user's s3 folder containing their manifests. A manifest is a JSON file that lists records a researcher may be interested in analyzing. This service stores a manifest to a user folder in an s3 bucket and delivers it for later use, such as when the researcher wants to mount the manifest in their workspace. If the "prefix" config variable is set, user folders will be stored in a directory of that name within the s3 bucket.
 
-Manifest files should contain JSON of the form 
-    
+Manifest files should contain JSON of the form
+
     [
       {
         "object_id": "757508f5-2697-4700-a69f-89d173a4c514",
@@ -21,7 +21,7 @@ Manifest files should contain JSON of the form
 For all endpoints, the request must contain an Authorization header with an access_token. The user needs read access and read-storage access
 on at least one project in order to use this service.
 
-Lists a user's manifests: 
+Lists a user's manifests:
 
     GET /
     Returns: { "manifests" : [ { "filename" : "manifest-2019-02-27T11-44-20.548126.json", "last_modified" : "2019-02-27 17:44:21" }, ... ] }
@@ -37,8 +37,8 @@ Read the contents of a manifest file in the user's folder:
     GET /file/<filename.json>
     Returns: { "body" : "the-body-of-the-manifest-file-as-a-string" }
 
-On failure, the above endpoints all return JSON in the form 
-    
+On failure, the above endpoints all return JSON in the form
+
     { "error" : "error-message" }
 
 ### Running the service locally
@@ -49,4 +49,4 @@ If you want to run this service locally, fill out the config.json file with the
 
 And then GET and POST to http://localhost:5000/
 
-You'll need AWS credentials in your environment to run this locally.
+You'll need AWS credentials in your environment to run this locally.

config.json

@@ -4,4 +4,4 @@
     "manifest_bucket_name" : "",
     "hostname": "",
     "prefix": ""
-}
+}

manifestservice/manifests/__init__.py

@@ -17,7 +17,7 @@
 def get_manifests():
     """
     Returns a list of filenames corresponding to the user's manifests.
-    We find the appropriate folder ("prefix") in the bucket by asking Fence for 
+    We find the appropriate folder ("prefix") in the bucket by asking Fence for
     info about the user's access token.
     ---
     responses:
@@ -52,7 +52,7 @@ def get_manifest_file(file_name):
     The argument is the filename of the manifest you want to downloaded,
     of the form "manifest-timestamp".json. The user folder prefix is encapsulated from
     the caller -- just provide the basepath.
-    ---    
+    ---
     responses:
         200:
             description: Success
@@ -126,7 +126,7 @@ def put_manifest():
 
 def _add_manifest_to_bucket(current_token, manifest_json):
     """
-    Puts the manifest_json string into a file and uploads it to s3. 
+    Puts the manifest_json string into a file and uploads it to s3.
     Generates and returns the name of the new file.
     """
     session = boto3.Session(
@@ -166,20 +166,20 @@ def _get_folder_name_from_token(user_info):
     """
     Returns the name of the user's manifest folder (their "prefix").
     It takes a "user_info" dict, which is the response that Fence returns at /user/user
-    The convention we'll use here is that a user's folder name will be "user-x" where x is 
+    The convention we'll use here is that a user's folder name will be "user-x" where x is
     their ID (integer).
 
-    According to the revproxy's helpers.js, it looks like the user_id is stored in a variable called "sub". Hm. 
+    According to the revproxy's helpers.js, it looks like the user_id is stored in a variable called "sub". Hm.
     """
-    result =  "user-" + str(user_info["sub"])
+    result = "user-" + str(user_info["sub"])
     if "PREFIX" in app.config:
-        result =  app.config["PREFIX"] + "/user-" + str(user_info["sub"])
+        result = app.config["PREFIX"] + "/user-" + str(user_info["sub"])
     return result
 
 
 def is_valid_manifest(manifest_json, required_keys):
     """
-    Returns True if the manifest.json is a list of the form [{'k' : v}, ...], 
+    Returns True if the manifest.json is a list of the form [{'k' : v}, ...],
     where each member dictionary contains an object_id key.
     Otherwise, returns False
     """
@@ -212,7 +212,7 @@ def _generate_unique_filename_with_timestamp_and_increment(
     """
     A helper function for _generate_unique_manifest_filename(), which facilitates unit testing.
     Adds an increment to the filename if there happens to be another timestamped file with the same name
-    (unlikely, but good to check). 
+    (unlikely, but good to check).
     """
     filename_without_extension = "manifest-" + timestamp.replace(":", "-")
     extension = ".json"
@@ -265,7 +265,7 @@ def _list_files_in_bucket(bucket_name, folder):
 
 def _get_file_contents(bucket_name, folder, filename):
     """
-    Returns the body of a requested file as a string. 
+    Returns the body of a requested file as a string.
     """
     client = boto3.client(
         "s3",

tests/app_test.py

@@ -67,10 +67,10 @@ def app(mocker):
 
 def test_generate_unique_manifest_filename_basic_date_generation():
     """
-	Tests that the _generate_unique_filename_with_timestamp_and_increment() function
-	generates a unique filename containing the given timestamp, based on the files in the
-	user's bucket. 
-	"""
+    Tests that the _generate_unique_filename_with_timestamp_and_increment() function
+    generates a unique filename containing the given timestamp, based on the files in the
+    user's bucket.
+    """
     timestamp = "a-b-c"
     users_existing_manifest_files = []
     filename = manifests._generate_unique_filename_with_timestamp_and_increment(
@@ -116,9 +116,9 @@ def test_generate_unique_manifest_filename_basic_date_generation():
 
 def test_is_valid_manifest():
     """
-	Tests that the function is_valid_manifest() correctly determines
-	if the input manifest string is valid.
-	"""
+    Tests that the function is_valid_manifest() correctly determines
+    if the input manifest string is valid.
+    """
     required_keys = ["object_id"]
     test_manifest = [{"foo": 44}]
     is_valid = manifests.is_valid_manifest(test_manifest, required_keys)
@@ -143,16 +143,16 @@ def test_is_valid_manifest():
 
 def test_POST_handles_invalid_json(client):
     """
-	Test that we get a 400 if flask.request.json is not filled in.
-	"""
+    Test that we get a 400 if flask.request.json is not filled in.
+    """
     r = client.post("/", data={"a": 1})
     assert r.status_code == 400
 
 
 def test_POST_handles_invalid_manifest_keys(client):
     """
-	Test that we get a 400 if the manifest is missing the required key -- object_id.
-	"""
+    Test that we get a 400 if the manifest is missing the required key -- object_id.
+    """
     test_manifest = [{"foo": 44, "bar": 88}]
     headers = {"Content-Type": "application/json", "Accept": "application/json"}
     r = client.post("/", json=test_manifest, headers=headers)
@@ -165,11 +165,11 @@ def test_POST_handles_invalid_manifest_keys(client):
 
 def test_POST_successful_manifest_upload(client):
     """
-	Test the full user pathway: a manifest is created, listed, and then downloaded.
-	Unfortunately, we cannot verify here that the manifest is present in the listed files, 
-	nor that the filebody is correct, as that would require a real s3 connection.
-	Instead, s3 is mocked and we assert that the correct functions are called. 
-	"""
+    Test the full user pathway: a manifest is created, listed, and then downloaded.
+    Unfortunately, we cannot verify here that the manifest is present in the listed files,
+    nor that the filebody is correct, as that would require a real s3 connection.
+    Instead, s3 is mocked and we assert that the correct functions are called.
+    """
     import random
 
     random_nums = [