From e8d767686ef9299c1396451a36942b01cccf3e8a Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 18:21:03 -0600 Subject: [PATCH 01/10] feat: First iteration of local auth; need to update GCP --- .gitignore | 1 + policyengine_api_light/auth/validation.py | 22 +++++++++++++++++++ policyengine_api_light/endpoints/household.py | 16 +++++++++++++- setup.py | 2 ++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 policyengine_api_light/auth/validation.py diff --git a/.gitignore b/.gitignore index a08ce8e..ddcdcfb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dist/* **/*.rdb **/*.h5 **/*.csv.gz +.env \ No newline at end of file diff --git a/policyengine_api_light/auth/validation.py b/policyengine_api_light/auth/validation.py new file mode 100644 index 0000000..5cbfb61 --- /dev/null +++ b/policyengine_api_light/auth/validation.py @@ -0,0 +1,22 @@ +import json +from urllib.request import urlopen + +from authlib.oauth2.rfc7523 import JWTBearerTokenValidator +from authlib.jose.rfc7517.jwk import JsonWebKey + + +class Auth0JWTBearerTokenValidator(JWTBearerTokenValidator): + def __init__(self, domain, audience): + issuer = f"https://{domain}/" + jsonurl = urlopen(f"{issuer}.well-known/jwks.json") + public_key = JsonWebKey.import_key_set( + json.loads(jsonurl.read()) + ) + super(Auth0JWTBearerTokenValidator, self).__init__( + public_key + ) + self.claims_options = { + "exp": {"essential": True}, + "aud": {"essential": True, "value": audience}, + "iss": {"essential": True, "value": issuer}, + } \ No newline at end of file diff --git a/policyengine_api_light/endpoints/household.py b/policyengine_api_light/endpoints/household.py index 8f798c2..a91a811 100644 --- a/policyengine_api_light/endpoints/household.py +++ b/policyengine_api_light/endpoints/household.py @@ -2,11 +2,25 @@ COUNTRIES, validate_country, ) +import os import json from flask import Response, request from policyengine_api_light.country import COUNTRIES import json import logging +from dotenv import load_dotenv, find_dotenv +from authlib.integrations.flask_oauth2 import ResourceProtector +from ..auth.validation import Auth0JWTBearerTokenValidator + +load_dotenv() + +# Configure authentication +require_auth = ResourceProtector() +validator = Auth0JWTBearerTokenValidator( + os.getenv("AUTH0_ADDRESS_NO_DOMAIN"), + os.getenv("AUTH0_AUDIENCE_NO_DOMAIN") +) +require_auth.register_token_validator(validator) def add_yearly_variables(household, country_id): @@ -42,7 +56,7 @@ def add_yearly_variables(household, country_id): ] = {2023: None} return household - +@require_auth(None) def get_calculate(country_id: str, add_missing: bool = False) -> dict: """Lightweight endpoint for passing in household and policy JSON objects and calculating without storing data. diff --git a/setup.py b/setup.py index 03a66c6..a6e8407 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ description="PolicyEngine API Light", packages=find_packages(), install_requires=[ + "Authlib<1.3.0", "flask>=1", "flask-cors>=3", "google-cloud-logging", @@ -21,6 +22,7 @@ "policyengine_us==0.571.2", "Flask-Caching==2.0.2", "urllib3<1.27,>=1.21.1", + "python-dotenv" ], extras_require={ "dev": [ From ff3829e8b468ea9231f8a11ad43a747875c24b13 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 18:26:03 -0600 Subject: [PATCH 02/10] feat: Update PR and push GitHub actions for new env secrets --- .github/workflows/pr.yml | 3 +++ .github/workflows/push.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 453c52a..13ec947 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -49,3 +49,6 @@ jobs: run: make install - name: Test the API run: make test + env: + AUTH0_ADDRESS_NO_DOMAIN: ${{ secrets.AUTH0_ADDRESS_NO_DOMAIN }} + AUTH0_AUDIENCE_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3e7c2c5..ce29f6a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -68,3 +68,5 @@ jobs: run: make deploy env: GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SA_KEY }} + AUTH0_ADDRESS_NO_DOMAIN: ${{ secrets.AUTH0_ADDRESS_NO_DOMAIN }} + AUTH0_AUDIENCE_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} From 163a0a68ced6a092b154859aacc307e534910047 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 18:35:11 -0600 Subject: [PATCH 03/10] feat: Update GCP env secret loading --- gcp/export.py | 22 ++++++++++++++++++++-- gcp/policyengine_api_light/Dockerfile | 2 ++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/gcp/export.py b/gcp/export.py index 7c7a930..1369a67 100644 --- a/gcp/export.py +++ b/gcp/export.py @@ -2,15 +2,33 @@ from pathlib import Path GAE = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] -# If it's a filepath, read the file. Otherwise, it'll be JSON +# If it's a filepath, read the file. Otherwise, it'll be text try: Path(GAE).resolve(strict=True) with open(GAE, "r") as f: GAE = f.read() except Exception as e: pass +ADDRESS = os.environ["AUTH0_ADDRESS_NO_DOMAIN"] +AUDIENCE = os.environ["AUTH0_AUDIENCE_NO_DOMAIN"] -# Export GAE to to .gac.json and DB_PD to .dbpw in the current directory +# Export GAE to to .gac.json with open(".gac.json", "w") as f: f.write(GAE) + +# in gcp/policyengine_api_light/Dockerfile, replace env variables +for dockerfile_location in [ + "gcp/policyengine_api_light/Dockerfile", +]: + with open(dockerfile_location, "r") as f: + dockerfile = f.read() + dockerfile = dockerfile.replace( + ".address", ADDRESS + ) + dockerfile = dockerfile.replace( + ".audience", AUDIENCE + ) + + with open(dockerfile_location, "w") as f: + f.write(dockerfile) diff --git a/gcp/policyengine_api_light/Dockerfile b/gcp/policyengine_api_light/Dockerfile index f45ca82..8a57bd0 100644 --- a/gcp/policyengine_api_light/Dockerfile +++ b/gcp/policyengine_api_light/Dockerfile @@ -1,5 +1,7 @@ FROM anthvolk/policyengine-api-light:latest ENV GOOGLE_APPLICATION_CREDENTIALS .gac.json +ENV AUTH0_ADDRESS_NO_DOMAIN .address +ENV AUTH0_AUDIENCE_NO_DOMAIN .audience ADD . /app RUN cd /app && make install && make test CMD ./start.sh From b500bf0ff651f24b06353e309fbdc107e7a42ef0 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 18:36:27 -0600 Subject: [PATCH 04/10] chore: Lint & changelog --- changelog_entry.yaml | 4 ++++ gcp/export.py | 10 +++------- policyengine_api_light/auth/validation.py | 10 +++------- policyengine_api_light/endpoints/household.py | 4 ++-- setup.py | 2 +- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29..64576a3 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,4 @@ +- bump: minor + changes: + added: + - Added authentication via auth0 \ No newline at end of file diff --git a/gcp/export.py b/gcp/export.py index 1369a67..2279b5c 100644 --- a/gcp/export.py +++ b/gcp/export.py @@ -23,12 +23,8 @@ ]: with open(dockerfile_location, "r") as f: dockerfile = f.read() - dockerfile = dockerfile.replace( - ".address", ADDRESS - ) - dockerfile = dockerfile.replace( - ".audience", AUDIENCE - ) - + dockerfile = dockerfile.replace(".address", ADDRESS) + dockerfile = dockerfile.replace(".audience", AUDIENCE) + with open(dockerfile_location, "w") as f: f.write(dockerfile) diff --git a/policyengine_api_light/auth/validation.py b/policyengine_api_light/auth/validation.py index 5cbfb61..d5512f2 100644 --- a/policyengine_api_light/auth/validation.py +++ b/policyengine_api_light/auth/validation.py @@ -9,14 +9,10 @@ class Auth0JWTBearerTokenValidator(JWTBearerTokenValidator): def __init__(self, domain, audience): issuer = f"https://{domain}/" jsonurl = urlopen(f"{issuer}.well-known/jwks.json") - public_key = JsonWebKey.import_key_set( - json.loads(jsonurl.read()) - ) - super(Auth0JWTBearerTokenValidator, self).__init__( - public_key - ) + public_key = JsonWebKey.import_key_set(json.loads(jsonurl.read())) + super(Auth0JWTBearerTokenValidator, self).__init__(public_key) self.claims_options = { "exp": {"essential": True}, "aud": {"essential": True, "value": audience}, "iss": {"essential": True, "value": issuer}, - } \ No newline at end of file + } diff --git a/policyengine_api_light/endpoints/household.py b/policyengine_api_light/endpoints/household.py index a91a811..3d8bf00 100644 --- a/policyengine_api_light/endpoints/household.py +++ b/policyengine_api_light/endpoints/household.py @@ -17,8 +17,7 @@ # Configure authentication require_auth = ResourceProtector() validator = Auth0JWTBearerTokenValidator( - os.getenv("AUTH0_ADDRESS_NO_DOMAIN"), - os.getenv("AUTH0_AUDIENCE_NO_DOMAIN") + os.getenv("AUTH0_ADDRESS_NO_DOMAIN"), os.getenv("AUTH0_AUDIENCE_NO_DOMAIN") ) require_auth.register_token_validator(validator) @@ -56,6 +55,7 @@ def add_yearly_variables(household, country_id): ] = {2023: None} return household + @require_auth(None) def get_calculate(country_id: str, add_missing: bool = False) -> dict: """Lightweight endpoint for passing in household and policy JSON objects and calculating without storing data. diff --git a/setup.py b/setup.py index a6e8407..3ced2e6 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ "policyengine_us==0.571.2", "Flask-Caching==2.0.2", "urllib3<1.27,>=1.21.1", - "python-dotenv" + "python-dotenv", ], extras_require={ "dev": [ From 95f81600d0f1b1c000bde8b2b2dec75e2ab2931a Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 19:49:45 -0600 Subject: [PATCH 05/10] fix: Update tests to utilize authentication --- policyengine_api_light/endpoints/household.py | 2 +- tests/python/test_liveness.py | 12 ++++++++++-- tests/python/test_sync.py | 9 ++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/policyengine_api_light/endpoints/household.py b/policyengine_api_light/endpoints/household.py index 3d8bf00..bd73f9b 100644 --- a/policyengine_api_light/endpoints/household.py +++ b/policyengine_api_light/endpoints/household.py @@ -8,7 +8,7 @@ from policyengine_api_light.country import COUNTRIES import json import logging -from dotenv import load_dotenv, find_dotenv +from dotenv import load_dotenv from authlib.integrations.flask_oauth2 import ResourceProtector from ..auth.validation import Auth0JWTBearerTokenValidator diff --git a/tests/python/test_liveness.py b/tests/python/test_liveness.py index e5364e8..b8cb94e 100644 --- a/tests/python/test_liveness.py +++ b/tests/python/test_liveness.py @@ -1,11 +1,19 @@ +import os +from dotenv import load_dotenv + from tests.python.utils import client +load_dotenv() + def test_calculate_liveness(client): - """This tests that, when passed relevant data, calculate endpoint functions properly""" + """This tests that, when passed relevant data, calculate endpoint returns something""" response = client.post( "/us/calculate", - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + "Authorization": "Bearer " + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN") + }, data=open( "./tests/python/data/calculate_us_1_data.json", "r", diff --git a/tests/python/test_sync.py b/tests/python/test_sync.py index 3855168..815a29c 100644 --- a/tests/python/test_sync.py +++ b/tests/python/test_sync.py @@ -1,6 +1,10 @@ +import os import requests import json import sys +from dotenv import load_dotenv + +load_dotenv() from tests.python.utils import client, extract_json_from_file @@ -25,7 +29,10 @@ def test_calculate_sync(client): # Mock a POST request to API-light resLight = client.post( "/" + country_id + "/calculate", - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + "Authorization": "Bearer " + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN") + }, json=input_data, ).get_json() From d56c7e3fd4eab5f5a33276724f209207961e649d Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 19:50:46 -0600 Subject: [PATCH 06/10] chore: Lint --- tests/python/test_liveness.py | 3 ++- tests/python/test_sync.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/python/test_liveness.py b/tests/python/test_liveness.py index b8cb94e..da1dba6 100644 --- a/tests/python/test_liveness.py +++ b/tests/python/test_liveness.py @@ -12,7 +12,8 @@ def test_calculate_liveness(client): "/us/calculate", headers={ "Content-Type": "application/json", - "Authorization": "Bearer " + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN") + "Authorization": "Bearer " + + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN"), }, data=open( "./tests/python/data/calculate_us_1_data.json", diff --git a/tests/python/test_sync.py b/tests/python/test_sync.py index 815a29c..1c71990 100644 --- a/tests/python/test_sync.py +++ b/tests/python/test_sync.py @@ -31,7 +31,8 @@ def test_calculate_sync(client): "/" + country_id + "/calculate", headers={ "Content-Type": "application/json", - "Authorization": "Bearer " + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN") + "Authorization": "Bearer " + + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN"), }, json=input_data, ).get_json() From 5e93fc657fff1909c171e32217b683eac6eca874 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 19:57:38 -0600 Subject: [PATCH 07/10] fix: Add test token into GCP and GitHub env --- .github/workflows/pr.yml | 1 + .github/workflows/push.yml | 1 + gcp/export.py | 2 ++ gcp/policyengine_api_light/Dockerfile | 1 + 4 files changed, 5 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 13ec947..4faea14 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -52,3 +52,4 @@ jobs: env: AUTH0_ADDRESS_NO_DOMAIN: ${{ secrets.AUTH0_ADDRESS_NO_DOMAIN }} AUTH0_AUDIENCE_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} + AUTH0_TEST_TOKEN_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ce29f6a..20bc6f7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -70,3 +70,4 @@ jobs: GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SA_KEY }} AUTH0_ADDRESS_NO_DOMAIN: ${{ secrets.AUTH0_ADDRESS_NO_DOMAIN }} AUTH0_AUDIENCE_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} + AUTH0_TEST_TOKEN_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} diff --git a/gcp/export.py b/gcp/export.py index 2279b5c..d59a88d 100644 --- a/gcp/export.py +++ b/gcp/export.py @@ -11,6 +11,7 @@ pass ADDRESS = os.environ["AUTH0_ADDRESS_NO_DOMAIN"] AUDIENCE = os.environ["AUTH0_AUDIENCE_NO_DOMAIN"] +TEST_TOKEN = os.environ["AUTH0_TEST_TOKEN_NO_DOMAIN"] # Export GAE to to .gac.json @@ -25,6 +26,7 @@ dockerfile = f.read() dockerfile = dockerfile.replace(".address", ADDRESS) dockerfile = dockerfile.replace(".audience", AUDIENCE) + dockerfile = dockerfile.replace(".test-token", TEST_TOKEN) with open(dockerfile_location, "w") as f: f.write(dockerfile) diff --git a/gcp/policyengine_api_light/Dockerfile b/gcp/policyengine_api_light/Dockerfile index 8a57bd0..415d616 100644 --- a/gcp/policyengine_api_light/Dockerfile +++ b/gcp/policyengine_api_light/Dockerfile @@ -2,6 +2,7 @@ FROM anthvolk/policyengine-api-light:latest ENV GOOGLE_APPLICATION_CREDENTIALS .gac.json ENV AUTH0_ADDRESS_NO_DOMAIN .address ENV AUTH0_AUDIENCE_NO_DOMAIN .audience +ENV AUTH0_TEST_TOKEN_NO_DOMAIN .test-token ADD . /app RUN cd /app && make install && make test CMD ./start.sh From d05d982cb423046ee761a2f85ce2b5048ec88380 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 20:06:27 -0600 Subject: [PATCH 08/10] test: Determine why tests are failing --- tests/python/test_sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/python/test_sync.py b/tests/python/test_sync.py index 1c71990..9d2ae46 100644 --- a/tests/python/test_sync.py +++ b/tests/python/test_sync.py @@ -14,6 +14,8 @@ def test_calculate_sync(client): """Confirm that the calculate endpoint outputs the same data as the main API""" + print("Bearer " + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN")) + country_id = "us" # Load the sample data From 80ffe38d15a11f80bf21034b8184318412cbe380 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 20:18:07 -0600 Subject: [PATCH 09/10] test: Determine why tests are failing --- tests/python/test_sync.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/python/test_sync.py b/tests/python/test_sync.py index 9d2ae46..d8c08f0 100644 --- a/tests/python/test_sync.py +++ b/tests/python/test_sync.py @@ -11,6 +11,14 @@ API_URL = "https://api.policyengine.org/" +def test_dotenv(client): + output = os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN") + print(output) + print(len(output)) + print(type(output)) + print(str(output)) + + def test_calculate_sync(client): """Confirm that the calculate endpoint outputs the same data as the main API""" From 1cd281931fcba4b6163aacf93678c9881a87685c Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 2 Jan 2024 20:25:29 -0600 Subject: [PATCH 10/10] fix: Repair test token import --- .github/workflows/pr.yml | 2 +- .github/workflows/push.yml | 2 +- tests/python/test_sync.py | 10 ---------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4faea14..5a30e88 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -52,4 +52,4 @@ jobs: env: AUTH0_ADDRESS_NO_DOMAIN: ${{ secrets.AUTH0_ADDRESS_NO_DOMAIN }} AUTH0_AUDIENCE_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} - AUTH0_TEST_TOKEN_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} + AUTH0_TEST_TOKEN_NO_DOMAIN: ${{ secrets.AUTH0_TEST_TOKEN_NO_DOMAIN }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 20bc6f7..c0b7792 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -70,4 +70,4 @@ jobs: GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SA_KEY }} AUTH0_ADDRESS_NO_DOMAIN: ${{ secrets.AUTH0_ADDRESS_NO_DOMAIN }} AUTH0_AUDIENCE_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} - AUTH0_TEST_TOKEN_NO_DOMAIN: ${{ secrets.AUTH0_AUDIENCE_NO_DOMAIN }} + AUTH0_TEST_TOKEN_NO_DOMAIN: ${{ secrets.AUTH0_TEST_TOKEN_NO_DOMAIN }} diff --git a/tests/python/test_sync.py b/tests/python/test_sync.py index d8c08f0..1c71990 100644 --- a/tests/python/test_sync.py +++ b/tests/python/test_sync.py @@ -11,19 +11,9 @@ API_URL = "https://api.policyengine.org/" -def test_dotenv(client): - output = os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN") - print(output) - print(len(output)) - print(type(output)) - print(str(output)) - - def test_calculate_sync(client): """Confirm that the calculate endpoint outputs the same data as the main API""" - print("Bearer " + os.getenv("AUTH0_TEST_TOKEN_NO_DOMAIN")) - country_id = "us" # Load the sample data