From bfef06ae5e5d153b2a3bdec8b2080dea4774489f Mon Sep 17 00:00:00 2001 From: Caleb Date: Mon, 20 May 2024 13:40:48 -0600 Subject: [PATCH 1/5] add calculator dependency injection back --- programs/programs/co/pe/member.py | 7 +- programs/programs/co/pe/tax.py | 2 +- programs/programs/federal/pe/member.py | 35 ++++++--- programs/programs/federal/pe/spm.py | 6 +- .../programs/policyengine/calculators/base.py | 30 ++++---- programs/programs/policyengine/engines.py | 71 +++++++++++++++++++ .../programs/policyengine/policy_engine.py | 40 +++++------ 7 files changed, 138 insertions(+), 53 deletions(-) create mode 100644 programs/programs/policyengine/engines.py diff --git a/programs/programs/co/pe/member.py b/programs/programs/co/pe/member.py index 0258cdc8..f1276202 100644 --- a/programs/programs/co/pe/member.py +++ b/programs/programs/co/pe/member.py @@ -61,11 +61,12 @@ class Chp(PolicyEngineMembersCalculator): def value(self): total = 0 - for pkey, pvalue in self.get_data().items(): - if not self.in_tax_unit(pkey): + for member in self.screen.household_members.all(): + if not self.in_tax_unit(member.id): continue - if pvalue['co_chp_eligible'][self.pe_period] > 0 and self.screen.has_insurance_types(('none',)): + chp_eligible = self.sim.value(self.pe_category, str(member.id), 'co_chp_eligible', self.pe_period) > 0 + if chp_eligible and self.screen.has_insurance_types(('none',)): total += self.amount return total diff --git a/programs/programs/co/pe/tax.py b/programs/programs/co/pe/tax.py index a500674e..53e4aa62 100644 --- a/programs/programs/co/pe/tax.py +++ b/programs/programs/co/pe/tax.py @@ -38,4 +38,4 @@ def value(self): multiplier = band['percent'] break - return self.get_data()[self.pe_name][self.pe_period] * multiplier + return self.get_variable() * multiplier diff --git a/programs/programs/federal/pe/member.py b/programs/programs/federal/pe/member.py index a68d4735..e6d842fb 100644 --- a/programs/programs/federal/pe/member.py +++ b/programs/programs/federal/pe/member.py @@ -23,9 +23,10 @@ class Wic(PolicyEngineMembersCalculator): def value(self): total = 0 - for _, pvalue in self.get_data().items(): - if pvalue[self.pe_name][self.pe_period] > 0: - total += self.wic_categories[pvalue['wic_category'][self.pe_period]] * 12 + for member in self.screen.household_members.all(): + if self.get_member_variable(member.id) > 0: + wic_category = self.sim.value('people', str(member.id), 'wic_category', self.pe_period) + total += self.wic_categories[wic_category] * 12 return total @@ -67,18 +68,29 @@ def _value_by_age(self, age: int): def value(self): total = 0 - for pkey, pvalue in self.get_data().items(): - # Skip any members that are not in the tax unit. - if not self.in_tax_unit(pkey): - continue + members = self.screen.household_members.all() - if pvalue[self.pe_name][self.pe_period] <= 0: + for member in members: + if self.get_member_variable(member.id) <= 0: continue - total += self._value_by_age(pvalue['age'][self.pe_period]) + # here we need to adjust for children as policy engine + # just uses the average which skews very high for adults and + # aged adults + + if self._get_age(member.id) <= 18: + medicaid_estimated_value = self.child_medicaid_average + elif self._get_age(member.id) > 18 and self._get_age(member.id) < 65: + medicaid_estimated_value = self.adult_medicaid_average + elif self._get_age(member.id) >= 65: + medicaid_estimated_value = self.aged_medicaid_average + else: + medicaid_estimated_value = 0 + + total += medicaid_estimated_value in_wic_demographic = False - for member in self.screen.household_members.all(): + for member in members: if member.pregnant is True or member.age <= 5: in_wic_demographic = True if total == 0 and in_wic_demographic: @@ -89,6 +101,9 @@ def value(self): return total + def _get_age(self, member_id: int) -> int: + return self.sim.value(self.pe_category, str(member_id), 'age', self.pe_period) + class PellGrant(PolicyEngineMembersCalculator): pe_name = 'pell_grant' diff --git a/programs/programs/federal/pe/spm.py b/programs/programs/federal/pe/spm.py index ec430ff7..c312d267 100644 --- a/programs/programs/federal/pe/spm.py +++ b/programs/programs/federal/pe/spm.py @@ -25,7 +25,7 @@ class Snap(PolicyEngineSpmCalulator): pe_output_period = SNAP_PERIOD def value(self): - return int(self.get_data()[self.pe_name][self.pe_output_period]) * 12 + return int(self.sim.value(self.pe_category, self.pe_sub_category, self.pe_name, self.pe_output_period)) * 12 class SchoolLunch(PolicyEngineSpmCalulator): @@ -39,8 +39,8 @@ def value(self): total = 0 num_children = self.screen.num_children(3, 18) - if self.get_data()[self.pe_name][self.pe_period] > 0 and num_children > 0: - if self.get_data()['school_meal_tier'][self.pe_period] != 'PAID': + if self.get_variable() > 0 and num_children > 0: + if self.sim.value(self.pe_category, self.pe_sub_category, 'school_meal_tier', self.pe_period) != 'PAID': total = SchoolLunch.amount * num_children return total diff --git a/programs/programs/policyengine/calculators/base.py b/programs/programs/policyengine/calculators/base.py index 8f3999bf..1eaea971 100644 --- a/programs/programs/policyengine/calculators/base.py +++ b/programs/programs/policyengine/calculators/base.py @@ -4,6 +4,7 @@ from .dependencies.base import PolicyEngineScreenInput from typing import List from .constants import YEAR, PREVIOUS_YEAR +from ..engines import Sim class PolicyEngineCalulator(ProgramCalculator): @@ -19,9 +20,9 @@ class PolicyEngineCalulator(ProgramCalculator): pe_sub_category = '' pe_period = YEAR - def __init__(self, screen: Screen, pe_data): + def __init__(self, screen: Screen, sim: Sim): self.screen = screen - self.pe_data = pe_data + self.sim = sim def eligible(self) -> Eligibility: e = Eligibility() @@ -31,13 +32,13 @@ def eligible(self) -> Eligibility: return e def value(self): - return self.get_data()[self.pe_name][self.pe_period] + return int(self.get_variable()) - def get_data(self): + def get_variable(self): ''' - Return Policy Engine dictionary of the program category and subcategory + Return value of the default variable ''' - return self.pe_data[self.pe_category][self.pe_sub_category] + return self.sim.value(self.pe_category, self.pe_sub_category, self.pe_name, self.pe_period) @classmethod def can_calc(cls, missing_dependencies: Dependencies): @@ -47,37 +48,40 @@ def can_calc(cls, missing_dependencies: Dependencies): return True + class PolicyEngineTaxUnitCalulator(PolicyEngineCalulator): pe_category = 'tax_units' pe_sub_category = 'tax_unit' tax_unit_dependent = True pe_period = PREVIOUS_YEAR + class PolicyEngineSpmCalulator(PolicyEngineCalulator): pe_category = 'spm_units' pe_sub_category = 'spm_unit' + class PolicyEngineMembersCalculator(PolicyEngineCalulator): tax_unit_dependent = True pe_category = 'people' def value(self): total = 0 - for pkey, pvalue in self.get_data().items(): + for member in self.screen.household_members.all(): # The following programs use income from the tax unit, # so we want to skip any members that are not in the tax unit. - if not self.in_tax_unit(pkey) and self.tax_unit_dependent: + if not self.in_tax_unit(member.id) and self.tax_unit_dependent: continue - pe_value = pvalue[self.pe_name][self.pe_period] + pe_value = self.get_member_variable(member.id) total += pe_value return total - def in_tax_unit(self, member_id) -> bool: - return str(member_id) in self.pe_data['tax_units']['tax_unit']['members'] + def in_tax_unit(self, member_id: int) -> bool: + return str(member_id) in self.sim.members('tax_units', 'tax_unit') - def get_data(self): - return self.pe_data[self.pe_category] + def get_member_variable(self, member_id: int): + return self.sim.value(self.pe_category, str(member_id), self.pe_name, self.pe_period) diff --git a/programs/programs/policyengine/engines.py b/programs/programs/policyengine/engines.py new file mode 100644 index 00000000..17c189f0 --- /dev/null +++ b/programs/programs/policyengine/engines.py @@ -0,0 +1,71 @@ +import requests + + +class Sim: + method = '' + + def __init__(self, data) -> None: + self.data = data + + def value(self, unit, sub_unit, variable, period): + ''' + Calculate variable at the period + ''' + raise NotImplementedError + + def members(self, unit, sub_unit): + ''' + Return a list of the members in the sub unit + ''' + raise NotImplementedError + + +class ApiSim(Sim): + method_name = 'Policy Engine API' + + def __init__(self, data) -> None: + response = requests.post("https://api.policyengine.org/us/calculate", json=data) + self.data = response.json()['result'] + + def value(self, unit, sub_unit, variable, period): + return self.data[unit][sub_unit][variable][period] + + def members(self, unit, sub_unit): + return self.data[unit][sub_unit]['members'] + + +class PrivateApi(ApiSim): + method_name = 'Private Policy Engine API' + + +# NOTE: Code to run Policy Engine locally. This is currently too CPU expensive to run in production. +# Requires the Policy Engine package to be installed and imported. +# class LocalSim(Sim): +# method_name = 'local package' +# +# def __init__(self, data) -> None: +# self.household = data['household'] +# +# self.entity_map = {} +# for entity in self.household.keys(): +# group_map = {} +# +# for i, group in enumerate(self.household[entity].keys()): +# group_map[group] = i +# +# self.entity_map[entity] = group_map +# +# self.sim = Simulation(situation=self.household) +# +# def value(self, unit, sub_unit, variable, period): +# data = self.sim.calculate(variable, period) +# +# index = self.entity_map[unit][sub_unit] +# +# return data[index] +# +# def members(self, unit, sub_unit): +# return self.household[unit][sub_unit]['members'] + + +pe_engines = [ApiSim] diff --git a/programs/programs/policyengine/policy_engine.py b/programs/programs/policyengine/policy_engine.py index c9c425db..fd5b4e9f 100644 --- a/programs/programs/policyengine/policy_engine.py +++ b/programs/programs/policyengine/policy_engine.py @@ -1,15 +1,11 @@ from screener.models import HouseholdMember, Screen -from .calculators import all_calculators, PolicyEngineCalulator +from .calculators import PolicyEngineCalulator from programs.programs.calc import Eligibility from programs.util import Dependencies from .calculators.dependencies.base import DependencyError from typing import List -import requests -from .calculators.dependencies.member import ( - TaxUnitDependentDependency, - TaxUnitHeadDependency, - TaxUnitSpouseDependency, -) +from sentry_sdk import capture_message +from .engines import Sim, pe_engines def calc_pe_eligibility( @@ -28,33 +24,31 @@ def calc_pe_eligibility( if len(valid_programs.values()) == 0 or len(screen.household_members.all()) == 0: return {} - data = policy_engine_calculate(pe_input(screen, valid_programs.values()))['result'] + input_data = pe_input(screen, valid_programs.values()) + for Method in pe_engines: + try: + return all_eligibility(Method(input_data), valid_programs, screen) + except Exception: + capture_message(f'Failed to calculate eligibility with the {Method.method_name} method', level='warning') + + error_message = 'Failed to calculate Policy Engine eligibility' + capture_message(error_message) + raise Exception(error_message) + + +def all_eligibility(method: Sim, valid_programs: dict[str, type[PolicyEngineCalulator]], screen: Screen): all_eligibility: dict[str, Eligibility] = {} - has_non_tax_unit_members = screen.has_members_ouside_of_tax_unit() for name_abbr, Calculator in valid_programs.items(): - calc = Calculator(screen, data) + calc = Calculator(screen, method) e = calc.eligible() e.value = calc.value() - - if Calculator.tax_unit_dependent and has_non_tax_unit_members: - e.multiple_tax_units = True - all_eligibility[name_abbr] = e.to_dict() return all_eligibility -def policy_engine_calculate(data): - response = requests.post( - "https://api.policyengine.org/us/calculate", - json=data - ) - data = response.json() - return data - - def pe_input(screen: Screen, programs: List[type[PolicyEngineCalulator]]): ''' Generate Policy Engine API request from the list of programs. From d697662a6b796498ab25ed3c0a87cf0f62433117 Mon Sep 17 00:00:00 2001 From: Caleb Date: Mon, 20 May 2024 15:41:12 -0600 Subject: [PATCH 2/5] add bearer token cache --- programs/programs/policyengine/engines.py | 41 +++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/programs/programs/policyengine/engines.py b/programs/programs/policyengine/engines.py index 17c189f0..ee1878ee 100644 --- a/programs/programs/policyengine/engines.py +++ b/programs/programs/policyengine/engines.py @@ -1,4 +1,8 @@ +import json +from integrations.util.cache import Cache +from decouple import config import requests +import http.client class Sim: @@ -34,12 +38,45 @@ def members(self, unit, sub_unit): return self.data[unit][sub_unit]['members'] -class PrivateApi(ApiSim): +class PolicyEngineBearerTokenCache(Cache): + expire_time = 60 * 60 * 24 * 30 + default = '' + client_id: str = config('POLICY_ENGINE_CLIENT_ID', '') + client_secret: str = config('POLICY_ENGINE_CLIENT_SECRET', '') + domain = 'https://policyengine.uk.auth0.com' + endpoint = '/oauth/token' + + def update(self): + # https://policyengine.org/us/api#fetch_token + + if self.client_id == '' or self.client_secret == '': + raise Exception('client id or secret not configured') + + payload = { + 'client_id': self.client_id, + 'client_secret': self.client_secret, + } + + + headers = { 'content-type': "application/json" } + + res = requests.post(self.domain + self.endpoint, json=payload, headers=headers) + + return res.json() + + +class PrivateApiSim(ApiSim): method_name = 'Private Policy Engine API' + token = PolicyEngineBearerTokenCache() + + def __init__(self, data) -> None: + token = self.token.fetch() + print(token) # NOTE: Code to run Policy Engine locally. This is currently too CPU expensive to run in production. # Requires the Policy Engine package to be installed and imported. +# # class LocalSim(Sim): # method_name = 'local package' # @@ -68,4 +105,4 @@ class PrivateApi(ApiSim): # return self.household[unit][sub_unit]['members'] -pe_engines = [ApiSim] +pe_engines = [PrivateApiSim, ApiSim] From 45d9af3fc89516a44c37f5e83966495d11dc0e3d Mon Sep 17 00:00:00 2001 From: Caleb Date: Tue, 21 May 2024 10:37:40 -0600 Subject: [PATCH 3/5] add ability to use private api --- integrations/util/cache.py | 8 +-- programs/programs/policyengine/engines.py | 62 ++++++++++--------- .../programs/policyengine/policy_engine.py | 51 +++++++-------- 3 files changed, 59 insertions(+), 62 deletions(-) diff --git a/integrations/util/cache.py b/integrations/util/cache.py index a340fb3e..f9e8301b 100644 --- a/integrations/util/cache.py +++ b/integrations/util/cache.py @@ -1,8 +1,8 @@ -from sentry_sdk import capture_message +from sentry_sdk import capture_exception, capture_message import datetime -class Cache(): +class Cache: expire_time = 0 default = 0 @@ -17,8 +17,8 @@ def _update_cache(self): try: self.data = self.update() self.last_update = datetime.datetime.now() - except Exception: - capture_message(f'Failed to update {self.__class__.__name__}', level='warning') + except Exception as e: + capture_exception(e, level="warning") def should_update(self): return datetime.datetime.now() > self.last_update + datetime.timedelta(seconds=self.expire_time) diff --git a/programs/programs/policyengine/engines.py b/programs/programs/policyengine/engines.py index ee1878ee..6370ad13 100644 --- a/programs/programs/policyengine/engines.py +++ b/programs/programs/policyengine/engines.py @@ -1,77 +1,83 @@ -import json from integrations.util.cache import Cache from decouple import config import requests -import http.client class Sim: - method = '' + method = "" def __init__(self, data) -> None: self.data = data def value(self, unit, sub_unit, variable, period): - ''' + """ Calculate variable at the period - ''' + """ raise NotImplementedError def members(self, unit, sub_unit): - ''' + """ Return a list of the members in the sub unit - ''' + """ raise NotImplementedError class ApiSim(Sim): - method_name = 'Policy Engine API' + method_name = "Policy Engine API" + pe_url = "https://api.policyengine.org/us/calculate" def __init__(self, data) -> None: - response = requests.post("https://api.policyengine.org/us/calculate", json=data) - self.data = response.json()['result'] + response = requests.post(self.pe_url, json=data) + self.data = response.json()["result"] def value(self, unit, sub_unit, variable, period): return self.data[unit][sub_unit][variable][period] def members(self, unit, sub_unit): - return self.data[unit][sub_unit]['members'] + return self.data[unit][sub_unit]["members"] class PolicyEngineBearerTokenCache(Cache): - expire_time = 60 * 60 * 24 * 30 - default = '' - client_id: str = config('POLICY_ENGINE_CLIENT_ID', '') - client_secret: str = config('POLICY_ENGINE_CLIENT_SECRET', '') - domain = 'https://policyengine.uk.auth0.com' - endpoint = '/oauth/token' + expire_time = 60 * 60 * 24 * 29 + default = "" + client_id: str = config("POLICY_ENGINE_CLIENT_ID", "") + client_secret: str = config("POLICY_ENGINE_CLIENT_SECRET", "") + domain = "https://policyengine.uk.auth0.com" + endpoint = "/oauth/token" def update(self): # https://policyengine.org/us/api#fetch_token - if self.client_id == '' or self.client_secret == '': - raise Exception('client id or secret not configured') + if self.client_id == "" or self.client_secret == "": + raise Exception("client id or secret not configured") payload = { - 'client_id': self.client_id, - 'client_secret': self.client_secret, + "client_id": self.client_id, + "client_secret": self.client_secret, + "grant_type": "client_credentials", + "audience": "https://household.api.policyengine.org", } + res = requests.post(self.domain + self.endpoint, json=payload) - headers = { 'content-type': "application/json" } - - res = requests.post(self.domain + self.endpoint, json=payload, headers=headers) - - return res.json() + return res.json()["access_token"] class PrivateApiSim(ApiSim): - method_name = 'Private Policy Engine API' + method_name = "Private Policy Engine API" token = PolicyEngineBearerTokenCache() + pe_url = "https://household.api.policyengine.org/us/calculate" def __init__(self, data) -> None: token = self.token.fetch() - print(token) + + headers = { + "Authorization": f"Bearer {token}", + } + + res = requests.post(self.pe_url, json=data, headers=headers) + + self.data = res.json()["result"] # NOTE: Code to run Policy Engine locally. This is currently too CPU expensive to run in production. diff --git a/programs/programs/policyengine/policy_engine.py b/programs/programs/policyengine/policy_engine.py index fd5b4e9f..5e5593da 100644 --- a/programs/programs/policyengine/policy_engine.py +++ b/programs/programs/policyengine/policy_engine.py @@ -4,14 +4,14 @@ from programs.util import Dependencies from .calculators.dependencies.base import DependencyError from typing import List -from sentry_sdk import capture_message +from sentry_sdk import capture_exception, capture_message from .engines import Sim, pe_engines def calc_pe_eligibility( - screen: Screen, - missing_fields: Dependencies, - calculators: dict[str, type[PolicyEngineCalulator]], + screen: Screen, + missing_fields: Dependencies, + calculators: dict[str, type[PolicyEngineCalulator]], ) -> dict[str, Eligibility]: valid_programs: dict[str, type[PolicyEngineCalulator]] = {} @@ -29,12 +29,11 @@ def calc_pe_eligibility( for Method in pe_engines: try: return all_eligibility(Method(input_data), valid_programs, screen) - except Exception: - capture_message(f'Failed to calculate eligibility with the {Method.method_name} method', level='warning') + except Exception as e: + capture_exception(e, level="warning", message="") + capture_message(f"Failed to calculate eligibility with the {Method.method_name} method", level="warning") - error_message = 'Failed to calculate Policy Engine eligibility' - capture_message(error_message) - raise Exception(error_message) + raise Exception("Failed to calculate Policy Engine eligibility") def all_eligibility(method: Sim, valid_programs: dict[str, type[PolicyEngineCalulator]], screen: Screen): @@ -50,9 +49,9 @@ def all_eligibility(method: Sim, valid_programs: dict[str, type[PolicyEngineCalu def pe_input(screen: Screen, programs: List[type[PolicyEngineCalulator]]): - ''' + """ Generate Policy Engine API request from the list of programs. - ''' + """ raw_input = { "household": { "people": {}, @@ -61,22 +60,14 @@ def pe_input(screen: Screen, programs: List[type[PolicyEngineCalulator]]): "members": [], } }, - "families": { - "family": { - "members": [] - } - }, - "households": { - "household": { - "members": [] - } - }, + "families": {"family": {"members": []}}, + "households": {"household": {"members": []}}, "spm_units": { "spm_unit": { "members": [], } }, - "marital_units": {} + "marital_units": {}, } } members: list[HouseholdMember] = screen.household_members.all() @@ -84,15 +75,15 @@ def pe_input(screen: Screen, programs: List[type[PolicyEngineCalulator]]): for member in members: member_id = str(member.id) - household = raw_input['household'] + household = raw_input["household"] - household['families']['family']['members'].append(member_id) - household['households']['household']['members'].append(member_id) - household['spm_units']['spm_unit']['members'].append(member_id) - household['people'][member_id] = {} + household["families"]["family"]["members"].append(member_id) + household["households"]["household"]["members"].append(member_id) + household["spm_units"]["spm_unit"]["members"].append(member_id) + household["people"][member_id] = {} if member.is_in_tax_unit(): - household['tax_units']['tax_unit']['members'].append(member_id) + household["tax_units"]["tax_unit"]["members"].append(member_id) already_added = set() for member_1, member_2 in relationship_map.items(): @@ -100,14 +91,14 @@ def pe_input(screen: Screen, programs: List[type[PolicyEngineCalulator]]): continue marital_unit = (str(member_1), str(member_2)) - raw_input['household']['marital_units']['-'.join(marital_unit)] = {'members': marital_unit} + raw_input["household"]["marital_units"]["-".join(marital_unit)] = {"members": marital_unit} already_added.add(member_1) already_added.add(member_2) for Program in programs: for Data in Program.pe_inputs + Program.pe_outputs: period = Program.pe_period - if hasattr(Program, 'pe_output_period') and Data in Program.pe_outputs: + if hasattr(Program, "pe_output_period") and Data in Program.pe_outputs: period = Program.pe_output_period if not Data.member: From d602164f6ef33e804b2f095f6868ec8020419fc2 Mon Sep 17 00:00:00 2001 From: Caleb Date: Tue, 21 May 2024 11:03:40 -0600 Subject: [PATCH 4/5] fix sim base class --- programs/programs/policyengine/engines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/programs/policyengine/engines.py b/programs/programs/policyengine/engines.py index 6370ad13..4e117681 100644 --- a/programs/programs/policyengine/engines.py +++ b/programs/programs/policyengine/engines.py @@ -4,7 +4,7 @@ class Sim: - method = "" + method_name = "" def __init__(self, data) -> None: self.data = data @@ -111,4 +111,4 @@ def __init__(self, data) -> None: # return self.household[unit][sub_unit]['members'] -pe_engines = [PrivateApiSim, ApiSim] +pe_engines: list[Sim] = [PrivateApiSim, ApiSim] From cc61f5a9a675b6495749e3c27228c9c6424f097a Mon Sep 17 00:00:00 2001 From: Caleb Date: Tue, 21 May 2024 11:05:27 -0600 Subject: [PATCH 5/5] remove unused import --- integrations/util/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/util/cache.py b/integrations/util/cache.py index f9e8301b..ca61ad37 100644 --- a/integrations/util/cache.py +++ b/integrations/util/cache.py @@ -1,4 +1,4 @@ -from sentry_sdk import capture_exception, capture_message +from sentry_sdk import capture_exception import datetime