diff --git a/configuration/management/commands/add_config.py b/configuration/management/commands/add_config.py index c97aa187..d49e85c9 100644 --- a/configuration/management/commands/add_config.py +++ b/configuration/management/commands/add_config.py @@ -169,7 +169,10 @@ class Command(BaseCommand): }, "sSI": {"_label": "incomeOptions.sSI", "_default_message": "Supplemental Security Income (SSI)"}, "childSupport": {"_label": "incomeOptions.childSupport", "_default_message": "Child Support (Received)"}, - "pension": {"_label": "incomeOptions.pension", "_default_message": "Military, Government, or Private Pension"}, + "pension": { + "_label": "incomeOptions.pension", + "_default_message": "Military, Government, or Private Pension (including PERA)", + }, "veteran": {"_label": "incomeOptions.veteran", "_default_message": "Veteran's Pension or Benefits"}, "sSSurvivor": { "_label": "incomeOptions.sSSurvivor", @@ -2747,10 +2750,7 @@ class Command(BaseCommand): } privacy_policy = { - "en-us": "https://co.myfriendben.org/en/data-privacy-policy", - "es": "https://co.myfriendben.org/es/data-privacy-policy", - "fr": "https://co.myfriendben.org/fr/data-privacy-policy", - "vi": "https://co.myfriendben.org/vi/data-privacy-policy", + "en-us": "https://co.myfriendben.org/privacy-policy/", } referrer_data = { @@ -2828,7 +2828,7 @@ class Command(BaseCommand): "state": "CO", "zip_code": 80202, "email": "myfriendben@garycommunity.org", - "privacy_policy_link": "https://co.myfriendben.org/en/data-privacy-policy", + "privacy_policy_link": "https://co.myfriendben.org/privacy-policy/", } @transaction.atomic diff --git a/integrations/admin.py b/integrations/admin.py new file mode 100644 index 00000000..91401bfb --- /dev/null +++ b/integrations/admin.py @@ -0,0 +1,11 @@ +from django.contrib import admin +from unfold.admin import ModelAdmin +from integrations.models import Link + + +class LinkAdmin(ModelAdmin): + search_fields = ("link",) + list_display = ["validated", "valid_status_code", "status_code", "in_use", "link"] + + +admin.site.register(Link, LinkAdmin) diff --git a/integrations/management/commands/health_check.py b/integrations/management/commands/health_check.py new file mode 100644 index 00000000..8e2da9e0 --- /dev/null +++ b/integrations/management/commands/health_check.py @@ -0,0 +1,139 @@ +from django.core.management.base import BaseCommand +from django.conf import settings +from decouple import config +from hubspot import HubSpot +from hubspot.crm.contacts.exceptions import ForbiddenException +from django.db.models import Q +from authentication.models import User +from integrations.models import Link +from programs.models import Navigator, Program, TranslationOverride, UrgentNeed +from translations.models import BLANK_TRANSLATION_PLACEHOLDER, Translation +from configuration.models import Configuration +import argparse +import json + + +class Command(BaseCommand): + help = "Check that we can't read from HubSpot, and there is no PII in the database" + + HUB_SPOT_TEXT = "Can't read Hub Spot" + PII_IN_DB_TEXT = "PII not in the database" + + def add_arguments(self, parser): + parser.add_argument( + "-s", + "--strict", + action=argparse.BooleanOptionalAction, + help="Compare the website hashes to the stored hashes", + ) + + def handle(self, *args, **options): + self.stdout.write("") + + self._check_links(options["strict"]) + + self.stdout.write("") + + self._output_condition(self._cant_read_hubspot(), self.HUB_SPOT_TEXT) + self._output_condition(self._no_pii_in_db(), self.PII_IN_DB_TEXT) + + def _cant_read_hubspot(self) -> bool: + client = HubSpot(access_token=config("HUBSPOT")) + + try: + client.crm.contacts.basic_api.get_page(limit=1, archived=False) + return False + except ForbiddenException as e: + return True + except Exception as e: + self.stdout.write(self.style.ERROR(f"Exception when calling basic_api->get_page: {e}")) + return False + + def _no_pii_in_db(self) -> bool: + users = User.objects.filter(is_staff=False).filter( + Q(first_name__isnull=False) + | Q(last_name__isnull=False) + | Q(cell__isnull=False) + | Q(email__isnull=False) + | Q(external_id__isnull=True) + ) + + if len(users) > 0: + return False + return True + + def _check_links(self, strict: bool = False) -> bool: + links = self._get_links() + + Link.objects.all().update(in_use=False) + + for link in links: + if link == BLANK_TRANSLATION_PLACEHOLDER or link == "": + continue + + try: + link_model: Link = Link.objects.get(link=link) + link_model.in_use = True + link_model.save() + except Link.DoesNotExist: + link_model: Link = Link.objects.create(link=link, validated=False, in_use=True) + + link_model.validate() + + if strict: + valid = link_model.validated + else: + valid = Link.good_status_code(link_model.status_code) + + self._output_condition(valid, f"{link} {link_model.status_code}") + + def _get_links(self) -> list[str]: + program_links = [p.apply_button_link for p in Program.objects.filter(active=True)] + urgent_need_links = [u.link for u in UrgentNeed.objects.filter(active=True)] + navigator_links = [n.assistance_link for n in Navigator.objects.filter(programs__isnull=False)] + translation_override_links = [ + o.translation + for o in TranslationOverride.objects.filter(field__in=["apply_button_link", "learn_more_link"]) + ] + config_links = [ + json.loads(Configuration.objects.get(name="public_charge_rule").data)["link"], + *[ + o["link"] + for o in json.loads(Configuration.objects.get(name="more_help_options").data)["moreHelpOptions"] + if "link" in o + ], + *json.loads(Configuration.objects.get(name="privacy_policy").data).values(), + *json.loads(Configuration.objects.get(name="consent_to_contact").data).values(), + ] + + links = { + *self._get_translation_links(program_links), + *self._get_translation_links(urgent_need_links), + *self._get_translation_links(navigator_links), + *self._get_translation_links(translation_override_links), + *config_links, + } + + links = list(links) + links.sort() + + return links + + def _get_translation_links(self, translations: list[Translation]) -> list[str]: + links = set() + for translation in translations: + for lang_setting in settings.LANGUAGES: + lang = lang_setting[0] + translation.set_current_language(lang) + + links.add(translation.text) + + return list(links) + + def _output_condition(self, validated: bool, message: str): + text = f"{message}: {'VALIDATED' if validated else 'FAILED'}" + + if validated: + self.stdout.write(self.style.SUCCESS(text)) + else: + self.stdout.write(self.style.ERROR(text)) diff --git a/integrations/migrations/0001_squashed_0004_alter_link_link.py b/integrations/migrations/0001_squashed_0004_alter_link_link.py new file mode 100644 index 00000000..a18430cd --- /dev/null +++ b/integrations/migrations/0001_squashed_0004_alter_link_link.py @@ -0,0 +1,40 @@ +# Generated by Django 4.2.15 on 2024-10-28 22:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [ + ("integrations", "0001_initial"), + ("integrations", "0002_alter_link_hash"), + ("integrations", "0003_alter_link_hash"), + ("integrations", "0004_alter_link_link"), + ] + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Link", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("link", models.URLField(max_length=2048, unique=True)), + ("validated", models.BooleanField(default=False)), + ( + "hash", + models.CharField(blank=True, default=None, max_length=2048, null=True), + ), + ], + ), + ] diff --git a/integrations/migrations/0005_link_status_code_alter_link_hash.py b/integrations/migrations/0005_link_status_code_alter_link_hash.py new file mode 100644 index 00000000..3d2c7950 --- /dev/null +++ b/integrations/migrations/0005_link_status_code_alter_link_hash.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.15 on 2024-10-29 15:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("integrations", "0001_squashed_0004_alter_link_link"), + ] + + operations = [ + migrations.AddField( + model_name="link", + name="status_code", + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name="link", + name="hash", + field=models.CharField(blank=True, max_length=2048, null=True), + ), + ] diff --git a/integrations/migrations/0006_link_valid_status_code.py b/integrations/migrations/0006_link_valid_status_code.py new file mode 100644 index 00000000..462b93da --- /dev/null +++ b/integrations/migrations/0006_link_valid_status_code.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-10-29 16:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("integrations", "0005_link_status_code_alter_link_hash"), + ] + + operations = [ + migrations.AddField( + model_name="link", + name="valid_status_code", + field=models.BooleanField(default=False), + ), + ] diff --git a/integrations/migrations/0007_link_in_use.py b/integrations/migrations/0007_link_in_use.py new file mode 100644 index 00000000..e43ff95e --- /dev/null +++ b/integrations/migrations/0007_link_in_use.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-10-29 16:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("integrations", "0006_link_valid_status_code"), + ] + + operations = [ + migrations.AddField( + model_name="link", + name="in_use", + field=models.BooleanField(default=False), + ), + ] diff --git a/integrations/models.py b/integrations/models.py new file mode 100644 index 00000000..0f785d7a --- /dev/null +++ b/integrations/models.py @@ -0,0 +1,91 @@ +from django.db import models +import hashlib +import requests +import random + + +class LinkManager(models.Manager): + def create(self, *args, **kwargs): + instance = super().create(*args, **kwargs) + instance.fill_hash() + instance.save() + + return instance + + +class Link(models.Model): + link = models.URLField(max_length=2_048, unique=True) + in_use = models.BooleanField(default=False) + validated = models.BooleanField(default=False) + status_code = models.IntegerField(blank=True, null=True) + valid_status_code = models.BooleanField(default=False) + hash = models.CharField(max_length=2_048, blank=True, null=True) + + objects = LinkManager() + + def __str__(self): + return self.link + + @staticmethod + def hash_data(data: str) -> str: + return hashlib.sha224(data.encode()).hexdigest() + + @staticmethod + def good_status_code(code: int) -> bool: + return 200 >= code < 300 + + def _get_request(self) -> requests.Response: + user_agents = [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11", + "Opera/9.25 (Windows NT 5.1; U; en)", + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", + "Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.142 Safari/535.19", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:8.0.1) Gecko/20100101 Firefox/8.0.1", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19", + ] + rand_agent_index = random.randint(0, len(user_agents) - 1) + header = {"User-agent": user_agents[rand_agent_index]} + req = requests.get(self.link, headers=header) + + return req + + def _get_request_parts(self) -> tuple[int, str]: + try: + req = self._get_request() + except requests.RequestException: + return 400, "error" + + return req.status_code, req.text + + def validate(self): + status_code, body = self._get_request_parts() + + self.status_code = status_code + self.valid_status_code = self.good_status_code(status_code) + self.save() + + if not self.valid_status_code: + self.validated = False + self.save() + return + + new_hash = self.hash_data(body) + + if self.hash != new_hash: + self.validated = False + self.save() + + def fill_hash(self): + if self.hash is None: + status_code, body = self._get_request_parts() + self.status_code = status_code + self.valid_status_code = self.good_status_code(status_code) + self.hash = self.hash_data(body) + self.validated = False + + def save(self, force_insert=False, force_update=False, using=None, update_fields=None): + self.fill_hash() + + return super().save(force_insert, force_update, using, update_fields) diff --git a/programs/models.py b/programs/models.py index 825a38d6..4c88b16f 100644 --- a/programs/models.py +++ b/programs/models.py @@ -208,12 +208,13 @@ class ProgramManager(models.Manager): "estimated_value", "website_description", ) + no_auto_fields = ("apply_button_link", "learn_more_link") def new_program(self, name_abbreviated): translations = {} for field in self.translated_fields: translations[field] = Translation.objects.add_translation( - f"program.{name_abbreviated}_temporary_key-{field}" + f"program.{name_abbreviated}_temporary_key-{field}", no_auto=(field in self.no_auto_fields) ) # try to set the external_name to the name_abbreviated @@ -451,11 +452,14 @@ class UrgentNeedManager(models.Manager): "warning", "website_description", ) + no_auto_fields = ("link",) def new_urgent_need(self, name, phone_number): translations = {} for field in self.translated_fields: - translations[field] = Translation.objects.add_translation(f"urgent_need.{name}_temporary_key-{field}") + translations[field] = Translation.objects.add_translation( + f"urgent_need.{name}_temporary_key-{field}", no_auto=(field in self.no_auto_fields) + ) # try to set the external_name to the name external_name_exists = self.filter(external_name=name).count() > 0 @@ -596,11 +600,14 @@ class NavigatorManager(models.Manager): "assistance_link", "description", ) + no_auto_fields = ("assistance_link",) def new_navigator(self, name, phone_number): translations = {} for field in self.translated_fields: - translations[field] = Translation.objects.add_translation(f"navigator.{name}_temporary_key-{field}") + translations[field] = Translation.objects.add_translation( + f"navigator.{name}_temporary_key-{field}", no_auto=(field in self.no_auto_fields) + ) # try to set the external_name to the name external_name_exists = self.filter(external_name=name).count() > 0 @@ -840,7 +847,8 @@ def new_translation_override(self, calculator: str, program_field: str, external translations = {} for field in self.translated_fields: translations[field] = Translation.objects.add_translation( - f"translation_override.{calculator}_temporary_key-{field}" + f"translation_override.{calculator}_temporary_key-{field}", + no_auto=(program_field in ProgramManager.no_auto_fields), ) if external_name is None: diff --git a/programs/programs/federal/pe/spm.py b/programs/programs/federal/pe/spm.py index 1ab67cc8..b3038e1e 100644 --- a/programs/programs/federal/pe/spm.py +++ b/programs/programs/federal/pe/spm.py @@ -8,10 +8,8 @@ class Snap(PolicyEngineSpmCalulator): dependency.spm.SnapChildSupportDeductionDependency, dependency.spm.SnapGrossIncomeDependency, dependency.spm.SnapAssetsDependency, - dependency.spm.MeetsSnapCategoricalEligibilityDependency, dependency.spm.SnapEmergencyAllotmentDependency, dependency.spm.HousingCostDependency, - dependency.spm.MeetsSnapGrossIncomeTestDependency, dependency.spm.HasPhoneExpenseDependency, dependency.spm.HasHeatingCoolingExpenseDependency, dependency.spm.HeatingCoolingExpenseDependency, diff --git a/programs/programs/policyengine/calculators/dependencies/spm.py b/programs/programs/policyengine/calculators/dependencies/spm.py index fe37707d..46a3fa7d 100644 --- a/programs/programs/policyengine/calculators/dependencies/spm.py +++ b/programs/programs/policyengine/calculators/dependencies/spm.py @@ -54,22 +54,6 @@ def value(self): return int(self.screen.calc_gross_income("yearly", ["all"])) -class MeetsSnapGrossIncomeTestDependency(SpmUnit): - field = "meets_snap_gross_income_test" - dependencies = ( - "income_amount", - "income_frequency", - "household_size", - ) - - def value(self): - fpl = FederalPoveryLimit.objects.get(year="THIS YEAR").as_dict() - snap_gross_income = self.screen.calc_gross_income("yearly", ["all"]) - snap_gross_limit = 2 * fpl[self.screen.household_size] - - return snap_gross_income < snap_gross_limit - - class SnapAlwaysUseSuaDependency(SpmUnit): field = "snap_state_using_standard_utility_allowance" @@ -91,13 +75,6 @@ def value(self): return True -class MeetsSnapCategoricalEligibilityDependency(SpmUnit): - field = "meets_snap_categorical_eligibility" - - def value(self): - return False - - class HasHeatingCoolingExpenseDependency(SpmUnit): field = "has_heating_cooling_expense" diff --git a/programs/serializers.py b/programs/serializers.py index 14da2fb1..189b6087 100644 --- a/programs/serializers.py +++ b/programs/serializers.py @@ -34,6 +34,7 @@ class ProgramCategorySerializer(serializers.ModelSerializer): name = ModelTranslationSerializer() class Meta: + ref_name = "Program Category List" model = ProgramCategory fields = ("id", "name", "icon", "programs") diff --git a/translations/models.py b/translations/models.py index db6dcc57..c55d1777 100644 --- a/translations/models.py +++ b/translations/models.py @@ -33,23 +33,30 @@ def update(self): class TranslationManager(TranslatableManager): use_in_migrations = True translation_cache = TranslationCache() + all_langs = [lang["code"] for lang in settings.PARLER_LANGUAGES[None]] def add_translation(self, label, default_message=BLANK_TRANSLATION_PLACEHOLDER, active=True, no_auto=False): default_lang = settings.LANGUAGE_CODE parent = self.get_or_create(label=label, defaults={"active": active, "no_auto": no_auto})[0] - if parent.active != active or parent.active != no_auto: + if parent.active != active or parent.no_auto != no_auto: parent.active = active parent.no_auto = no_auto parent.save() parent.create_translation(default_lang, text=default_message, edited=True) + + for lang in self.all_langs: + if lang == default_lang: + continue + + parent.create_translation(lang, text="", edited=False) self.translation_cache.invalid = True return parent def edit_translation(self, label, lang, translation, manual=True): parent = self.language(lang).get(label=label) - if manual is False and parent.no_auto: + if manual is False and parent.no_auto and parent.edited: return parent parent.text = translation @@ -61,7 +68,7 @@ def edit_translation(self, label, lang, translation, manual=True): def edit_translation_by_id(self, id, lang, translation, manual=True): parent = self.prefetch_related("translations").language(lang).get(pk=id) - if manual is False and parent.no_auto: + if manual is False and parent.no_auto and parent.edited: return parent parent.text = translation @@ -70,7 +77,7 @@ def edit_translation_by_id(self, id, lang, translation, manual=True): self.translation_cache.invalid = True return parent - def all_translations(self, langs=[lang["code"] for lang in settings.PARLER_LANGUAGES[None]]): + def all_translations(self, langs=all_langs): translations_dict = {} for lang in langs: translations_dict[lang] = self.translation_cache.fetch()[lang] diff --git a/translations/views.py b/translations/views.py index f5655a0c..94be9980 100644 --- a/translations/views.py +++ b/translations/views.py @@ -185,10 +185,12 @@ def edit_translation(request, id=0, lang="en-us"): translation = Translation.objects.edit_translation_by_id(id, lang, text) if lang == settings.LANGUAGE_CODE: - translations = Translate().bulk_translate(["__all__"], [text])[text] + if not translation.no_auto: + translations = Translate().bulk_translate(["__all__"], [text])[text] - for [language, translation] in translations.items(): - Translation.objects.edit_translation_by_id(id, language, translation, False) + for language in Translate.languages: + translated_text = text if translation.no_auto else translations[language] + Translation.objects.edit_translation_by_id(id, language, translated_text, False) parent = Translation.objects.get(pk=id) forms = {t.language_code: TranslationForm({"text": t.text}) for t in parent.translations.all()}