From dbbaea1bc2f71f9ee78865c7e0e27ed4c0e827aa Mon Sep 17 00:00:00 2001 From: alburde1 Date: Tue, 11 Dec 2018 11:09:11 -0500 Subject: [PATCH 01/52] Updating submission loader and tests --- .../etl/management/commands/load_submission.py | 12 +++++++----- usaspending_api/etl/tests/data/submission_data.json | 2 +- usaspending_api/etl/tests/etl_test_data.json | 10 +++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/usaspending_api/etl/management/commands/load_submission.py b/usaspending_api/etl/management/commands/load_submission.py index 5e7670fa65..84e421010b 100644 --- a/usaspending_api/etl/management/commands/load_submission.py +++ b/usaspending_api/etl/management/commands/load_submission.py @@ -86,7 +86,7 @@ def signal_handler(signal, frame): submission_attributes = get_submission_attributes(broker_submission_id, submission_data) logger.info('Getting File A data') - db_cursor.execute('SELECT * FROM appropriation WHERE submission_id = %s', [submission_id]) + db_cursor.execute('SELECT * FROM certified_appropriation WHERE submission_id = %s', [submission_id]) appropriation_data = dictfetchall(db_cursor) logger.info('Acquired File A (appropriation) data for ' + str(submission_id) + ', there are ' + str( len(appropriation_data)) + ' rows.') @@ -108,7 +108,8 @@ def signal_handler(signal, frame): logger.info('Getting File C data') # we dont have sub-tier agency info, so we'll do our best # to match them to the more specific award records - award_financial_query = 'SELECT * FROM award_financial WHERE submission_id = {0}'.format(submission_id) + award_financial_query = 'SELECT * FROM certified_award_financial WHERE submission_id = {0}'.\ + format(submission_id) if isinstance(db_cursor, PhonyCursor): # spoofed data for test award_financial_frame = pd.DataFrame(db_cursor.db_responses[award_financial_query]) else: # real data @@ -408,7 +409,7 @@ class code but with one of the direct reimbursable flags set to NULL. # does this file B have the dupe object class edge case? check_dupe_oc = ( 'SELECT count(*) ' - 'FROM object_class_program_activity ' + 'FROM certified_object_class_program_activity ' 'WHERE submission_id = %s ' 'AND length(object_class) = 4 ' 'GROUP BY tas_id, program_activity_code, object_class ' @@ -419,7 +420,8 @@ class code but with one of the direct reimbursable flags set to NULL. if dupe_oc_count == 0: # there are no object class duplicates, so proceed as usual - db_cursor.execute('SELECT * FROM object_class_program_activity WHERE submission_id = %s', [submission_id]) + db_cursor.execute('SELECT * FROM certified_object_class_program_activity WHERE submission_id = %s', + [submission_id]) else: # file b contains at least one case of duplicate 4 digit object classes for the same program activity/tas, # so combine the records in question @@ -474,7 +476,7 @@ class code but with one of the direct reimbursable flags set to NULL. 'SUM(ussgl497200_downward_adjus_cpe) AS ussgl497200_downward_adjus_cpe, ' 'SUM(ussgl498100_upward_adjustm_cpe) AS ussgl498100_upward_adjustm_cpe, ' 'SUM(ussgl498200_upward_adjustm_cpe) AS ussgl498200_upward_adjustm_cpe ' - 'FROM object_class_program_activity ' + 'FROM certified_object_class_program_activity ' 'WHERE submission_id = %s ' 'GROUP BY ' 'submission_id, ' diff --git a/usaspending_api/etl/tests/data/submission_data.json b/usaspending_api/etl/tests/data/submission_data.json index ade9e7bde1..6b0221951e 100644 --- a/usaspending_api/etl/tests/data/submission_data.json +++ b/usaspending_api/etl/tests/data/submission_data.json @@ -26,7 +26,7 @@ "account_number": -99999 } ], - "SELECT * FROM award_financial WHERE submission_id = -9999": [ + "SELECT * FROM certified_award_financial WHERE submission_id = -9999": [ { "created_at": null, "updated_at": null, diff --git a/usaspending_api/etl/tests/etl_test_data.json b/usaspending_api/etl/tests/etl_test_data.json index 1816c2bcb1..c88062c6f6 100644 --- a/usaspending_api/etl/tests/etl_test_data.json +++ b/usaspending_api/etl/tests/etl_test_data.json @@ -2691,7 +2691,7 @@ "created_at": "2017-04-21T02:17:27.743" } ], - "SELECT * FROM object_class_program_activity WHERE submission_id = %s": [ + "SELECT * FROM certified_object_class_program_activity WHERE submission_id = %s": [ { "obligations_delivered_orde_cpe": "4500", "gross_outlay_amount_by_pro_fyb": "4500", @@ -3547,7 +3547,7 @@ "fiscal_year_and_quarter_co": "20162" } ], - "SELECT * FROM award_financial WHERE submission_id = %s": [ + "SELECT * FROM certified_award_financial WHERE submission_id = %s": [ { "obligations_delivered_orde_cpe": "26000", "transaction_obligated_amou": "6500", @@ -4415,7 +4415,7 @@ "awarding_sub_tier_agency_n": "Federal Aviation Administration" } ], - "SELECT count(*) FROM object_class_program_activity WHERE submission_id = %s AND length(object_class) = 4 GROUP BY tas_id, program_activity_code, object_class HAVING COUNT(*) > 1": [ + "SELECT count(*) FROM certified_object_class_program_activity WHERE submission_id = %s AND length(object_class) = 4 GROUP BY tas_id, program_activity_code, object_class HAVING COUNT(*) > 1": [ { "count": 1 } @@ -4431,7 +4431,7 @@ "allocation_transfer_agency": null } ], - "SELECT * FROM appropriation WHERE submission_id = %s": [ + "SELECT * FROM certified_appropriation WHERE submission_id = %s": [ { "borrowing_authority_amount_cpe": "1.51", "tas_id": 0, @@ -5629,7 +5629,7 @@ "awarding_sub_tier_agency_n": "6920: FEDERAL AVIATION ADMINISTRATION" } ], - "SELECT submission_id, job_id, agency_identifier,allocation_transfer_agency, availability_type_code, beginning_period_of_availa, ending_period_of_availabil, main_account_code, RIGHT(object_class, 3) AS object_class, CASE WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '1' THEN 'd' WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '2' THEN 'r' ELSE by_direct_reimbursable_fun END AS by_direct_reimbursable_fun, tas, tas_id, program_activity_code, program_activity_name, sub_account_code, SUM(deobligations_recov_by_pro_cpe) AS deobligations_recov_by_pro_cpe, SUM(gross_outlay_amount_by_pro_cpe) AS gross_outlay_amount_by_pro_cpe, SUM(gross_outlay_amount_by_pro_fyb) AS gross_outlay_amount_by_pro_fyb, SUM(gross_outlays_delivered_or_cpe) AS gross_outlays_delivered_or_cpe, SUM(gross_outlays_delivered_or_fyb) AS gross_outlays_delivered_or_fyb, SUM(gross_outlays_undelivered_cpe) AS gross_outlays_undelivered_cpe, SUM(gross_outlays_undelivered_fyb) AS gross_outlays_undelivered_fyb, SUM(obligations_delivered_orde_cpe) AS obligations_delivered_orde_cpe, SUM(obligations_delivered_orde_fyb) AS obligations_delivered_orde_fyb, SUM(obligations_incurred_by_pr_cpe) AS obligations_incurred_by_pr_cpe, SUM(obligations_undelivered_or_cpe) AS obligations_undelivered_or_cpe, SUM(obligations_undelivered_or_fyb) AS obligations_undelivered_or_fyb, SUM(ussgl480100_undelivered_or_cpe) AS ussgl480100_undelivered_or_cpe, SUM(ussgl480100_undelivered_or_fyb) AS ussgl480100_undelivered_or_fyb, SUM(ussgl480200_undelivered_or_cpe) AS ussgl480200_undelivered_or_cpe, SUM(ussgl480200_undelivered_or_fyb) AS ussgl480200_undelivered_or_fyb, SUM(ussgl483100_undelivered_or_cpe) AS ussgl483100_undelivered_or_cpe, SUM(ussgl483200_undelivered_or_cpe) AS ussgl483200_undelivered_or_cpe, SUM(ussgl487100_downward_adjus_cpe) AS ussgl487100_downward_adjus_cpe, SUM(ussgl487200_downward_adjus_cpe) AS ussgl487200_downward_adjus_cpe, SUM(ussgl488100_upward_adjustm_cpe) AS ussgl488100_upward_adjustm_cpe, SUM(ussgl488200_upward_adjustm_cpe) AS ussgl488200_upward_adjustm_cpe, SUM(ussgl490100_delivered_orde_cpe) AS ussgl490100_delivered_orde_cpe, SUM(ussgl490100_delivered_orde_fyb) AS ussgl490100_delivered_orde_fyb, SUM(ussgl490200_delivered_orde_cpe) AS ussgl490200_delivered_orde_cpe, SUM(ussgl490800_authority_outl_cpe) AS ussgl490800_authority_outl_cpe, SUM(ussgl490800_authority_outl_fyb) AS ussgl490800_authority_outl_fyb, SUM(ussgl493100_delivered_orde_cpe) AS ussgl493100_delivered_orde_cpe, SUM(ussgl497100_downward_adjus_cpe) AS ussgl497100_downward_adjus_cpe, SUM(ussgl497200_downward_adjus_cpe) AS ussgl497200_downward_adjus_cpe, SUM(ussgl498100_upward_adjustm_cpe) AS ussgl498100_upward_adjustm_cpe, SUM(ussgl498200_upward_adjustm_cpe) AS ussgl498200_upward_adjustm_cpe FROM object_class_program_activity WHERE submission_id = %s GROUP BY submission_id, job_id, agency_identifier, allocation_transfer_agency, availability_type_code, beginning_period_of_availa, ending_period_of_availabil, main_account_code, RIGHT(object_class, 3), CASE WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '1' THEN 'd' WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '2' THEN 'r' ELSE by_direct_reimbursable_fun END, program_activity_code, program_activity_name, sub_account_code, tas, tas_id": [ + "SELECT submission_id, job_id, agency_identifier,allocation_transfer_agency, availability_type_code, beginning_period_of_availa, ending_period_of_availabil, main_account_code, RIGHT(object_class, 3) AS object_class, CASE WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '1' THEN 'd' WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '2' THEN 'r' ELSE by_direct_reimbursable_fun END AS by_direct_reimbursable_fun, tas, tas_id, program_activity_code, program_activity_name, sub_account_code, SUM(deobligations_recov_by_pro_cpe) AS deobligations_recov_by_pro_cpe, SUM(gross_outlay_amount_by_pro_cpe) AS gross_outlay_amount_by_pro_cpe, SUM(gross_outlay_amount_by_pro_fyb) AS gross_outlay_amount_by_pro_fyb, SUM(gross_outlays_delivered_or_cpe) AS gross_outlays_delivered_or_cpe, SUM(gross_outlays_delivered_or_fyb) AS gross_outlays_delivered_or_fyb, SUM(gross_outlays_undelivered_cpe) AS gross_outlays_undelivered_cpe, SUM(gross_outlays_undelivered_fyb) AS gross_outlays_undelivered_fyb, SUM(obligations_delivered_orde_cpe) AS obligations_delivered_orde_cpe, SUM(obligations_delivered_orde_fyb) AS obligations_delivered_orde_fyb, SUM(obligations_incurred_by_pr_cpe) AS obligations_incurred_by_pr_cpe, SUM(obligations_undelivered_or_cpe) AS obligations_undelivered_or_cpe, SUM(obligations_undelivered_or_fyb) AS obligations_undelivered_or_fyb, SUM(ussgl480100_undelivered_or_cpe) AS ussgl480100_undelivered_or_cpe, SUM(ussgl480100_undelivered_or_fyb) AS ussgl480100_undelivered_or_fyb, SUM(ussgl480200_undelivered_or_cpe) AS ussgl480200_undelivered_or_cpe, SUM(ussgl480200_undelivered_or_fyb) AS ussgl480200_undelivered_or_fyb, SUM(ussgl483100_undelivered_or_cpe) AS ussgl483100_undelivered_or_cpe, SUM(ussgl483200_undelivered_or_cpe) AS ussgl483200_undelivered_or_cpe, SUM(ussgl487100_downward_adjus_cpe) AS ussgl487100_downward_adjus_cpe, SUM(ussgl487200_downward_adjus_cpe) AS ussgl487200_downward_adjus_cpe, SUM(ussgl488100_upward_adjustm_cpe) AS ussgl488100_upward_adjustm_cpe, SUM(ussgl488200_upward_adjustm_cpe) AS ussgl488200_upward_adjustm_cpe, SUM(ussgl490100_delivered_orde_cpe) AS ussgl490100_delivered_orde_cpe, SUM(ussgl490100_delivered_orde_fyb) AS ussgl490100_delivered_orde_fyb, SUM(ussgl490200_delivered_orde_cpe) AS ussgl490200_delivered_orde_cpe, SUM(ussgl490800_authority_outl_cpe) AS ussgl490800_authority_outl_cpe, SUM(ussgl490800_authority_outl_fyb) AS ussgl490800_authority_outl_fyb, SUM(ussgl493100_delivered_orde_cpe) AS ussgl493100_delivered_orde_cpe, SUM(ussgl497100_downward_adjus_cpe) AS ussgl497100_downward_adjus_cpe, SUM(ussgl497200_downward_adjus_cpe) AS ussgl497200_downward_adjus_cpe, SUM(ussgl498100_upward_adjustm_cpe) AS ussgl498100_upward_adjustm_cpe, SUM(ussgl498200_upward_adjustm_cpe) AS ussgl498200_upward_adjustm_cpe FROM certified_object_class_program_activity WHERE submission_id = %s GROUP BY submission_id, job_id, agency_identifier, allocation_transfer_agency, availability_type_code, beginning_period_of_availa, ending_period_of_availabil, main_account_code, RIGHT(object_class, 3), CASE WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '1' THEN 'd' WHEN length(object_class) = 4 AND LEFT(object_class, 1) = '2' THEN 'r' ELSE by_direct_reimbursable_fun END, program_activity_code, program_activity_name, sub_account_code, tas, tas_id": [ { "tas_id": 0, "obligations_delivered_orde_cpe": "4500", From 99205be0b54c667dbd639eb39e03f2ffa6505055 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 00:25:12 -0500 Subject: [PATCH 02/52] refactored IDV response for v2/awards/ to not use serializers --- .../awards/serializers_v2/serializers.py | 3 - .../awards/v2/data_layer/__init__.py | 0 usaspending_api/awards/v2/data_layer/orm.py | 167 ++++++++++++++++++ .../awards/v2/data_layer/orm_mappers.py | 101 +++++++++++ usaspending_api/awards/v2/views/awards.py | 8 +- 5 files changed, 271 insertions(+), 8 deletions(-) create mode 100644 usaspending_api/awards/v2/data_layer/__init__.py create mode 100644 usaspending_api/awards/v2/data_layer/orm.py create mode 100644 usaspending_api/awards/v2/data_layer/orm_mappers.py diff --git a/usaspending_api/awards/serializers_v2/serializers.py b/usaspending_api/awards/serializers_v2/serializers.py index aed9c283f6..cc3f6184aa 100644 --- a/usaspending_api/awards/serializers_v2/serializers.py +++ b/usaspending_api/awards/serializers_v2/serializers.py @@ -1,12 +1,9 @@ -import logging from rest_framework import serializers from usaspending_api.awards.models import Award, TransactionFPDS from usaspending_api.references.models import (Agency, LegalEntity, Location, LegalEntityOfficers, SubtierAgency, ToptierAgency, OfficeAgency) -logger = logging.getLogger("console") - class AwardTypeAwardSpendingSerializer(serializers.Serializer): award_category = serializers.CharField() diff --git a/usaspending_api/awards/v2/data_layer/__init__.py b/usaspending_api/awards/v2/data_layer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py new file mode 100644 index 0000000000..e854714457 --- /dev/null +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -0,0 +1,167 @@ +import copy +import logging +from collections import OrderedDict, MutableMapping +from django.db.models import F + +from usaspending_api.awards.v2.data_layer.orm_mappers import FPDS_CONTRACT_FIELDS, OFFICER_FIELDS, FPDS_AWARD_FIELDS +from usaspending_api.awards.models import Award, TransactionFPDS, ParentAward +from usaspending_api.references.models import Agency, LegalEntity, LegalEntityOfficers + +logger = logging.getLogger("console") + + +def delete_keys_from_dict(dictionary): + modified_dict = OrderedDict() + for key, value in dictionary.items(): + if not key.startswith("_"): + if isinstance(value, MutableMapping): + modified_dict[key] = delete_keys_from_dict(value) + else: + modified_dict[key] = copy.deepcopy(value) + return modified_dict + + +def split_mapper_into_qs(mapper): + values_list = [k for k, v in mapper.items() if k == v] + annotate_dict = {v: F(k) for k, v in mapper.items() if k != v} + + return values_list, annotate_dict + + +def construct_idv_response(requested_award): + """ + [x] base award data + [x] awarding_agency + [x] funding_agency + [x] recipient + [x] place_of_performance + [x] executive_details + [x] idv_dates + [x] latest_transaction_contract_data + """ + f = {"generated_unique_award_id": requested_award} + if str(requested_award).isdigit(): + f = {"pk": requested_award} + + idv_specific_award_fields = OrderedDict( + [ + ("period_of_performance_star", "_start_date"), + ("last_modified", "_last_modified_date"), + ("ordering_period_end_date", "_end_date"), + ] + ) + + mapper = copy.deepcopy(FPDS_CONTRACT_FIELDS) + mapper.update(idv_specific_award_fields) + + response = OrderedDict() + + award = fetch_fpds_award(f) + if not award: + return None + response.update(award) + + response["parent_generated_unique_award_id"] = fetch_parent_award_id(award["generated_unique_award_id"]) + response["executive_details"] = fetch_officers_by_legal_entity_id(award["_lei"]) + response["latest_transaction_contract_data"] = fetch_fpds_details_by_pk(award["_trx"], mapper) + response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) + response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) + response["idv_dates"] = OrderedDict( + [ + ("start_date", response["latest_transaction_contract_data"]["_start_date"]), + ("last_modified_date", response["latest_transaction_contract_data"]["_last_modified_date"]), + ("end_date", response["latest_transaction_contract_data"]["_end_date"]), + ] + ) + response["recipient"] = OrderedDict( + [ + ("recipient_name", response["latest_transaction_contract_data"]["_recipient_name"]), + ("recipient_unique_id", response["latest_transaction_contract_data"]["_recipient_unique_id"]), + ("parent_recipient_unique_id", response["latest_transaction_contract_data"]["_parent_recipient_unique_id"]), + ("business_categories", fetch_business_categories_by_legal_entity_id(award["_lei"])), + ] + ) + response["place_of_performance"] = OrderedDict( + [ + ("location_country_code", response["latest_transaction_contract_data"]["_country_code"]), + ("country_name", response["latest_transaction_contract_data"]["_country_name"]), + ("county_name", response["latest_transaction_contract_data"]["_county_name"]), + ("city_name", response["latest_transaction_contract_data"]["_city_name"]), + ("state_code", response["latest_transaction_contract_data"]["_state_code"]), + ("congressional_code", response["latest_transaction_contract_data"]["_congressional_code"]), + ("zip4", response["latest_transaction_contract_data"]["_zip4"]), + ("zip5", response["latest_transaction_contract_data"]["_zip5"]), + ("address_line1", None), + ("address_line2", None), + ("address_line3", None), + ("foreign_province", None), + ("foreign_postal_code", None), + ] + ) + + return delete_keys_from_dict(response) + + +def fetch_fpds_award(filter_q): + vals, ann = split_mapper_into_qs(FPDS_AWARD_FIELDS) + return Award.objects.filter(**filter_q).values(*vals).annotate(**ann).first() + + +def fetch_parent_award_id(guai): + parent_award = ( + ParentAward.objects.filter(generated_unique_award_id=guai) + .values("parent_award__generated_unique_award_id") + .first() + ) + + return parent_award.get("generated_unique_award_id") if parent_award else None + + +def fetch_fpds_details_by_pk(primary_key, mapper): + vals, ann = split_mapper_into_qs(mapper) + return TransactionFPDS.objects.filter(pk=primary_key).values(*vals).annotate(**ann).first() + + +def fetch_agency_details(agency_id): + values = [ + "toptier_agency__fpds_code", + "toptier_agency__name", + "subtier_agency__subtier_code", + "subtier_agency__name", + "office_agency__name", + ] + agency = Agency.objects.filter(pk=agency_id).values(*values).first() + + agency_details = None + if agency: + agency_details = { + "id": agency_id, + "toptier_agency": {"name": agency["toptier_agency__name"], "code": agency["toptier_agency__fpds_code"]}, + "subtier_agency": {"name": agency["subtier_agency__name"], "code": agency["subtier_agency__subtier_code"]}, + "office_agency_name": agency["office_agency__name"], + } + return agency_details + + +def fetch_business_categories_by_legal_entity_id(legal_entity_id): + le = LegalEntity.objects.filter(pk=legal_entity_id).values("business_categories").first() + + if le: + return le["business_categories"] + return None + + +def fetch_officers_by_legal_entity_id(legal_entity_id): + officer_info = LegalEntityOfficers.objects.filter(pk=legal_entity_id).values(*OFFICER_FIELDS.keys()).first() + + officers = [] + if officer_info: + for x in range(1, 6): + officers.append( + { + "name": officer_info["officer_{}_name".format(x)], + "amount": officer_info["officer_{}_amount".format(x)], + } + ) + + return {"officers": officers} diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py new file mode 100644 index 0000000000..8498620fae --- /dev/null +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -0,0 +1,101 @@ +from collections import OrderedDict + +# For all *_FIELDS ordered dictionaries: +# Key:Value => (DB field, API response field) + +FPDS_AWARD_FIELDS = OrderedDict( + [ + ("id", "id"), + ("generated_unique_award_id", "generated_unique_award_id"), + ("piid", "piid"), + ("parent_award_piid", "parent_award_piid"), + ("type", "type"), + ("category", "category"), + ("type_description", "type_description"), + ("description", "description"), + ("total_obligation", "total_obligation"), + ("base_exercised_options_val", "base_exercised_options_val"), + ("base_and_all_options_value", "base_and_all_options_value"), + ("subaward_count", "subaward_count"), + ("total_subaward_amount", "total_subaward_amount"), + # extra fields + ("recipient_id", "_lei"), + ("latest_transaction_id", "_trx"), + ("awarding_agency_id", "_awarding_agency"), + ("funding_agency_id", "_funding_agency"), + ] +) + + +FPDS_CONTRACT_FIELDS = OrderedDict( + [ + ("idv_type_description", "idv_type_description"), + ("type_of_idc_description", "type_of_idc_description"), + ("referenced_idv_agency_iden", "referenced_idv_agency_iden"), + ("multiple_or_single_aw_desc", "multiple_or_single_aw_desc"), + ("solicitation_identifier", "solicitation_identifier"), + ("solicitation_procedures", "solicitation_procedures"), + ("number_of_offers_received", "number_of_offers_received"), + ("extent_competed", "extent_competed"), + ("other_than_full_and_o_desc", "other_than_full_and_o_desc"), + ("type_set_aside_description", "type_set_aside_description"), + ("commercial_item_acquisitio", "commercial_item_acquisitio"), + ("commercial_item_test_desc", "commercial_item_test_desc"), + ("evaluated_preference_desc", "evaluated_preference_desc"), + ("fed_biz_opps_description", "fed_biz_opps_description"), + ("small_business_competitive", "small_business_competitive"), + ("fair_opportunity_limi_desc", "fair_opportunity_limi_desc"), + ("product_or_service_code", "product_or_service_code"), + ("product_or_service_co_desc", "product_or_service_co_desc"), + ("naics", "naics"), + ("dod_claimant_program_code", "dod_claimant_program_code"), + ("program_system_or_equipmen", "program_system_or_equipmen"), + ("information_technolog_desc", "information_technolog_desc"), + ("sea_transportation_desc", "sea_transportation_desc"), + ("clinger_cohen_act_pla_desc", "clinger_cohen_act_pla_desc"), + ("construction_wage_rat_desc", "construction_wage_rat_desc"), + ("labor_standards_descrip", "labor_standards_descrip"), + ("materials_supplies_descrip", "materials_supplies_descrip"), + ("cost_or_pricing_data_desc", "cost_or_pricing_data_desc"), + ("domestic_or_foreign_e_desc", "domestic_or_foreign_e_desc"), + ("foreign_funding_desc", "foreign_funding_desc"), + ("interagency_contract_desc", "interagency_contract_desc"), + ("major_program", "major_program"), + ("price_evaluation_adjustmen", "price_evaluation_adjustmen"), + ("program_acronym", "program_acronym"), + ("subcontracting_plan", "subcontracting_plan"), + ("multi_year_contract_desc", "multi_year_contract_desc"), + ("purchase_card_as_paym_desc", "purchase_card_as_paym_desc"), + ("consolidated_contract_desc", "consolidated_contract_desc"), + ("type_of_contract_pric_desc", "type_of_contract_pric_desc"), + + # "Legal Entity" fields below + ("awardee_or_recipient_legal", "_recipient_name"), + ("awardee_or_recipient_uniqu", "_recipient_unique_id"), + ("ultimate_parent_unique_ide", "_parent_recipient_unique_id"), + + # "Place of Performance Location" + ("place_of_perform_country_c", "_country_code"), + ("place_of_perf_country_desc", "_country_name"), + ("place_of_performance_state", "_state_code"), + ("place_of_perform_city_name", "_city_name"), + ("place_of_perform_county_na", "_county_name"), + ("place_of_performance_zip4a", "_zip4"), + ("place_of_performance_congr", "_congressional_code"), + ("place_of_performance_zip5", "_zip5"), + ] +) + + +OFFICER_FIELDS = OrderedDict([ + ("officer_1_name", "officer_1_name"), + ("officer_1_amount", "officer_1_amount"), + ("officer_2_name", "officer_2_name"), + ("officer_2_amount", "officer_2_amount"), + ("officer_3_name", "officer_3_name"), + ("officer_3_amount", "officer_3_amount"), + ("officer_4_name", "officer_4_name"), + ("officer_4_amount", "officer_4_amount"), + ("officer_5_name", "officer_5_name"), + ("officer_5_amount", "officer_5_amount"), +]) diff --git a/usaspending_api/awards/v2/views/awards.py b/usaspending_api/awards/v2/views/awards.py index 7f97ef8188..b6019a74a1 100644 --- a/usaspending_api/awards/v2/views/awards.py +++ b/usaspending_api/awards/v2/views/awards.py @@ -9,8 +9,8 @@ from usaspending_api.awards.models import Award, ParentAward -from usaspending_api.awards.serializers_v2.serializers import AwardContractSerializerV2, AwardMiscSerializerV2,\ - AwardIDVSerializerV2 +from usaspending_api.awards.v2.data_layer.orm import construct_idv_response +from usaspending_api.awards.serializers_v2.serializers import AwardContractSerializerV2, AwardMiscSerializerV2 from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.exceptions import UnprocessableEntityException from usaspending_api.common.views import APIDocumentationView @@ -73,9 +73,7 @@ def _business_logic(self, request_dict: dict) -> dict: serialized = AwardContractSerializerV2(award).data serialized['recipient']['parent_recipient_name'] = parent_recipient_name elif award.category == "idv": - parent_recipient_name = award.latest_transaction.contract_data.ultimate_parent_legal_enti - serialized = AwardIDVSerializerV2(award).data - serialized['recipient']['parent_recipient_name'] = parent_recipient_name + serialized = construct_idv_response(request_dict[dict_key]) else: parent_recipient_name = award.latest_transaction.assistance_data.ultimate_parent_legal_enti serialized = AwardMiscSerializerV2(award).data From 68c90e62f5b10c4158bbfcf05d89eae98665adb9 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 00:26:52 -0500 Subject: [PATCH 03/52] removed idv serializers --- .../awards/serializers_v2/serializers.py | 97 ------------------- 1 file changed, 97 deletions(-) diff --git a/usaspending_api/awards/serializers_v2/serializers.py b/usaspending_api/awards/serializers_v2/serializers.py index cc3f6184aa..e55c0ebd1e 100644 --- a/usaspending_api/awards/serializers_v2/serializers.py +++ b/usaspending_api/awards/serializers_v2/serializers.py @@ -350,103 +350,6 @@ class Meta: } -class AwardIDVSerializerV2(LimitableSerializerV2): - latest_transaction_contract_data = serializers.SerializerMethodField('latest_transaction_func') - idv_dates = serializers.SerializerMethodField("idv_dates_func") - executive_details = serializers.SerializerMethodField("executive_details_func") - parent_generated_unique_award_id = serializers.SerializerMethodField('parent_unique_id_func') - - def idv_dates_func(self, award): - transaction_data = self.latest_transaction_func(award) - return { - "start_date": award.period_of_performance_start_date, - "last_modified_date": award.last_modified_date, - "end_date": transaction_data["ordering_period_end_date"] - } - - def latest_transaction_func(self, award): - return TransactionFPDSSerializerV2(award.latest_transaction.contract_data).data - - def executive_details_func(self, award): - entity = LegalEntityOfficerPassThroughSerializerV2(award.recipient).data - response = [] - if "officers" in entity and entity["officers"]: - for x in range(1, 6): - response.append({"name": entity["officers"]["officer_" + str(x) + "_name"], - "amount": entity["officers"]["officer_" + str(x) + "_amount"]}) - return {"officers": response} - - def parent_unique_id_func(self, award): - parent_transaction = TransactionFPDS.objects.filter(agency_id=award.fpds_parent_agency_id, - piid=award.parent_award_piid).values( - "agency_id", - "referenced_idv_agency_iden", - "piid", - "parent_award_id").first() - - if parent_transaction: - parent_generated_unique_id = ( - "CONT_AW_" + - (parent_transaction["agency_id"] if parent_transaction["agency_id"] else "-NONE-") + - "_" + - (parent_transaction["referenced_idv_agency_iden"] if parent_transaction[ - "referenced_idv_agency_iden"] else "-NONE-") + - "_" + - (parent_transaction["piid"] if parent_transaction["piid"] else "-NONE-") + - "_" + - (parent_transaction["parent_award_id"] if parent_transaction["parent_award_id"] else "-NONE-") - ) - else: - parent_generated_unique_id = None - - return parent_generated_unique_id - - class Meta: - - model = Award - fields = [ - "id", - "generated_unique_award_id", - "piid", - "parent_generated_unique_award_id", - "parent_award_piid", - "type", - "category", - "type_description", - "description", - "total_obligation", - "base_exercised_options_val", - "base_and_all_options_value", - "subaward_count", - "total_subaward_amount", - "awarding_agency", - "funding_agency", - "recipient", - "place_of_performance", - "executive_details", - "idv_dates", - "latest_transaction_contract_data", - ] - nested_serializers = { - "recipient": { - "class": LegalEntitySerializerV2, - "kwargs": {"read_only": True} - }, - "awarding_agency": { - "class": AgencySerializerV2, - "kwargs": {"read_only": True} - }, - "funding_agency": { - "class": AgencySerializerV2, - "kwargs": {"read_only": True} - }, - "place_of_performance": { - "class": LocationSerializerV2, - "kwargs": {"read_only": True} - } - } - - class AwardMiscSerializerV2(LimitableSerializerV2): period_of_performance = serializers.SerializerMethodField("period_of_performance_func") executive_details = serializers.SerializerMethodField("executive_details_func") From 3db6ca0ed937d371ede5c633cd9ef77961057d0d Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 02:39:32 -0500 Subject: [PATCH 04/52] refactored all of v2/awards to not use serializers --- .../awards/serializers_v2/serializers.py | 406 ------------------ usaspending_api/awards/v2/data_layer/orm.py | 227 ++++++++-- .../awards/v2/data_layer/orm_mappers.py | 108 ++++- .../awards/v2/data_layer/orm_utils.py | 21 + usaspending_api/awards/v2/views/awards.py | 21 +- 5 files changed, 323 insertions(+), 460 deletions(-) create mode 100644 usaspending_api/awards/v2/data_layer/orm_utils.py diff --git a/usaspending_api/awards/serializers_v2/serializers.py b/usaspending_api/awards/serializers_v2/serializers.py index e55c0ebd1e..3896f19f2f 100644 --- a/usaspending_api/awards/serializers_v2/serializers.py +++ b/usaspending_api/awards/serializers_v2/serializers.py @@ -1,9 +1,5 @@ from rest_framework import serializers -from usaspending_api.awards.models import Award, TransactionFPDS -from usaspending_api.references.models import (Agency, LegalEntity, Location, LegalEntityOfficers, - SubtierAgency, ToptierAgency, OfficeAgency) - class AwardTypeAwardSpendingSerializer(serializers.Serializer): award_category = serializers.CharField() @@ -19,405 +15,3 @@ class RecipientAwardSpendingSerializer(serializers.Serializer): award_category = serializers.CharField() obligated_amount = serializers.DecimalField(None, 2) recipient = RecipientSerializer(source='*') - - -class LimitableSerializerV2(serializers.ModelSerializer): - """Extends the model serializer to support field limiting.""" - - def __init__(self, *args, **kwargs): - - # Grab any kwargs include and exclude fields, these are typically - # passed in by a parent serializer to the child serializer - include_fields = kwargs.pop('fields', []) - - # Initialize now that kwargs have been cleared - super(LimitableSerializerV2, self).__init__(*args, **kwargs) - - if len(include_fields) == 0 and hasattr(self.Meta, "default_fields"): - include_fields = self.Meta.default_fields - - # For the include list, we need to include the parent field of - # any nested fields - include_fields = include_fields + self.identify_missing_children(self.Meta.model, include_fields) - - # Create and initialize the child serializers - if hasattr(self.Meta, "nested_serializers"): - # Initialize the child serializers - children = self.Meta.nested_serializers - for field in children.keys(): - # Get child kwargs - kwargs = children[field].get("kwargs", {}) - - # Pop the default fields from the child serializer kwargs - default_fields = kwargs.pop("default_fields", []) - - child_include_fields, matched_include = self.get_child_fields(field, include_fields) - - if len(child_include_fields) == 0: - child_include_fields = default_fields - - child_args = { - **kwargs, - "fields": child_include_fields, - } - self.fields[field] = children[field]["class"](**child_args) - - if len(include_fields) > 0: - allowed = set(include_fields) - existing = set(self.fields) - for field_name in existing - allowed: - # If we have a coded field, always include its description - if field_name.split("_")[-1] == "description" and "_".join(field_name.split("_")[:-1]) in allowed: - continue - self.fields.pop(field_name) - - # Returns a list of child names for fields in the field list - # This is necessary so that if a user requests "recipient__recipient_name" - # we also include "recipient" so that it is serialized - def identify_missing_children(self, model, fields): - children = [] - model_fields = [f.name for f in model._meta.get_fields()] - for field in fields: - split = field.split("__") - if len(split) > 0 and split[0] in model_fields and split[0] not in fields: - children.append(split[0]) - - return children - - # Takes a child's name and a list of fields, and returns a set of fields - # that belong to that child - def get_child_fields(self, child, fields): - # An included field is a child's field if the field begins with that - # child's name and two underscores - pattern = "{}__".format(child) - - matched = [] - child_fields = [] - for field in fields: - if field[:len(pattern)] == pattern: - child_fields.append(field[len(pattern):]) - matched.append(field) - return child_fields, matched - - -class LegalEntityOfficersSerializerV2(LimitableSerializerV2): - class Meta: - model = LegalEntityOfficers - fields = [ - "officer_1_name", - "officer_1_amount", - "officer_2_name", - "officer_2_amount", - "officer_3_name", - "officer_3_amount", - "officer_4_name", - "officer_4_amount", - "officer_5_name", - "officer_5_amount", - ] - - -class ToptierAgencySerializerV2(LimitableSerializerV2): - - class Meta: - model = ToptierAgency - fields = [ - "name", - "abbreviation" - ] - - -class SubtierAgencySerializerV2(LimitableSerializerV2): - - class Meta: - model = SubtierAgency - fields = [ - - "name", - "abbreviation" - ] - - -class OfficeAgencySerializerV2(LimitableSerializerV2): - - class Meta: - model = OfficeAgency - fields = [ - "aac_code", - "name" - ] - - -class LocationSerializerV2(LimitableSerializerV2): - - class Meta: - model = Location - fields = [ - "address_line1", - "address_line2", - "address_line3", - "foreign_province", - "city_name", - "county_name", - "state_code", - "zip5", - "zip4", - "foreign_postal_code", - "country_name", - "location_country_code", - "congressional_code" - ] - - -class AgencySerializerV2(LimitableSerializerV2): - office_agency_name = serializers.SerializerMethodField('office_agency_name_func') - - def office_agency_name_func(self, agency): - if agency.office_agency: - return agency.office_agency.name - else: - return None - - class Meta: - model = Agency - fields = [ - "id", - "toptier_agency", - "subtier_agency", - "office_agency_name", - ] - nested_serializers = { - "toptier_agency": { - "class": ToptierAgencySerializerV2, - "kwargs": {"read_only": True} - }, - "subtier_agency": { - "class": SubtierAgencySerializerV2, - "kwargs": {"read_only": True} - }, - "office_agency": { - "class": OfficeAgencySerializerV2, - "kwargs": {"read_only": True} - }, - } - - -class LegalEntitySerializerV2(LimitableSerializerV2): - - class Meta: - model = LegalEntity - fields = [ - "recipient_name", - "recipient_unique_id", - "parent_recipient_unique_id", - "business_categories", - "location" - ] - nested_serializers = { - "location": { - "class": LocationSerializerV2, - "kwargs": {"read_only": True} - } - } - - -class LegalEntityOfficerPassThroughSerializerV2(LimitableSerializerV2): - - class Meta: - model = LegalEntity - fields = [ - "officers" - ] - nested_serializers = { - "officers": { - "class": LegalEntityOfficersSerializerV2, - "kwargs": {"read_only": True} - } - } - - -class TransactionFPDSSerializerV2(LimitableSerializerV2): - - class Meta: - model = TransactionFPDS - fields = [ - 'idv_type_description', - 'type_of_idc_description', - 'referenced_idv_agency_iden', - 'multiple_or_single_aw_desc', - 'solicitation_identifier', - 'solicitation_procedures', - 'number_of_offers_received', - 'extent_competed', - 'other_than_full_and_o_desc', - 'type_set_aside_description', - 'commercial_item_acquisitio', - 'commercial_item_test_desc', - 'evaluated_preference_desc', - 'fed_biz_opps_description', - 'small_business_competitive', - 'fair_opportunity_limi_desc', - 'product_or_service_code', - 'product_or_service_co_desc', - 'naics', - 'dod_claimant_program_code', - 'program_system_or_equipmen', - 'information_technolog_desc', - 'sea_transportation_desc', - 'clinger_cohen_act_pla_desc', - 'construction_wage_rat_desc', - 'labor_standards_descrip', - 'materials_supplies_descrip', - 'cost_or_pricing_data_desc', - 'domestic_or_foreign_e_desc', - 'foreign_funding_desc', - 'interagency_contract_desc', - 'major_program', - 'price_evaluation_adjustmen', - 'program_acronym', - 'subcontracting_plan', - 'multi_year_contract_desc', - 'purchase_card_as_paym_desc', - 'consolidated_contract_desc', - 'type_of_contract_pric_desc', - 'ordering_period_end_date', - ] - - -class AwardContractSerializerV2(LimitableSerializerV2): - executive_details = serializers.SerializerMethodField("executive_details_func") - period_of_performance = serializers.SerializerMethodField("period_of_performance_func") - latest_transaction_contract_data = serializers.SerializerMethodField('latest_transaction_func') - - def latest_transaction_func(self, award): - return TransactionFPDSSerializerV2(award.latest_transaction.contract_data).data - - def period_of_performance_func(self, award): - return { - "period_of_performance_start_date": award.period_of_performance_start_date, - "period_of_performance_current_end_date": award.period_of_performance_current_end_date - } - - def executive_details_func(self, award): - entity = LegalEntityOfficerPassThroughSerializerV2(award.recipient).data - response = [] - if "officers" in entity and entity["officers"]: - for x in range(1, 6): - response.append({"name": entity["officers"]["officer_" + str(x) + "_name"], - "amount": entity["officers"]["officer_" + str(x) + "_amount"]}) - return {"officers": response} - - class Meta: - - model = Award - fields = [ - "id", - "type", - "category", - "type_description", - "piid", - "parent_award_piid", - "description", - "awarding_agency", - "funding_agency", - "recipient", - "total_obligation", - "base_and_all_options_value", - "period_of_performance", - "place_of_performance", - "latest_transaction_contract_data", - "subaward_count", - "total_subaward_amount", - "executive_details" - ] - nested_serializers = { - "recipient": { - "class": LegalEntitySerializerV2, - "kwargs": {"read_only": True} - }, - "awarding_agency": { - "class": AgencySerializerV2, - "kwargs": {"read_only": True} - }, - "funding_agency": { - "class": AgencySerializerV2, - "kwargs": {"read_only": True} - }, - "place_of_performance": { - "class": LocationSerializerV2, - "kwargs": {"read_only": True} - }, - } - - -class AwardMiscSerializerV2(LimitableSerializerV2): - period_of_performance = serializers.SerializerMethodField("period_of_performance_func") - executive_details = serializers.SerializerMethodField("executive_details_func") - cfda_objectives = serializers.SerializerMethodField('cfda_objectives_func') - cfda_title = serializers.SerializerMethodField('cfda_title_func') - cfda_number = serializers.SerializerMethodField('cfda_number_func') - - def cfda_objectives_func(self, award): - return award.latest_transaction.assistance_data.cfda_objectives - - def cfda_title_func(self, award): - return award.latest_transaction.assistance_data.cfda_title - - def cfda_number_func(self, award): - return award.latest_transaction.assistance_data.cfda_number - - def period_of_performance_func(self, award): - return { - "period_of_performance_start_date": award.period_of_performance_start_date, - "period_of_performance_current_end_date": award.period_of_performance_current_end_date - } - - def executive_details_func(self, award): - entity = LegalEntityOfficerPassThroughSerializerV2(award.recipient).data - response = [] - if "officers" in entity and entity["officers"]: - for x in range(1, 6): - response.append({"name": entity["officers"]["officer_" + str(x) + "_name"], - "amount": entity["officers"]["officer_" + str(x) + "_amount"]}) - return {"officers": response} - - class Meta: - - model = Award - fields = [ - "id", - "type", - "category", - "type_description", - "piid", - "description", - "cfda_objectives", - "cfda_number", - "cfda_title", - "awarding_agency", - "funding_agency", - "recipient", - "subaward_count", - "total_subaward_amount", - "period_of_performance", - "place_of_performance", - "executive_details", - ] - nested_serializers = { - "recipient": { - "class": LegalEntitySerializerV2, - "kwargs": {"read_only": True} - }, - "awarding_agency": { - "class": AgencySerializerV2, - "kwargs": {"read_only": True} - }, - "funding_agency": { - "class": AgencySerializerV2, - "kwargs": {"read_only": True} - }, - "place_of_performance": { - "class": LocationSerializerV2, - "kwargs": {"read_only": True} - } - } diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index e854714457..38794e0750 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -1,31 +1,177 @@ import copy -import logging -from collections import OrderedDict, MutableMapping -from django.db.models import F +from collections import OrderedDict + +from usaspending_api.awards.v2.data_layer.orm_mappers import ( + FABS_AWARD_FIELDS, + FPDS_CONTRACT_FIELDS, + OFFICER_FIELDS, + FPDS_AWARD_FIELDS, + FABS_ASSISTANCE_FIELDS, +) +from usaspending_api.awards.models import Award, TransactionFABS, TransactionFPDS, ParentAward +from usaspending_api.recipient.models import RecipientLookup +from usaspending_api.references.models import Agency, LegalEntity, LegalEntityOfficers, Cfda +from usaspending_api.awards.v2.data_layer.orm_utils import delete_keys_from_dict, split_mapper_into_qs + + +def construct_assistance_response(requested_award): + """ + [x] base award data + [x] awarding_agency + [x] funding_agency + [x] recipient + [x] place_of_performance + [x] period_of_performance + """ + f = {"generated_unique_award_id": requested_award} + if str(requested_award).isdigit(): + f = {"pk": requested_award} + + response = OrderedDict() -from usaspending_api.awards.v2.data_layer.orm_mappers import FPDS_CONTRACT_FIELDS, OFFICER_FIELDS, FPDS_AWARD_FIELDS -from usaspending_api.awards.models import Award, TransactionFPDS, ParentAward -from usaspending_api.references.models import Agency, LegalEntity, LegalEntityOfficers + award = fetch_award_details(f, FABS_AWARD_FIELDS) + if not award: + return None + response.update(award) -logger = logging.getLogger("console") + transaction = fetch_fabs_details_by_pk(award["_trx"], FABS_ASSISTANCE_FIELDS) + cfda_info = fetch_cfda_details_using_cfda_number(transaction["cfda_number"]) + response["cfda_number"] = transaction["cfda_number"] + response["cfda_objectives"] = cfda_info.get("objectives") + response["cfda_title"] = cfda_info.get("program_title") -def delete_keys_from_dict(dictionary): - modified_dict = OrderedDict() - for key, value in dictionary.items(): - if not key.startswith("_"): - if isinstance(value, MutableMapping): - modified_dict[key] = delete_keys_from_dict(value) - else: - modified_dict[key] = copy.deepcopy(value) - return modified_dict + response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) + response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) + response["period_of_performance"] = OrderedDict( + [ + ("period_of_performance_start_date", award["_start_date"]), + ("period_of_performance_current_end_date", award["_end_date"]), + ] + ) + response["recipient"] = OrderedDict( + [ + ( + "recipient_hash", + fetch_recipient_hash_using_name_and_duns( + transaction["_recipient_name"], transaction["_recipient_unique_id"] + ), + ), + ("recipient_name", transaction["_recipient_name"]), + ("recipient_unique_id", transaction["_recipient_unique_id"]), + ("parent_recipient_unique_id", transaction["_parent_recipient_unique_id"]), + ("parent_recipient_name", transaction["_parent_recipient_name"]), + ("business_categories", fetch_business_categories_by_legal_entity_id(award["_lei"])), + ( + "location", + OrderedDict( + [ + ("legal_entity_country_code", transaction["_rl_location_country_code"]), + ("legal_entity_country_name", transaction["_rl_country_name"]), + ("legal_entity_state_code", transaction["_rl_state_code"]), + ("legal_entity_city_name", transaction["_rl_city_name"]), + ("legal_entity_county_name", transaction["_rl_county_name"]), + ("legal_entity_address_line1", transaction["_rl_address_line1"]), + ("legal_entity_address_line2", transaction["_rl_address_line2"]), + ("legal_entity_address_line3", transaction["_rl_address_line3"]), + ("legal_entity_congressional", transaction["_rl_congressional_code"]), + ("legal_entity_zip_last4", transaction["_rl_zip4"]), + ("legal_entity_zip5", transaction["_rl_zip5"]), + ("legal_entity_foreign_posta", transaction["_rl_foreign_postal_code"]), + ("legal_entity_foreign_provi", transaction["_rl_foreign_province"]), + ] + ), + ), + ] + ) + response["place_of_performance"] = OrderedDict( + [ + ("location_country_code", transaction["_pop_location_country_code"]), + ("country_name", transaction["_pop_country_name"]), + ("county_name", transaction["_pop_county_name"]), + ("city_name", transaction["_pop_city_name"]), + ("state_code", transaction["_pop_state_code"]), + ("congressional_code", transaction["_pop_congressional_code"]), + ("zip4", transaction["_pop_zip4"]), + ("zip5", transaction["_pop_zip5"]), + ("address_line1", None), + ("address_line2", None), + ("address_line3", None), + ("foreign_province", transaction["_pop_foreign_province"]), + ("foreign_postal_code", None), + ] + ) + return delete_keys_from_dict(response) -def split_mapper_into_qs(mapper): - values_list = [k for k, v in mapper.items() if k == v] - annotate_dict = {v: F(k) for k, v in mapper.items() if k != v} - return values_list, annotate_dict +def construct_contract_response(requested_award): + """ + [x] base award data + [x] awarding_agency + [x] funding_agency + [x] recipient + [x] place_of_performance + [x] executive_details + [x] period_of_performance + [x] latest_transaction_contract_data + """ + f = {"generated_unique_award_id": requested_award} + if str(requested_award).isdigit(): + f = {"pk": requested_award} + + response = OrderedDict() + + award = fetch_award_details(f, FPDS_AWARD_FIELDS) + if not award: + return None + response.update(award) + + response["executive_details"] = fetch_officers_by_legal_entity_id(award["_lei"]) + response["latest_transaction_contract_data"] = fetch_fpds_details_by_pk(award["_trx"], FPDS_CONTRACT_FIELDS) + response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) + response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) + response["period_of_performance"] = OrderedDict( + [ + ("period_of_performance_start_date", award["_start_date"]), + ("period_of_performance_current_end_date", award["_end_date"]), + ] + ) + response["recipient"] = OrderedDict( + [ + ( + "recipient_hash", + fetch_recipient_hash_using_name_and_duns( + response["latest_transaction_contract_data"]["_recipient_name"], + response["latest_transaction_contract_data"]["_recipient_unique_id"], + ), + ), + ("recipient_name", response["latest_transaction_contract_data"]["_recipient_name"]), + ("recipient_unique_id", response["latest_transaction_contract_data"]["_recipient_unique_id"]), + ("parent_recipient_name", response["latest_transaction_contract_data"]["_parent_recipient_name"]), + ("parent_recipient_unique_id", response["latest_transaction_contract_data"]["_parent_recipient_unique_id"]), + ("business_categories", fetch_business_categories_by_legal_entity_id(award["_lei"])), + ] + ) + response["place_of_performance"] = OrderedDict( + [ + ("location_country_code", response["latest_transaction_contract_data"]["_country_code"]), + ("country_name", response["latest_transaction_contract_data"]["_country_name"]), + ("county_name", response["latest_transaction_contract_data"]["_county_name"]), + ("city_name", response["latest_transaction_contract_data"]["_city_name"]), + ("state_code", response["latest_transaction_contract_data"]["_state_code"]), + ("congressional_code", response["latest_transaction_contract_data"]["_congressional_code"]), + ("zip4", response["latest_transaction_contract_data"]["_zip4"]), + ("zip5", response["latest_transaction_contract_data"]["_zip5"]), + ("address_line1", None), + ("address_line2", None), + ("address_line3", None), + ("foreign_province", None), + ("foreign_postal_code", None), + ] + ) + + return delete_keys_from_dict(response) def construct_idv_response(requested_award): @@ -56,7 +202,7 @@ def construct_idv_response(requested_award): response = OrderedDict() - award = fetch_fpds_award(f) + award = fetch_award_details(f, FPDS_AWARD_FIELDS) if not award: return None response.update(award) @@ -68,15 +214,23 @@ def construct_idv_response(requested_award): response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) response["idv_dates"] = OrderedDict( [ - ("start_date", response["latest_transaction_contract_data"]["_start_date"]), + ("start_date", award["_start_date"]), ("last_modified_date", response["latest_transaction_contract_data"]["_last_modified_date"]), ("end_date", response["latest_transaction_contract_data"]["_end_date"]), ] ) response["recipient"] = OrderedDict( [ + ( + "recipient_hash", + fetch_recipient_hash_using_name_and_duns( + response["latest_transaction_contract_data"]["_recipient_name"], + response["latest_transaction_contract_data"]["_recipient_unique_id"], + ), + ), ("recipient_name", response["latest_transaction_contract_data"]["_recipient_name"]), ("recipient_unique_id", response["latest_transaction_contract_data"]["_recipient_unique_id"]), + ("parent_recipient_name", response["latest_transaction_contract_data"]["_parent_recipient_name"]), ("parent_recipient_unique_id", response["latest_transaction_contract_data"]["_parent_recipient_unique_id"]), ("business_categories", fetch_business_categories_by_legal_entity_id(award["_lei"])), ] @@ -102,8 +256,8 @@ def construct_idv_response(requested_award): return delete_keys_from_dict(response) -def fetch_fpds_award(filter_q): - vals, ann = split_mapper_into_qs(FPDS_AWARD_FIELDS) +def fetch_award_details(filter_q, mapper_fields): + vals, ann = split_mapper_into_qs(mapper_fields) return Award.objects.filter(**filter_q).values(*vals).annotate(**ann).first() @@ -117,6 +271,11 @@ def fetch_parent_award_id(guai): return parent_award.get("generated_unique_award_id") if parent_award else None +def fetch_fabs_details_by_pk(primary_key, mapper): + vals, ann = split_mapper_into_qs(mapper) + return TransactionFABS.objects.filter(pk=primary_key).values(*vals).annotate(**ann).first() + + def fetch_fpds_details_by_pk(primary_key, mapper): vals, ann = split_mapper_into_qs(mapper) return TransactionFPDS.objects.filter(pk=primary_key).values(*vals).annotate(**ann).first() @@ -165,3 +324,23 @@ def fetch_officers_by_legal_entity_id(legal_entity_id): ) return {"officers": officers} + + +def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id): + recipient = RecipientLookup.objects.filter(duns=recipient_unique_id).values("recipient_hash").first() + + if not recipient: + # SQL: MD5(UPPER(CONCAT(awardee_or_recipient_uniqu, legal_business_name)))::uuid + import hashlib + import uuid + + h = hashlib.md5("{}{}".format(recipient_unique_id, recipient_name).upper().encode("utf-8")).hexdigest() + return str(uuid.UUID(h)) + return recipient["recipient_hash"] + + +def fetch_cfda_details_using_cfda_number(cfda): + c = Cfda.objects.filter(program_number=cfda).values("program_title", "objectives").first() + if not c: + return {} + return c diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index 8498620fae..f0322e1543 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -3,14 +3,50 @@ # For all *_FIELDS ordered dictionaries: # Key:Value => (DB field, API response field) +FABS_AWARD_FIELDS = OrderedDict( + [ + ("id", "id"), + ("generated_unique_award_id", "generated_unique_award_id"), + ("fain", "fain"), + ("uri", "uri"), + ("category", "category"), + ("type", "type"), + ("type_description", "type_description"), + ("description", "description"), + ("subaward_count", "subaward_count"), + ("total_subaward_amount", "total_subaward_amount"), + ("awarding_agency", "awarding_agency"), + ("funding_agency", "funding_agency"), + ("recipient", "recipient"), + ("subaward_count", "subaward_count"), + ("total_subaward_amount", "total_subaward_amount"), + ("total_subsidy_cost", "total_subsidy_cost"), + ("total_loan_value", "total_loan_value"), + ("total_obligation", "total_obligation"), + ("base_and_all_options_value", "base_and_all_options_value"), + # ("funding_obligated", "funding_obligated"), + ("base_exercised_options_val", "base_exercised_options"), + ("non_federal_funding_amount", "non_federal_funding"), + ("total_funding_amount", "total_funding"), + # extra fields + ("recipient_id", "_lei"), + ("latest_transaction_id", "_trx"), + ("awarding_agency_id", "_awarding_agency"), + ("funding_agency_id", "_funding_agency"), + ("period_of_performance_start_date", "_start_date"), + ("period_of_performance_current_end_date", "_end_date"), + ] +) + + FPDS_AWARD_FIELDS = OrderedDict( [ ("id", "id"), ("generated_unique_award_id", "generated_unique_award_id"), ("piid", "piid"), ("parent_award_piid", "parent_award_piid"), - ("type", "type"), ("category", "category"), + ("type", "type"), ("type_description", "type_description"), ("description", "description"), ("total_obligation", "total_obligation"), @@ -23,6 +59,45 @@ ("latest_transaction_id", "_trx"), ("awarding_agency_id", "_awarding_agency"), ("funding_agency_id", "_funding_agency"), + ("period_of_performance_start_date", "_start_date"), + ("period_of_performance_current_end_date", "_end_date"), + ] +) + + +FABS_ASSISTANCE_FIELDS = OrderedDict( + [ + ("cfda_number", "cfda_number"), + # "Recipient" fields below + ("awardee_or_recipient_legal", "_recipient_name"), + ("awardee_or_recipient_uniqu", "_recipient_unique_id"), + ("ultimate_parent_legal_enti", "_parent_recipient_name"), + ("ultimate_parent_unique_ide", "_parent_recipient_unique_id"), + + ("legal_entity_country_code", "_rl_location_country_code"), + ("legal_entity_country_name", "_rl_country_name"), + ("legal_entity_state_code", "_rl_state_code"), + ("legal_entity_city_name", "_rl_city_name"), + ("legal_entity_county_name", "_rl_county_name"), + ("legal_entity_address_line1", "_rl_address_line1"), + ("legal_entity_address_line2", "_rl_address_line2"), + ("legal_entity_address_line3", "_rl_address_line3"), + ("legal_entity_congressional", "_rl_congressional_code"), + ("legal_entity_zip_last4", "_rl_zip4"), + ("legal_entity_zip5", "_rl_zip5"), + ("legal_entity_foreign_posta", "_rl_foreign_postal_code"), + ("legal_entity_foreign_provi", "_rl_foreign_province"), + + # "Place of Performance" fields below + ("place_of_perform_country_c", "_pop_location_country_code"), + ("place_of_perform_country_n", "_pop_country_name"), + ("place_of_perform_county_na", "_pop_county_name"), + ("place_of_performance_city", "_pop_city_name"), + ("place_of_perfor_state_code", "_pop_state_code"), + ("place_of_performance_congr", "_pop_congressional_code"), + ("place_of_perform_zip_last4", "_pop_zip4"), + ("place_of_performance_zip5", "_pop_zip5"), + ("place_of_performance_forei", "_pop_foreign_province"), ] ) @@ -68,12 +143,11 @@ ("purchase_card_as_paym_desc", "purchase_card_as_paym_desc"), ("consolidated_contract_desc", "consolidated_contract_desc"), ("type_of_contract_pric_desc", "type_of_contract_pric_desc"), - - # "Legal Entity" fields below + # "Recipient" fields below ("awardee_or_recipient_legal", "_recipient_name"), ("awardee_or_recipient_uniqu", "_recipient_unique_id"), + ("ultimate_parent_legal_enti", "_parent_recipient_name"), ("ultimate_parent_unique_ide", "_parent_recipient_unique_id"), - # "Place of Performance Location" ("place_of_perform_country_c", "_country_code"), ("place_of_perf_country_desc", "_country_name"), @@ -87,15 +161,17 @@ ) -OFFICER_FIELDS = OrderedDict([ - ("officer_1_name", "officer_1_name"), - ("officer_1_amount", "officer_1_amount"), - ("officer_2_name", "officer_2_name"), - ("officer_2_amount", "officer_2_amount"), - ("officer_3_name", "officer_3_name"), - ("officer_3_amount", "officer_3_amount"), - ("officer_4_name", "officer_4_name"), - ("officer_4_amount", "officer_4_amount"), - ("officer_5_name", "officer_5_name"), - ("officer_5_amount", "officer_5_amount"), -]) +OFFICER_FIELDS = OrderedDict( + [ + ("officer_1_name", "officer_1_name"), + ("officer_1_amount", "officer_1_amount"), + ("officer_2_name", "officer_2_name"), + ("officer_2_amount", "officer_2_amount"), + ("officer_3_name", "officer_3_name"), + ("officer_3_amount", "officer_3_amount"), + ("officer_4_name", "officer_4_name"), + ("officer_4_amount", "officer_4_amount"), + ("officer_5_name", "officer_5_name"), + ("officer_5_amount", "officer_5_amount"), + ] +) diff --git a/usaspending_api/awards/v2/data_layer/orm_utils.py b/usaspending_api/awards/v2/data_layer/orm_utils.py new file mode 100644 index 0000000000..e63e9da399 --- /dev/null +++ b/usaspending_api/awards/v2/data_layer/orm_utils.py @@ -0,0 +1,21 @@ +from collections import OrderedDict, MutableMapping +from copy import deepcopy +from django.db.models import F + + +def delete_keys_from_dict(dictionary): + modified_dict = OrderedDict() + for key, value in dictionary.items(): + if not key.startswith("_"): + if isinstance(value, MutableMapping): + modified_dict[key] = delete_keys_from_dict(value) + else: + modified_dict[key] = deepcopy(value) + return modified_dict + + +def split_mapper_into_qs(mapper): + values_list = [k for k, v in mapper.items() if k == v] + annotate_dict = OrderedDict([(v, F(k)) for k, v in mapper.items() if k != v]) + + return values_list, annotate_dict diff --git a/usaspending_api/awards/v2/views/awards.py b/usaspending_api/awards/v2/views/awards.py index b6019a74a1..dd54962c92 100644 --- a/usaspending_api/awards/v2/views/awards.py +++ b/usaspending_api/awards/v2/views/awards.py @@ -4,13 +4,13 @@ from django.db.models import Max from rest_framework import status +from rest_framework.exceptions import NotFound from rest_framework.request import Request from rest_framework.response import Response - from usaspending_api.awards.models import Award, ParentAward -from usaspending_api.awards.v2.data_layer.orm import construct_idv_response -from usaspending_api.awards.serializers_v2.serializers import AwardContractSerializerV2, AwardMiscSerializerV2 +from usaspending_api.awards.v2.data_layer.orm import (construct_contract_response, construct_idv_response, + construct_assistance_response) from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.exceptions import UnprocessableEntityException from usaspending_api.common.views import APIDocumentationView @@ -65,19 +65,15 @@ def _business_logic(self, request_dict: dict) -> dict: award = Award.objects.get(**{dict_key: request_dict[dict_key]}) except Award.DoesNotExist: logger.info("No Award found with '{}' in '{}'".format(request_dict[dict_key], dict_key)) - return {"message": "No award found with this id"} # Consider returning 404 or 410 error code + raise NotFound("No Award found with this id: '{}'".format(request_dict[dict_key])) try: if award.category == 'contract': - parent_recipient_name = award.latest_transaction.contract_data.ultimate_parent_legal_enti - serialized = AwardContractSerializerV2(award).data - serialized['recipient']['parent_recipient_name'] = parent_recipient_name + serialized = construct_contract_response(request_dict[dict_key]) elif award.category == "idv": serialized = construct_idv_response(request_dict[dict_key]) else: - parent_recipient_name = award.latest_transaction.assistance_data.ultimate_parent_legal_enti - serialized = AwardMiscSerializerV2(award).data - serialized['recipient']['parent_recipient_name'] = parent_recipient_name + serialized = construct_assistance_response(request_dict[dict_key]) except AttributeError: raise UnprocessableEntityException("Unable to complete response due to missing Award data") @@ -135,10 +131,7 @@ def _business_logic(request_data: dict) -> dict: } except ParentAward.DoesNotExist: logger.info("No IDV Award found where '%s' is '%s'" % next(iter(request_data.items()))) - return { - 'data': OrderedDict({'message': 'No IDV award found with this id'}), - 'status': status.HTTP_404_NOT_FOUND - } + raise NotFound("No IDV award found with this id") @cache_response() def get(self, request: Request, requested_award: str) -> Response: From 17ef5d0786a22e55a18ca0a6a949a053a76b78a0 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 03:10:30 -0500 Subject: [PATCH 05/52] Added location to recipient data --- .../awards/tests/test_awards_v2.py | 4 +- usaspending_api/awards/v2/data_layer/orm.py | 147 ++++++++---------- .../awards/v2/data_layer/orm_mappers.py | 30 ++-- 3 files changed, 84 insertions(+), 97 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 37271167c1..ab5d925ed1 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -230,11 +230,11 @@ def test_idv_award_amount_endpoint(client): resp = client.get('/api/v2/awards/idvs/amounts/3/') assert resp.status_code == status.HTTP_404_NOT_FOUND - assert json.loads(resp.content.decode('utf-8')) == {'message': 'No IDV award found with this id'} + assert json.loads(resp.content.decode('utf-8')) == {'detail': 'No IDV award found with this id'} resp = client.get('/api/v2/awards/idvs/amounts/BOGUS_ID/') assert resp.status_code == status.HTTP_404_NOT_FOUND - assert json.loads(resp.content.decode('utf-8')) == {'message': 'No IDV award found with this id'} + assert json.loads(resp.content.decode('utf-8')) == {'detail': 'No IDV award found with this id'} resp = client.get('/api/v2/awards/idvs/amounts/INVALID_ID_&&&/') assert resp.status_code == status.HTTP_404_NOT_FOUND diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 38794e0750..619b75ace1 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -49,41 +49,8 @@ def construct_assistance_response(requested_award): ("period_of_performance_current_end_date", award["_end_date"]), ] ) - response["recipient"] = OrderedDict( - [ - ( - "recipient_hash", - fetch_recipient_hash_using_name_and_duns( - transaction["_recipient_name"], transaction["_recipient_unique_id"] - ), - ), - ("recipient_name", transaction["_recipient_name"]), - ("recipient_unique_id", transaction["_recipient_unique_id"]), - ("parent_recipient_unique_id", transaction["_parent_recipient_unique_id"]), - ("parent_recipient_name", transaction["_parent_recipient_name"]), - ("business_categories", fetch_business_categories_by_legal_entity_id(award["_lei"])), - ( - "location", - OrderedDict( - [ - ("legal_entity_country_code", transaction["_rl_location_country_code"]), - ("legal_entity_country_name", transaction["_rl_country_name"]), - ("legal_entity_state_code", transaction["_rl_state_code"]), - ("legal_entity_city_name", transaction["_rl_city_name"]), - ("legal_entity_county_name", transaction["_rl_county_name"]), - ("legal_entity_address_line1", transaction["_rl_address_line1"]), - ("legal_entity_address_line2", transaction["_rl_address_line2"]), - ("legal_entity_address_line3", transaction["_rl_address_line3"]), - ("legal_entity_congressional", transaction["_rl_congressional_code"]), - ("legal_entity_zip_last4", transaction["_rl_zip4"]), - ("legal_entity_zip5", transaction["_rl_zip5"]), - ("legal_entity_foreign_posta", transaction["_rl_foreign_postal_code"]), - ("legal_entity_foreign_provi", transaction["_rl_foreign_province"]), - ] - ), - ), - ] - ) + transaction["_lei"] = award["_lei"] + response["recipient"] = create_recipient_object(transaction) response["place_of_performance"] = OrderedDict( [ ("location_country_code", transaction["_pop_location_country_code"]), @@ -137,32 +104,18 @@ def construct_contract_response(requested_award): ("period_of_performance_current_end_date", award["_end_date"]), ] ) - response["recipient"] = OrderedDict( - [ - ( - "recipient_hash", - fetch_recipient_hash_using_name_and_duns( - response["latest_transaction_contract_data"]["_recipient_name"], - response["latest_transaction_contract_data"]["_recipient_unique_id"], - ), - ), - ("recipient_name", response["latest_transaction_contract_data"]["_recipient_name"]), - ("recipient_unique_id", response["latest_transaction_contract_data"]["_recipient_unique_id"]), - ("parent_recipient_name", response["latest_transaction_contract_data"]["_parent_recipient_name"]), - ("parent_recipient_unique_id", response["latest_transaction_contract_data"]["_parent_recipient_unique_id"]), - ("business_categories", fetch_business_categories_by_legal_entity_id(award["_lei"])), - ] - ) + response["latest_transaction_contract_data"]["_lei"] = award["_lei"] + response["recipient"] = create_recipient_object(response["latest_transaction_contract_data"]) response["place_of_performance"] = OrderedDict( [ - ("location_country_code", response["latest_transaction_contract_data"]["_country_code"]), - ("country_name", response["latest_transaction_contract_data"]["_country_name"]), - ("county_name", response["latest_transaction_contract_data"]["_county_name"]), - ("city_name", response["latest_transaction_contract_data"]["_city_name"]), - ("state_code", response["latest_transaction_contract_data"]["_state_code"]), - ("congressional_code", response["latest_transaction_contract_data"]["_congressional_code"]), - ("zip4", response["latest_transaction_contract_data"]["_zip4"]), - ("zip5", response["latest_transaction_contract_data"]["_zip5"]), + ("location_country_code", response["latest_transaction_contract_data"]["_pop_location_country_code"]), + ("country_name", response["latest_transaction_contract_data"]["_pop_country_name"]), + ("county_name", response["latest_transaction_contract_data"]["_pop_county_name"]), + ("city_name", response["latest_transaction_contract_data"]["_pop_city_name"]), + ("state_code", response["latest_transaction_contract_data"]["_pop_state_code"]), + ("congressional_code", response["latest_transaction_contract_data"]["_pop_congressional_code"]), + ("zip4", response["latest_transaction_contract_data"]["_pop_zip4"]), + ("zip5", response["latest_transaction_contract_data"]["_pop_zip5"]), ("address_line1", None), ("address_line2", None), ("address_line3", None), @@ -219,32 +172,18 @@ def construct_idv_response(requested_award): ("end_date", response["latest_transaction_contract_data"]["_end_date"]), ] ) - response["recipient"] = OrderedDict( - [ - ( - "recipient_hash", - fetch_recipient_hash_using_name_and_duns( - response["latest_transaction_contract_data"]["_recipient_name"], - response["latest_transaction_contract_data"]["_recipient_unique_id"], - ), - ), - ("recipient_name", response["latest_transaction_contract_data"]["_recipient_name"]), - ("recipient_unique_id", response["latest_transaction_contract_data"]["_recipient_unique_id"]), - ("parent_recipient_name", response["latest_transaction_contract_data"]["_parent_recipient_name"]), - ("parent_recipient_unique_id", response["latest_transaction_contract_data"]["_parent_recipient_unique_id"]), - ("business_categories", fetch_business_categories_by_legal_entity_id(award["_lei"])), - ] - ) + response["latest_transaction_contract_data"]["_lei"] = award["_lei"] + response["recipient"] = create_recipient_object(response["latest_transaction_contract_data"]) response["place_of_performance"] = OrderedDict( [ - ("location_country_code", response["latest_transaction_contract_data"]["_country_code"]), - ("country_name", response["latest_transaction_contract_data"]["_country_name"]), - ("county_name", response["latest_transaction_contract_data"]["_county_name"]), - ("city_name", response["latest_transaction_contract_data"]["_city_name"]), - ("state_code", response["latest_transaction_contract_data"]["_state_code"]), - ("congressional_code", response["latest_transaction_contract_data"]["_congressional_code"]), - ("zip4", response["latest_transaction_contract_data"]["_zip4"]), - ("zip5", response["latest_transaction_contract_data"]["_zip5"]), + ("location_country_code", response["latest_transaction_contract_data"]["_pop_location_country_code"]), + ("country_name", response["latest_transaction_contract_data"]["_pop_country_name"]), + ("county_name", response["latest_transaction_contract_data"]["_pop_county_name"]), + ("city_name", response["latest_transaction_contract_data"]["_pop_city_name"]), + ("state_code", response["latest_transaction_contract_data"]["_pop_state_code"]), + ("congressional_code", response["latest_transaction_contract_data"]["_pop_congressional_code"]), + ("zip4", response["latest_transaction_contract_data"]["_pop_zip4"]), + ("zip5", response["latest_transaction_contract_data"]["_pop_zip5"]), ("address_line1", None), ("address_line2", None), ("address_line3", None), @@ -256,6 +195,44 @@ def construct_idv_response(requested_award): return delete_keys_from_dict(response) +def create_recipient_object(db_row_dict): + return OrderedDict( + [ + ( + "recipient_hash", + fetch_recipient_hash_using_name_and_duns( + db_row_dict["_recipient_name"], db_row_dict["_recipient_unique_id"] + ), + ), + ("recipient_name", db_row_dict["_recipient_name"]), + ("recipient_unique_id", db_row_dict["_recipient_unique_id"]), + ("parent_recipient_unique_id", db_row_dict["_parent_recipient_unique_id"]), + ("parent_recipient_name", db_row_dict["_parent_recipient_name"]), + ("business_categories", fetch_business_categories_by_legal_entity_id(db_row_dict["_lei"])), + ( + "location", + OrderedDict( + [ + ("legal_entity_country_code", db_row_dict["_rl_location_country_code"]), + ("legal_entity_country_name", db_row_dict["_rl_country_name"]), + ("legal_entity_state_code", db_row_dict["_rl_state_code"]), + ("legal_entity_city_name", db_row_dict["_rl_city_name"]), + ("legal_entity_county_name", db_row_dict["_rl_county_name"]), + ("legal_entity_address_line1", db_row_dict["_rl_address_line1"]), + ("legal_entity_address_line2", db_row_dict["_rl_address_line2"]), + ("legal_entity_address_line3", db_row_dict["_rl_address_line3"]), + ("legal_entity_congressional", db_row_dict["_rl_congressional_code"]), + ("legal_entity_zip_last4", db_row_dict["_rl_zip4"]), + ("legal_entity_zip5", db_row_dict["_rl_zip5"]), + ("legal_entity_foreign_posta", db_row_dict.get("_rl_foreign_postal_code")), + ("legal_entity_foreign_provi", db_row_dict.get("_rl_foreign_province")), + ] + ), + ), + ] + ) + + def fetch_award_details(filter_q, mapper_fields): vals, ann = split_mapper_into_qs(mapper_fields) return Award.objects.filter(**filter_q).values(*vals).annotate(**ann).first() @@ -327,7 +304,9 @@ def fetch_officers_by_legal_entity_id(legal_entity_id): def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id): - recipient = RecipientLookup.objects.filter(duns=recipient_unique_id).values("recipient_hash").first() + recipient = None + if recipient_unique_id: + recipient = RecipientLookup.objects.filter(duns=recipient_unique_id).values("recipient_hash").first() if not recipient: # SQL: MD5(UPPER(CONCAT(awardee_or_recipient_uniqu, legal_business_name)))::uuid diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index f0322e1543..4242e49add 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -24,7 +24,6 @@ ("total_loan_value", "total_loan_value"), ("total_obligation", "total_obligation"), ("base_and_all_options_value", "base_and_all_options_value"), - # ("funding_obligated", "funding_obligated"), ("base_exercised_options_val", "base_exercised_options"), ("non_federal_funding_amount", "non_federal_funding"), ("total_funding_amount", "total_funding"), @@ -73,7 +72,6 @@ ("awardee_or_recipient_uniqu", "_recipient_unique_id"), ("ultimate_parent_legal_enti", "_parent_recipient_name"), ("ultimate_parent_unique_ide", "_parent_recipient_unique_id"), - ("legal_entity_country_code", "_rl_location_country_code"), ("legal_entity_country_name", "_rl_country_name"), ("legal_entity_state_code", "_rl_state_code"), @@ -87,7 +85,6 @@ ("legal_entity_zip5", "_rl_zip5"), ("legal_entity_foreign_posta", "_rl_foreign_postal_code"), ("legal_entity_foreign_provi", "_rl_foreign_province"), - # "Place of Performance" fields below ("place_of_perform_country_c", "_pop_location_country_code"), ("place_of_perform_country_n", "_pop_country_name"), @@ -148,15 +145,26 @@ ("awardee_or_recipient_uniqu", "_recipient_unique_id"), ("ultimate_parent_legal_enti", "_parent_recipient_name"), ("ultimate_parent_unique_ide", "_parent_recipient_unique_id"), + ("legal_entity_country_code", "_rl_location_country_code"), + ("legal_entity_country_name", "_rl_country_name"), + ("legal_entity_state_code", "_rl_state_code"), + ("legal_entity_city_name", "_rl_city_name"), + ("legal_entity_county_name", "_rl_county_name"), + ("legal_entity_address_line1", "_rl_address_line1"), + ("legal_entity_address_line2", "_rl_address_line2"), + ("legal_entity_address_line3", "_rl_address_line3"), + ("legal_entity_congressional", "_rl_congressional_code"), + ("legal_entity_zip_last4", "_rl_zip4"), + ("legal_entity_zip5", "_rl_zip5"), # "Place of Performance Location" - ("place_of_perform_country_c", "_country_code"), - ("place_of_perf_country_desc", "_country_name"), - ("place_of_performance_state", "_state_code"), - ("place_of_perform_city_name", "_city_name"), - ("place_of_perform_county_na", "_county_name"), - ("place_of_performance_zip4a", "_zip4"), - ("place_of_performance_congr", "_congressional_code"), - ("place_of_performance_zip5", "_zip5"), + ("place_of_perform_country_c", "_pop_location_country_code"), + ("place_of_perf_country_desc", "_pop_country_name"), + ("place_of_performance_state", "_pop_state_code"), + ("place_of_perform_city_name", "_pop_city_name"), + ("place_of_perform_county_na", "_pop_county_name"), + ("place_of_performance_zip4a", "_pop_zip4"), + ("place_of_performance_congr", "_pop_congressional_code"), + ("place_of_performance_zip5", "_pop_zip5"), ] ) From 873321307d9f4e99f4a79344208c87f70364eb80 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 03:17:06 -0500 Subject: [PATCH 06/52] reusing identical logic --- usaspending_api/awards/v2/data_layer/orm.py | 74 +++++++-------------- 1 file changed, 23 insertions(+), 51 deletions(-) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 619b75ace1..f800b2e891 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -51,23 +51,7 @@ def construct_assistance_response(requested_award): ) transaction["_lei"] = award["_lei"] response["recipient"] = create_recipient_object(transaction) - response["place_of_performance"] = OrderedDict( - [ - ("location_country_code", transaction["_pop_location_country_code"]), - ("country_name", transaction["_pop_country_name"]), - ("county_name", transaction["_pop_county_name"]), - ("city_name", transaction["_pop_city_name"]), - ("state_code", transaction["_pop_state_code"]), - ("congressional_code", transaction["_pop_congressional_code"]), - ("zip4", transaction["_pop_zip4"]), - ("zip5", transaction["_pop_zip5"]), - ("address_line1", None), - ("address_line2", None), - ("address_line3", None), - ("foreign_province", transaction["_pop_foreign_province"]), - ("foreign_postal_code", None), - ] - ) + response["place_of_performance"] = create_place_of_performance_object(transaction) return delete_keys_from_dict(response) @@ -106,23 +90,7 @@ def construct_contract_response(requested_award): ) response["latest_transaction_contract_data"]["_lei"] = award["_lei"] response["recipient"] = create_recipient_object(response["latest_transaction_contract_data"]) - response["place_of_performance"] = OrderedDict( - [ - ("location_country_code", response["latest_transaction_contract_data"]["_pop_location_country_code"]), - ("country_name", response["latest_transaction_contract_data"]["_pop_country_name"]), - ("county_name", response["latest_transaction_contract_data"]["_pop_county_name"]), - ("city_name", response["latest_transaction_contract_data"]["_pop_city_name"]), - ("state_code", response["latest_transaction_contract_data"]["_pop_state_code"]), - ("congressional_code", response["latest_transaction_contract_data"]["_pop_congressional_code"]), - ("zip4", response["latest_transaction_contract_data"]["_pop_zip4"]), - ("zip5", response["latest_transaction_contract_data"]["_pop_zip5"]), - ("address_line1", None), - ("address_line2", None), - ("address_line3", None), - ("foreign_province", None), - ("foreign_postal_code", None), - ] - ) + response["place_of_performance"] = create_place_of_performance_object(response["latest_transaction_contract_data"]) return delete_keys_from_dict(response) @@ -174,23 +142,7 @@ def construct_idv_response(requested_award): ) response["latest_transaction_contract_data"]["_lei"] = award["_lei"] response["recipient"] = create_recipient_object(response["latest_transaction_contract_data"]) - response["place_of_performance"] = OrderedDict( - [ - ("location_country_code", response["latest_transaction_contract_data"]["_pop_location_country_code"]), - ("country_name", response["latest_transaction_contract_data"]["_pop_country_name"]), - ("county_name", response["latest_transaction_contract_data"]["_pop_county_name"]), - ("city_name", response["latest_transaction_contract_data"]["_pop_city_name"]), - ("state_code", response["latest_transaction_contract_data"]["_pop_state_code"]), - ("congressional_code", response["latest_transaction_contract_data"]["_pop_congressional_code"]), - ("zip4", response["latest_transaction_contract_data"]["_pop_zip4"]), - ("zip5", response["latest_transaction_contract_data"]["_pop_zip5"]), - ("address_line1", None), - ("address_line2", None), - ("address_line3", None), - ("foreign_province", None), - ("foreign_postal_code", None), - ] - ) + response["place_of_performance"] = create_place_of_performance_object(response["latest_transaction_contract_data"]) return delete_keys_from_dict(response) @@ -233,6 +185,26 @@ def create_recipient_object(db_row_dict): ) +def create_place_of_performance_object(db_row_dict): + return OrderedDict( + [ + ("location_country_code", db_row_dict["_pop_location_country_code"]), + ("country_name", db_row_dict["_pop_country_name"]), + ("county_name", db_row_dict["_pop_county_name"]), + ("city_name", db_row_dict["_pop_city_name"]), + ("state_code", db_row_dict["_pop_state_code"]), + ("congressional_code", db_row_dict["_pop_congressional_code"]), + ("zip4", db_row_dict["_pop_zip4"]), + ("zip5", db_row_dict["_pop_zip5"]), + ("address_line1", None), + ("address_line2", None), + ("address_line3", None), + ("foreign_province", db_row_dict.get("_pop_foreign_province")), + ("foreign_postal_code", None), + ] + ) + + def fetch_award_details(filter_q, mapper_fields): vals, ann = split_mapper_into_qs(mapper_fields) return Award.objects.filter(**filter_q).values(*vals).annotate(**ann).first() From 2702591dc4685273c4f42bec07730cd37e5e6b56 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 17:12:00 -0500 Subject: [PATCH 07/52] cleaned some code --- usaspending_api/awards/v2/data_layer/orm.py | 55 ++++++------------ usaspending_api/awards/v2/views/awards.py | 64 +++++++++------------ 2 files changed, 45 insertions(+), 74 deletions(-) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index f800b2e891..321d87c1a3 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -14,22 +14,17 @@ from usaspending_api.awards.v2.data_layer.orm_utils import delete_keys_from_dict, split_mapper_into_qs -def construct_assistance_response(requested_award): +def construct_assistance_response(requested_award_dict): """ - [x] base award data - [x] awarding_agency - [x] funding_agency - [x] recipient - [x] place_of_performance - [x] period_of_performance + Build the Python object to return FABS Award summary or meta-data via the API + + parameter(s): `requested_award` either award.id (int) or generated_unique_award_id (str) + returns: an OrderedDict """ - f = {"generated_unique_award_id": requested_award} - if str(requested_award).isdigit(): - f = {"pk": requested_award} response = OrderedDict() - award = fetch_award_details(f, FABS_AWARD_FIELDS) + award = fetch_award_details(requested_award_dict, FABS_AWARD_FIELDS) if not award: return None response.update(award) @@ -56,24 +51,17 @@ def construct_assistance_response(requested_award): return delete_keys_from_dict(response) -def construct_contract_response(requested_award): +def construct_contract_response(requested_award_dict): """ - [x] base award data - [x] awarding_agency - [x] funding_agency - [x] recipient - [x] place_of_performance - [x] executive_details - [x] period_of_performance - [x] latest_transaction_contract_data + Build the Python object to return FPDS Award summary or meta-data via the API + + parameter(s): `requested_award` either award.id (int) or generated_unique_award_id (str) + returns: an OrderedDict """ - f = {"generated_unique_award_id": requested_award} - if str(requested_award).isdigit(): - f = {"pk": requested_award} response = OrderedDict() - award = fetch_award_details(f, FPDS_AWARD_FIELDS) + award = fetch_award_details(requested_award_dict, FPDS_AWARD_FIELDS) if not award: return None response.update(award) @@ -95,20 +83,13 @@ def construct_contract_response(requested_award): return delete_keys_from_dict(response) -def construct_idv_response(requested_award): +def construct_idv_response(requested_award_dict): """ - [x] base award data - [x] awarding_agency - [x] funding_agency - [x] recipient - [x] place_of_performance - [x] executive_details - [x] idv_dates - [x] latest_transaction_contract_data + Build the Python object to return FPDS IDV summary or meta-data via the API + + parameter(s): `requested_award` either award.id (int) or generated_unique_award_id (str) + returns: an OrderedDict """ - f = {"generated_unique_award_id": requested_award} - if str(requested_award).isdigit(): - f = {"pk": requested_award} idv_specific_award_fields = OrderedDict( [ @@ -123,7 +104,7 @@ def construct_idv_response(requested_award): response = OrderedDict() - award = fetch_award_details(f, FPDS_AWARD_FIELDS) + award = fetch_award_details(requested_award_dict, FPDS_AWARD_FIELDS) if not award: return None response.update(award) diff --git a/usaspending_api/awards/v2/views/awards.py b/usaspending_api/awards/v2/views/awards.py index dd54962c92..bafc712c22 100644 --- a/usaspending_api/awards/v2/views/awards.py +++ b/usaspending_api/awards/v2/views/awards.py @@ -9,10 +9,12 @@ from rest_framework.response import Response from usaspending_api.awards.models import Award, ParentAward -from usaspending_api.awards.v2.data_layer.orm import (construct_contract_response, construct_idv_response, - construct_assistance_response) +from usaspending_api.awards.v2.data_layer.orm import ( + construct_contract_response, + construct_idv_response, + construct_assistance_response, +) from usaspending_api.common.cache_decorator import cache_response -from usaspending_api.common.exceptions import UnprocessableEntityException from usaspending_api.common.views import APIDocumentationView from usaspending_api.core.validator.tinyshield import TinyShield @@ -40,45 +42,38 @@ class AwardRetrieveViewSet(APIDocumentationView): """ def _parse_and_validate_request(self, provided_award_id: str) -> dict: - try: + request_dict = {"generated_unique_award_id": provided_award_id} + models = [ + { + "key": "generated_unique_award_id", + "name": "generated_unique_award_id", + "type": "text", + "text_type": "search", + "optional": False, + } + ] + if str(provided_award_id).isdigit(): request_dict = {"id": int(provided_award_id)} models = [{"key": "id", "name": "id", "type": "integer", "optional": False}] - except Exception as e: - models = [ - { - "key": "generated_unique_award_id", - "name": "generated_unique_award_id", - "type": "text", - "text_type": "search", - "optional": False, - } - ] - request_dict = {"generated_unique_award_id": provided_award_id} validated_request_data = TinyShield(models).block(request_dict) return validated_request_data def _business_logic(self, request_dict: dict) -> dict: - dict_key = "id" if "id" in request_dict else "generated_unique_award_id" - try: - award = Award.objects.get(**{dict_key: request_dict[dict_key]}) + award = Award.objects.get(**request_dict) except Award.DoesNotExist: - logger.info("No Award found with '{}' in '{}'".format(request_dict[dict_key], dict_key)) - raise NotFound("No Award found with this id: '{}'".format(request_dict[dict_key])) - - try: - if award.category == 'contract': - serialized = construct_contract_response(request_dict[dict_key]) - elif award.category == "idv": - serialized = construct_idv_response(request_dict[dict_key]) - else: - serialized = construct_assistance_response(request_dict[dict_key]) + logger.info("No Award found with: '{}'".format(request_dict)) + raise NotFound("No Award found with: '{}'".format(request_dict)) - except AttributeError: - raise UnprocessableEntityException("Unable to complete response due to missing Award data") + if award.category == "contract": + response_content = construct_contract_response(request_dict) + elif award.category == "idv": + response_content = construct_idv_response(request_dict) + else: + response_content = construct_assistance_response(request_dict) - return serialized + return response_content @cache_response() def get(self, request: Request, requested_award: str) -> Response: @@ -96,12 +91,7 @@ class IDVAmountsViewSet(APIDocumentationView): def _parse_and_validate_request(requested_award: str) -> dict: try: request_dict = {'award_id': int(requested_award)} - models = [{ - 'key': 'award_id', - 'name': 'award_id', - 'type': 'integer', - 'optional': False, - }] + models = [{'key': 'award_id', 'name': 'award_id', 'type': 'integer', 'optional': False}] except ValueError: request_dict = {'generated_unique_award_id': requested_award.upper()} models = [{ From 4f1c6113679dafa1c3e50f05795f441c7458f640 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 22:14:20 -0500 Subject: [PATCH 08/52] fixed a number of tests --- .../awards/tests/test_awards_v2.py | 243 ++++++++++++------ usaspending_api/awards/v2/data_layer/orm.py | 42 +-- .../awards/v2/data_layer/orm_mappers.py | 1 + 3 files changed, 193 insertions(+), 93 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index ab5d925ed1..d7480c6814 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -41,11 +41,8 @@ def awards_and_transactions(db): le = { "pk": 1, - "recipient_name": "John's Pizza", - "recipient_unique_id": 456, - "parent_recipient_unique_id": 123, - "business_categories": ['small_business'], - "location": Location.objects.get(pk=1), + "business_categories": ["small_business"], + # "location": Location.objects.get(pk=1), } ag = { @@ -64,11 +61,96 @@ def awards_and_transactions(db): "pk": 1, "transaction": TransactionNormalized.objects.get(pk=1), "cfda_number": 1234, - "cfda_title": "Shazam", + "cfda_title": "Shiloh", + "awardee_or_recipient_legal": "John's Pizza", + "awardee_or_recipient_uniqu": "456", + "ultimate_parent_unique_ide": "123", + "legal_entity_country_code": "USA", + "legal_entity_country_name": "UNITED STATES", + "legal_entity_state_code": "NC", + "legal_entity_city_name": "Charlotte", + "legal_entity_county_name": "BUNCOMBE", + "legal_entity_address_line1": "123 main st", + "legal_entity_address_line2": None, + "legal_entity_address_line3": None, + "legal_entity_congressional": "90", + "legal_entity_zip_last4": "5312", + "legal_entity_zip5": "12204", + "place_of_perform_country_c": "PDA", + "place_of_perform_country_n": "Pacific Delta Amazon", + "place_of_perform_county_na": "Tripoli", + "place_of_performance_city": "Austin", + "place_of_perfor_state_code": "TX", + "place_of_performance_congr": "-0-", + "place_of_perform_zip_last4": "2135", + "place_of_performance_zip5": "40221", + "place_of_performance_forei": None, } cont_data = { "pk": 2, "transaction": TransactionNormalized.objects.get(pk=2), + "idv_type_description": "", + "type_of_idc_description": "", + "referenced_idv_agency_iden": "", + "multiple_or_single_aw_desc": "", + "solicitation_identifier": "", + "solicitation_procedures": "", + "number_of_offers_received": "", + "extent_competed": "", + "other_than_full_and_o_desc": "", + "type_set_aside_description": "", + "commercial_item_acquisitio": "", + "commercial_item_test_desc": "", + "evaluated_preference_desc": "", + "fed_biz_opps_description": "", + "small_business_competitive": "", + "fair_opportunity_limi_desc": "", + "product_or_service_code": "", + "product_or_service_co_desc": None, + "naics": "", + "dod_claimant_program_code": "", + "program_system_or_equipmen": "", + "information_technolog_desc": "", + "sea_transportation_desc": "", + "clinger_cohen_act_pla_desc": "", + "construction_wage_rat_desc": "", + "labor_standards_descrip": "", + "materials_supplies_descrip": "", + "cost_or_pricing_data_desc": "", + "domestic_or_foreign_e_desc": "", + "foreign_funding_desc": "", + "interagency_contract_desc": "", + "major_program": "", + "price_evaluation_adjustmen": "", + "program_acronym": "", + "subcontracting_plan": "", + "multi_year_contract_desc": "", + "purchase_card_as_paym_desc": "", + "consolidated_contract_desc": "", + "type_of_contract_pric_desc": "", + "awardee_or_recipient_legal": "John's Pizza", + "awardee_or_recipient_uniqu": "456", + "ultimate_parent_legal_enti": None, + "ultimate_parent_unique_ide": "123", + "legal_entity_country_code": "USA", + "legal_entity_country_name": "UNITED STATES", + "legal_entity_state_code": "NC", + "legal_entity_city_name": "Charlotte", + "legal_entity_county_name": "BUNCOMBE", + "legal_entity_address_line1": "123 main st", + "legal_entity_address_line2": None, + "legal_entity_address_line3": None, + "legal_entity_congressional": "90", + "legal_entity_zip_last4": "5312", + "legal_entity_zip5": "12204", + "place_of_perform_country_c": "USA", + "place_of_perf_country_desc": "UNITED STATES", + "place_of_performance_state": "NC", + "place_of_perform_city_name": "Charlotte", + "place_of_perform_county_na": "BUNCOMBE", + "place_of_performance_zip4a": "5312", + "place_of_performance_congr": "90", + "place_of_performance_zip5": "12204", "type_of_contract_pric_desc": "FIRM FIXED PRICE", "naics": "333911", "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", @@ -116,7 +198,7 @@ def awards_and_transactions(db): "type": "11", "type_description": "OTHER FINANCIAL ASSISTANCE", "category": "grant", - "piid": 1234, + "uri": 1234, "description": "lorem ipsum", "period_of_performance_start_date": "2004-02-04", "period_of_performance_current_end_date": "2005-02-04", @@ -151,8 +233,8 @@ def awards_and_transactions(db): "total_subaward_amount": 12345.00, "subaward_count": 10, } - mommy.make('awards.Award', **award_1_model) - mommy.make('awards.Award', **award_2_model) + mommy.make("awards.Award", **award_1_model) + mommy.make("awards.Award", **award_2_model) @pytest.mark.django_db @@ -160,32 +242,32 @@ def test_award_last_updated_endpoint(client): """Test the awards endpoint.""" test_date = datetime.datetime.now() - test_date_reformatted = test_date.strftime('%m/%d/%Y') + test_date_reformatted = test_date.strftime("%m/%d/%Y") - mommy.make('awards.Award', update_date=test_date) - mommy.make('awards.Award', update_date='') + mommy.make("awards.Award", update_date=test_date) + mommy.make("awards.Award", update_date="") - resp = client.get('/api/v2/awards/last_updated/') + resp = client.get("/api/v2/awards/last_updated/") assert resp.status_code == status.HTTP_200_OK - assert resp.data['last_updated'] == test_date_reformatted + assert resp.data["last_updated"] == test_date_reformatted @pytest.mark.django_db def test_award_endpoint_generated_id(client, awards_and_transactions): - resp = client.get('/api/v2/awards/ASST_AW_3620_-NONE-_1830212.0481163/') + resp = client.get("/api/v2/awards/ASST_AW_3620_-NONE-_1830212.0481163/") assert resp.status_code == status.HTTP_200_OK assert json.loads(resp.content.decode("utf-8")) == expected_response_asst - resp = client.get('/api/v2/awards/CONT_AW_9700_9700_03VD_SPM30012D3486/') + resp = client.get("/api/v2/awards/CONT_AW_9700_9700_03VD_SPM30012D3486/") assert resp.status_code == status.HTTP_200_OK assert json.loads(resp.content.decode("utf-8")) == expected_response_cont - resp = client.get('/api/v2/awards/1/') + resp = client.get("/api/v2/awards/1/") assert resp.status_code == status.HTTP_200_OK assert json.loads(resp.content.decode("utf-8")) == expected_response_asst - resp = client.get('/api/v2/awards/2/') + resp = client.get("/api/v2/awards/2/") assert resp.status_code == status.HTTP_200_OK assert json.loads(resp.content.decode("utf-8")) == expected_response_cont @@ -193,50 +275,50 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): @pytest.mark.django_db def test_idv_award_amount_endpoint(client): - mommy.make('awards.Award', pk=1) + mommy.make("awards.Award", pk=1) mommy.make( - 'awards.ParentAward', + "awards.ParentAward", award_id=1, - generated_unique_award_id='CONT_AW_2', + generated_unique_award_id="CONT_AW_2", direct_idv_count=3, direct_contract_count=4, - direct_total_obligation='5.01', - direct_base_and_all_options_value='6.02', - direct_base_exercised_options_val='7.03', + direct_total_obligation="5.01", + direct_base_and_all_options_value="6.02", + direct_base_exercised_options_val="7.03", rollup_idv_count=8, rollup_contract_count=9, - rollup_total_obligation='10.04', - rollup_base_and_all_options_value='11.05', - rollup_base_exercised_options_val='12.06', + rollup_total_obligation="10.04", + rollup_base_and_all_options_value="11.05", + rollup_base_exercised_options_val="12.06", ) output_idv_amounts = { - 'award_id': 1, - 'generated_unique_award_id': 'CONT_AW_2', - 'idv_count': 3, - 'contract_count': 4, - 'rollup_total_obligation': 10.04, - 'rollup_base_and_all_options_value': 11.05, - 'rollup_base_exercised_options_val': 12.06, + "award_id": 1, + "generated_unique_award_id": "CONT_AW_2", + "idv_count": 3, + "contract_count": 4, + "rollup_total_obligation": 10.04, + "rollup_base_and_all_options_value": 11.05, + "rollup_base_exercised_options_val": 12.06, } - resp = client.get('/api/v2/awards/idvs/amounts/1/') + resp = client.get("/api/v2/awards/idvs/amounts/1/") assert resp.status_code == status.HTTP_200_OK - assert json.loads(resp.content.decode('utf-8')) == output_idv_amounts + assert json.loads(resp.content.decode("utf-8")) == output_idv_amounts - resp = client.get('/api/v2/awards/idvs/amounts/CONT_AW_2/') + resp = client.get("/api/v2/awards/idvs/amounts/CONT_AW_2/") assert resp.status_code == status.HTTP_200_OK - assert json.loads(resp.content.decode('utf-8')) == output_idv_amounts + assert json.loads(resp.content.decode("utf-8")) == output_idv_amounts - resp = client.get('/api/v2/awards/idvs/amounts/3/') + resp = client.get("/api/v2/awards/idvs/amounts/3/") assert resp.status_code == status.HTTP_404_NOT_FOUND - assert json.loads(resp.content.decode('utf-8')) == {'detail': 'No IDV award found with this id'} + assert json.loads(resp.content.decode("utf-8")) == {"detail": "No IDV award found with this id"} - resp = client.get('/api/v2/awards/idvs/amounts/BOGUS_ID/') + resp = client.get("/api/v2/awards/idvs/amounts/BOGUS_ID/") assert resp.status_code == status.HTTP_404_NOT_FOUND - assert json.loads(resp.content.decode('utf-8')) == {'detail': 'No IDV award found with this id'} + assert json.loads(resp.content.decode("utf-8")) == {"detail": "No IDV award found with this id"} - resp = client.get('/api/v2/awards/idvs/amounts/INVALID_ID_&&&/') + resp = client.get("/api/v2/awards/idvs/amounts/INVALID_ID_&&&/") assert resp.status_code == status.HTTP_404_NOT_FOUND @@ -245,26 +327,34 @@ def test_idv_award_amount_endpoint(client): "type": "11", "category": "grant", "type_description": "OTHER FINANCIAL ASSISTANCE", - "piid": "1234", + "uri": "1234", + "fain": None, + "generated_unique_award_id": "ASST_AW_3620_-NONE-_1830212.0481163", "description": "lorem ipsum", "cfda_objectives": None, "cfda_number": "1234", - "cfda_title": "Shazam", + "cfda_title": "Shiloh", + "base_and_all_options_value": None, + "base_exercised_options": None, + "non_federal_funding": None, + "total_funding": None, + "total_loan_value": None, + "total_obligation": None, + "total_subsidy_cost": None, "awarding_agency": { "id": 1, - "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, - "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, + "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, + "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, "office_agency_name": "office_agency", - "office_agency": {"aac_code": None, "name": "office_agency"}, }, "funding_agency": { "id": 1, - "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, - "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, + "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, + "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, "office_agency_name": "office_agency", - "office_agency": {"aac_code": None, "name": "office_agency"}, }, "recipient": { + "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367", "recipient_name": "John's Pizza", "recipient_unique_id": "456", "parent_recipient_unique_id": "123", @@ -278,7 +368,7 @@ def test_idv_award_amount_endpoint(client): "county_name": "BUNCOMBE", "state_code": "NC", "zip5": "12204", - "zip4": "122045312", + "zip4": "5312", "foreign_postal_code": None, "country_name": "UNITED STATES", "location_country_code": "USA", @@ -287,31 +377,31 @@ def test_idv_award_amount_endpoint(client): "parent_recipient_name": None, }, "subaward_count": 10, - "total_subaward_amount": "12345.00", + "total_subaward_amount": 12345.0, "period_of_performance": { - "period_of_performance_start_date": "2004-02-04", "period_of_performance_current_end_date": "2005-02-04", + "period_of_performance_start_date": "2004-02-04", }, "place_of_performance": { - "address_line1": "123 main st", + "address_line1": None, "address_line2": None, "address_line3": None, "foreign_province": None, - "city_name": "Charlotte", - "county_name": "BUNCOMBE", - "state_code": "NC", - "zip5": "12204", - "zip4": "122045312", + "city_name": "Austin", + "county_name": "Tripoli", + "state_code": "TX", + "zip5": "40221", + "zip4": "2135", "foreign_postal_code": None, - "country_name": "UNITED STATES", - "location_country_code": "USA", - "congressional_code": "90", + "country_name": "Pacific Delta Amazon", + "location_country_code": "PDA", + "congressional_code": "-0-", }, - "executive_details": {"officers": []}, } expected_response_cont = { "id": 2, + "generated_unique_award_id": "CONT_AW_9700_9700_03VD_SPM30012D3486", "type": "A", "category": "contract", "type_description": "DEFINITIVE CONTRACT", @@ -320,19 +410,18 @@ def test_idv_award_amount_endpoint(client): "description": "lorem ipsum", "awarding_agency": { "id": 1, - "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, - "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, + "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, + "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, "office_agency_name": "office_agency", - "office_agency": {"aac_code": None, "name": "office_agency"}, }, "funding_agency": { "id": 1, - "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, - "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, + "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, + "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, "office_agency_name": "office_agency", - "office_agency": {"aac_code": None, "name": "office_agency"}, }, "recipient": { + "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367", "recipient_name": "John's Pizza", "recipient_unique_id": "456", "parent_recipient_unique_id": "123", @@ -346,7 +435,7 @@ def test_idv_award_amount_endpoint(client): "county_name": "BUNCOMBE", "state_code": "NC", "zip5": "12204", - "zip4": "122045312", + "zip4": "5312", "foreign_postal_code": None, "country_name": "UNITED STATES", "location_country_code": "USA", @@ -354,14 +443,15 @@ def test_idv_award_amount_endpoint(client): }, "parent_recipient_name": None, }, - "total_obligation": "1000.00", - "base_and_all_options_value": "2000.00", + "total_obligation": 1000.0, + "base_and_all_options_value": 2000.0, + "base_exercised_options_val": None, "period_of_performance": { "period_of_performance_start_date": "2004-02-04", "period_of_performance_current_end_date": "2005-02-04", }, "place_of_performance": { - "address_line1": "123 main st", + "address_line1": None, "address_line2": None, "address_line3": None, "foreign_province": None, @@ -369,7 +459,7 @@ def test_idv_award_amount_endpoint(client): "county_name": "BUNCOMBE", "state_code": "NC", "zip5": "12204", - "zip4": "122045312", + "zip4": "5312", "foreign_postal_code": None, "country_name": "UNITED STATES", "location_country_code": "USA", @@ -415,9 +505,8 @@ def test_idv_award_amount_endpoint(client): "purchase_card_as_paym_desc": "NO", "consolidated_contract_desc": "NOT CONSOLIDATED", "type_of_contract_pric_desc": "FIRM FIXED PRICE", - "ordering_period_end_date": None, }, "subaward_count": 10, - "total_subaward_amount": "12345.00", + "total_subaward_amount": 12345.0, "executive_details": {"officers": []}, } diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 321d87c1a3..6b12fc03ec 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -33,8 +33,8 @@ def construct_assistance_response(requested_award_dict): cfda_info = fetch_cfda_details_using_cfda_number(transaction["cfda_number"]) response["cfda_number"] = transaction["cfda_number"] + response["cfda_title"] = transaction["cfda_title"] response["cfda_objectives"] = cfda_info.get("objectives") - response["cfda_title"] = cfda_info.get("program_title") response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) @@ -146,19 +146,19 @@ def create_recipient_object(db_row_dict): "location", OrderedDict( [ - ("legal_entity_country_code", db_row_dict["_rl_location_country_code"]), - ("legal_entity_country_name", db_row_dict["_rl_country_name"]), - ("legal_entity_state_code", db_row_dict["_rl_state_code"]), - ("legal_entity_city_name", db_row_dict["_rl_city_name"]), - ("legal_entity_county_name", db_row_dict["_rl_county_name"]), - ("legal_entity_address_line1", db_row_dict["_rl_address_line1"]), - ("legal_entity_address_line2", db_row_dict["_rl_address_line2"]), - ("legal_entity_address_line3", db_row_dict["_rl_address_line3"]), - ("legal_entity_congressional", db_row_dict["_rl_congressional_code"]), - ("legal_entity_zip_last4", db_row_dict["_rl_zip4"]), - ("legal_entity_zip5", db_row_dict["_rl_zip5"]), - ("legal_entity_foreign_posta", db_row_dict.get("_rl_foreign_postal_code")), - ("legal_entity_foreign_provi", db_row_dict.get("_rl_foreign_province")), + ("location_country_code", db_row_dict["_rl_location_country_code"]), + ("country_name", db_row_dict["_rl_country_name"]), + ("state_code", db_row_dict["_rl_state_code"]), + ("city_name", db_row_dict["_rl_city_name"]), + ("county_name", db_row_dict["_rl_county_name"]), + ("address_line1", db_row_dict["_rl_address_line1"]), + ("address_line2", db_row_dict["_rl_address_line2"]), + ("address_line3", db_row_dict["_rl_address_line3"]), + ("congressional_code", db_row_dict["_rl_congressional_code"]), + ("zip4", db_row_dict["_rl_zip4"]), + ("zip5", db_row_dict["_rl_zip5"]), + ("foreign_postal_code", db_row_dict.get("_rl_foreign_postal_code")), + ("foreign_province", db_row_dict.get("_rl_foreign_province")), ] ), ), @@ -215,8 +215,10 @@ def fetch_agency_details(agency_id): values = [ "toptier_agency__fpds_code", "toptier_agency__name", + "toptier_agency__abbreviation", "subtier_agency__subtier_code", "subtier_agency__name", + "subtier_agency__abbreviation", "office_agency__name", ] agency = Agency.objects.filter(pk=agency_id).values(*values).first() @@ -225,8 +227,16 @@ def fetch_agency_details(agency_id): if agency: agency_details = { "id": agency_id, - "toptier_agency": {"name": agency["toptier_agency__name"], "code": agency["toptier_agency__fpds_code"]}, - "subtier_agency": {"name": agency["subtier_agency__name"], "code": agency["subtier_agency__subtier_code"]}, + "toptier_agency": { + "name": agency["toptier_agency__name"], + "code": agency["toptier_agency__fpds_code"], + "abbreviation": agency["toptier_agency__abbreviation"], + }, + "subtier_agency": { + "name": agency["subtier_agency__name"], + "code": agency["subtier_agency__subtier_code"], + "abbreviation": agency["subtier_agency__abbreviation"], + }, "office_agency_name": agency["office_agency__name"], } return agency_details diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index 4242e49add..23d8fcfca6 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -67,6 +67,7 @@ FABS_ASSISTANCE_FIELDS = OrderedDict( [ ("cfda_number", "cfda_number"), + ("cfda_title", "cfda_title"), # "Recipient" fields below ("awardee_or_recipient_legal", "_recipient_name"), ("awardee_or_recipient_uniqu", "_recipient_unique_id"), From d3a776b3c821a184c3abd9f6f50d728b8714dd35 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 22:50:03 -0500 Subject: [PATCH 09/52] fixed idv tests --- .../awards/tests/test_awards_idv_v2.py | 137 +++++++++++------- .../awards/tests/test_awards_v2.py | 6 +- 2 files changed, 87 insertions(+), 56 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index 745e6948d3..94b3077ea8 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -1,21 +1,14 @@ import datetime -import pytest import json +import pytest -from rest_framework import status from model_mommy import mommy +from rest_framework import status from usaspending_api.awards.models import TransactionNormalized, Award from usaspending_api.references.models import Agency, Location, ToptierAgency, SubtierAgency, OfficeAgency, LegalEntity -@pytest.fixture -def awards_data(db): - mommy.make("awards.Award", piid="zzz", fain="abc123", type="B", total_obligation=1000) - mommy.make("awards.Award", piid="###", fain="ABC789", type="B", total_obligation=1000) - mommy.make("awards.Award", fain="XYZ789", type="C", total_obligation=1000) - - @pytest.fixture def awards_and_transactions(db): loc = { @@ -75,53 +68,76 @@ def awards_and_transactions(db): } latest_transaction_contract_data = { - "pk": 2, - "transaction": TransactionNormalized.objects.get(pk=2), - "type_of_contract_pric_desc": "FIRM FIXED PRICE", - "naics": "333911", - "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", - "idv_type_description": "IDC", - "type_of_idc_description": "INDEFINITE DELIVERY / INDEFINITE QUANTITY", - "multiple_or_single_aw_desc": "MULTIPLE AWARD", - "dod_claimant_program_code": "C9E", + "agency_id": "192", + "awardee_or_recipient_legal": "John's Pizza", + "awardee_or_recipient_uniqu": "456", "clinger_cohen_act_pla_desc": "NO", "commercial_item_acquisitio": "A", "commercial_item_test_desc": "NO", "consolidated_contract_desc": "NOT CONSOLIDATED", - "cost_or_pricing_data_desc": "NO", "construction_wage_rat_desc": "NO", + "cost_or_pricing_data_desc": "NO", + "dod_claimant_program_code": "C9E", + "domestic_or_foreign_e_desc": "U.S. OWNED BUSINESS", "evaluated_preference_desc": "NO PREFERENCE USED", "extent_competed": "D", + "fair_opportunity_limi_desc": None, "fed_biz_opps_description": "YES", "foreign_funding_desc": "NOT APPLICABLE", + "idv_type_description": "IDC", "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", "interagency_contract_desc": "NOT APPLICABLE", + "labor_standards_descrip": "NO", + "last_modified": "2018-08-24", + "legal_entity_address_line1": "123 main st", + "legal_entity_address_line2": None, + "legal_entity_address_line3": None, + "legal_entity_city_name": "Charlotte", + "legal_entity_congressional": "90", + "legal_entity_country_code": "USA", + "legal_entity_country_name": "UNITED STATES", + "legal_entity_county_name": "BUNCOMBE", + "legal_entity_state_code": "NC", + "legal_entity_zip5": "12204", + "legal_entity_zip_last4": "5312", "major_program": None, - "purchase_card_as_paym_desc": "NO", + "materials_supplies_descrip": "NO", "multi_year_contract_desc": "NO", + "multiple_or_single_aw_desc": "MULTIPLE AWARD", + "naics": "333911", + "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", "number_of_offers_received": None, + "ordering_period_end_date": "2025-06-30", + "other_than_full_and_o_desc": None, + "parent_award_id": "1", + "period_of_performance_star": "2010-09-23", + "piid": "0", + "pk": 2, + "place_of_perf_country_desc": "Pacific Delta Amazon", + "place_of_perform_city_name": "Austin", + "place_of_perform_country_c": "PDA", + "place_of_perform_county_na": "Tripoli", + "place_of_performance_congr": "-0-", + "place_of_performance_state": "TX", + "place_of_performance_zip4a": "2135", + "place_of_performance_zip5": "40221", "price_evaluation_adjustmen": None, "product_or_service_code": "4730", "program_acronym": None, - "other_than_full_and_o_desc": None, + "program_system_or_equipmen": "000", + "purchase_card_as_paym_desc": "NO", + "referenced_idv_agency_iden": "168", "sea_transportation_desc": "NO", - "labor_standards_descrip": "NO", "small_business_competitive": "False", "solicitation_identifier": None, "solicitation_procedures": "NP", - "fair_opportunity_limi_desc": None, "subcontracting_plan": "B", - "program_system_or_equipmen": "000", + "transaction": TransactionNormalized.objects.get(pk=2), + "type_of_contract_pric_desc": "FIRM FIXED PRICE", + "type_of_idc_description": "INDEFINITE DELIVERY / INDEFINITE QUANTITY", "type_set_aside_description": None, - "materials_supplies_descrip": "NO", - "domestic_or_foreign_e_desc": "U.S. OWNED BUSINESS", - "ordering_period_end_date": "2025-06-30", - "period_of_performance_star": "2010-09-23", - "last_modified": "2018-08-24", - "agency_id": "192", - "referenced_idv_agency_iden": "168", - "piid": "0", - "parent_award_id": "1", + "ultimate_parent_legal_enti": None, + "ultimate_parent_unique_ide": "123", } mommy.make("awards.TransactionFABS", **asst_data) mommy.make("awards.TransactionFPDS", **latest_transaction_contract_data) @@ -144,9 +160,9 @@ def awards_and_transactions(db): award_2_model = { "pk": 2, - "type": "A", - "type_description": "DEFINITIVE CONTRACT", - "category": "contract", + "type": "IDV_A", + "type_description": "GWAC", + "category": "idv", "piid": "5678", "parent_award_piid": "1234", "description": "lorem ipsum", @@ -169,7 +185,7 @@ def test_no_data_idv_award_endpoint(client): """Test the /v2/awards endpoint.""" resp = client.get("/api/v2/awards/27254436/", content_type="application/json") - assert resp.status_code == status.HTTP_200_OK + assert resp.status_code == status.HTTP_404_NOT_FOUND @pytest.mark.django_db @@ -206,27 +222,29 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): expected_response_idv = { "id": 2, - "type": "A", - "category": "contract", - "type_description": "DEFINITIVE CONTRACT", + "type": "IDV_A", + "parent_generated_unique_award_id": None, + "generated_unique_award_id": "CONT_AW_9700_9700_03VD_SPM30012D3486", + "category": "idv", + "type_description": "GWAC", "piid": "5678", "parent_award_piid": "1234", "description": "lorem ipsum", + "idv_dates": {"end_date": "2025-06-30", "last_modified_date": "2018-08-24", "start_date": None}, "awarding_agency": { "id": 1, - "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, - "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, + "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, + "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, "office_agency_name": "office_agency", - "office_agency": {"aac_code": None, "name": "office_agency"}, }, "funding_agency": { "id": 1, - "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, - "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff"}, + "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, + "subtier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, "office_agency_name": "office_agency", - "office_agency": {"aac_code": None, "name": "office_agency"}, }, "recipient": { + "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367", "recipient_name": "John's Pizza", "recipient_unique_id": "456", "parent_recipient_unique_id": "123", @@ -240,7 +258,7 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "county_name": "BUNCOMBE", "state_code": "NC", "zip5": "12204", - "zip4": "122045312", + "zip4": "5312", "foreign_postal_code": None, "country_name": "UNITED STATES", "location_country_code": "USA", @@ -248,10 +266,24 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): }, "parent_recipient_name": None, }, - "total_obligation": "1000.00", - "base_and_all_options_value": "2000.00", - "period_of_performance": {"period_of_performance_current_end_date": None, "period_of_performance_start_date": None}, - "place_of_performance": None, + "total_obligation": 1000.0, + "base_and_all_options_value": 2000.0, + "base_exercised_options_val": None, + "place_of_performance": { + "address_line1": None, + "address_line2": None, + "address_line3": None, + "foreign_province": None, + "city_name": "Austin", + "county_name": "Tripoli", + "state_code": "TX", + "zip5": "40221", + "zip4": "2135", + "foreign_postal_code": None, + "country_name": "Pacific Delta Amazon", + "location_country_code": "PDA", + "congressional_code": "-0-", + }, "latest_transaction_contract_data": { "idv_type_description": "IDC", "type_of_idc_description": "INDEFINITE DELIVERY / INDEFINITE QUANTITY", @@ -292,9 +324,8 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "purchase_card_as_paym_desc": "NO", "consolidated_contract_desc": "NOT CONSOLIDATED", "type_of_contract_pric_desc": "FIRM FIXED PRICE", - "ordering_period_end_date": "2025-06-30", }, "subaward_count": 10, - "total_subaward_amount": "12345.00", + "total_subaward_amount": 12345.0, "executive_details": {"officers": []}, } diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index d7480c6814..de44a57af0 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -27,16 +27,16 @@ def awards_and_transactions(db): "foreign_postal_code": None, "foreign_province": None, } - subag = {"pk": 1, "name": "agency name", "abbreviation": "some other stuff"} + sub_agency = {"pk": 1, "name": "agency name", "abbreviation": "some other stuff"} trans_asst = {"pk": 1} trans_cont = {"pk": 2} duns = {"awardee_or_recipient_uniqu": 123, "legal_business_name": "Sams Club"} mommy.make("references.Cfda", program_number=1234) mommy.make("references.Location", **loc) mommy.make("recipient.DUNS", **duns) - mommy.make("references.SubtierAgency", **subag) - mommy.make("references.ToptierAgency", **subag) + mommy.make("references.SubtierAgency", **sub_agency) + mommy.make("references.ToptierAgency", **sub_agency) mommy.make("references.OfficeAgency", name="office_agency", office_agency_id=1) le = { From 9c0f8fb95a17e4af966e1b55d828ec817c9b5647 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Sat, 15 Dec 2018 23:20:07 -0500 Subject: [PATCH 10/52] Updated API markdown and fixed flake8 --- .../api_documentation/awards/awards.md | 452 ++++++++++++------ .../awards/tests/test_awards_v2.py | 172 +++---- 2 files changed, 368 insertions(+), 256 deletions(-) diff --git a/usaspending_api/api_docs/api_documentation/awards/awards.md b/usaspending_api/api_docs/api_documentation/awards/awards.md index c2f77007cc..8ecc408c25 100644 --- a/usaspending_api/api_docs/api_documentation/awards/awards.md +++ b/usaspending_api/api_docs/api_documentation/awards/awards.md @@ -16,6 +16,19 @@ If the award has a `category` of `contract`, the response will look like this: ``` { + "category": "contract", + "parent_award_piid": null, + "total_obligation": 2208, + "type_description": "DEFINITIVE CONTRACT", + "type": "D", + "piid": "47PB0618C0001", + "base_and_all_options_value": 2208, + "description": "IGF::CT::IGF COPIER MAINTENANCE SERVICES FOR GSA OFFICE, SPRINGFIELD, MA", + "total_subaward_amount": null, + "base_exercised_options_val": null, + "id": 68584146, + "subaward_count": 0, + "generated_unique_award_id": "CONT_AW_4740_-NONE-_47PB0618C0001_-NONE-", "executive_details": { "officers": [ { @@ -40,226 +53,363 @@ If the award has a `category` of `contract`, the response will look like this: } ] }, - "period_of_performance": { - "period_of_performance_start_date": "2015-12-29", - "period_of_performance_current_end_date": "2016-01-14" - }, "latest_transaction_contract_data": { + "small_business_competitive": false, + "program_system_or_equipmen": null, + "fair_opportunity_limi_desc": null, "type_of_contract_pric_desc": "FIRM FIXED PRICE", - "naics": "311812", - "naics_description": "COMMERCIAL BAKERIES", - "referenced_idv_agency_iden": "9700", - "idv_type_description": null, - "multiple_or_single_aw_desc": null, - "type_of_idc_description": null, - "dod_claimant_program_code": "B2", - "clinger_cohen_act_pla_desc": "NO", - "commercial_item_acquisitio": "A", + "program_acronym": null, + "product_or_service_code": "7540", + "subcontracting_plan": "B", + "referenced_idv_agency_iden": null, "commercial_item_test_desc": "NO", - "consolidated_contract_desc": "NO", + "construction_wage_rat_desc": "NOT APPLICABLE", + "price_evaluation_adjustmen": "0.00", + "labor_standards_descrip": "NOT APPLICABLE", + "materials_supplies_descrip": "NOT APPLICABLE", + "solicitation_identifier": null, + "clinger_cohen_act_pla_desc": "NO", "cost_or_pricing_data_desc": "NO", - "construction_wage_rat_desc": "NO", + "sea_transportation_desc": null, + "multi_year_contract_desc": "NO", + "fed_biz_opps_description": "NOT APPLICABLE", + "other_than_full_and_o_desc": "ONLY ONE SOURCE-OTHER (FAR 6.302-1 OTHER)", + "commercial_item_acquisitio": "D", + "idv_type_description": null, + "domestic_or_foreign_e_desc": "U.S. OWNED BUSINESS", "evaluated_preference_desc": "NO PREFERENCE USED", - "extent_competed": "A", - "fed_biz_opps_description": "NO", - "foreign_funding_desc": "NOT APPLICABLE", - "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", + "type_set_aside_description": "NO SET ASIDE USED.", + "product_or_service_co_desc": "STANDARD FORMS", + "information_technolog_desc": null, "interagency_contract_desc": "NOT APPLICABLE", + "type_of_idc_description": null, + "dod_claimant_program_code": null, "major_program": null, + "extent_competed": "C", + "multiple_or_single_aw_desc": null, + "solicitation_procedures": "SSS", + "naics": "811212", "purchase_card_as_paym_desc": "NO", - "multi_year_contract_desc": "NO", - "number_of_offers_received": "4", - "price_evaluation_adjustmen": null, - "product_or_service_code": "8940", - "program_acronym": null, - "other_than_full_and_o_desc": null, - "sea_transportation_desc": "UNKNOWN", - "labor_standards_descrip": "NO", - "small_business_competitive": "False", - "solicitation_identifier": null, - "solicitation_procedures": "NP", - "fair_opportunity_limi_desc": null, - "subcontracting_plan": null, - "program_system_or_equipmen": "000", - "type_set_aside_description": "NO SET ASIDE USED.", - "materials_supplies_descrip": "NO", - "domestic_or_foreign_e_desc": "FOREIGN-OWNED BUSINESS NOT INCORPORATED IN THE U.S." + "foreign_funding_desc": "NOT APPLICABLE", + "consolidated_contract_desc": "NOT CONSOLIDATED", + "number_of_offers_received": "1" }, - "type": "C", - "type_description": "DELIVERY ORDER", - "category": "contract", - "piid": "03VD", - "total_obligation": "932.10", - "description": "4532842749!EGGS, SHELL, FRESH, MED,", - "base_and_all_options_value": "932.10", - "parent_award_piid": "SPM30012D3486", - "total_subaward_amount": "0.00", - "subaward_count": 0, - "awarding_agency": { + "funding_agency": { + "subtier_agency": { + "name": "Public Buildings Service", + "abbreviation": "PBS", + "code": "4740" + }, "office_agency_name": null, + "id": 636, "toptier_agency": { - "abbreviation": "DOD", - "name": "Department of Defense" - }, + "name": "General Services Administration", + "abbreviation": "GSA", + "code": "4700" + } + }, + "awarding_agency": { "subtier_agency": { - "abbreviation": "DLA", - "name": "Defense Logistics Agency" + "name": "Public Buildings Service", + "abbreviation": "PBS", + "code": "4740" + }, + "office_agency_name": null, + "id": 636, + "toptier_agency": { + "name": "General Services Administration", + "abbreviation": "GSA", + "code": "4700" + } + }, + "period_of_performance": { + "period_of_performance_start_date": "2017-10-01", + "period_of_performance_current_end_date": "2018-09-30" + }, + "recipient": { + "recipient_hash": "41c9da13-f8e8-7620-9e00-8681acd9c720", + "recipient_name": "DOCUSOURCE BUSINESS SOLUTIONS, LLC", + "recipient_unique_id": "828026281", + "parent_recipient_unique_id": "828026281", + "parent_recipient_name": "DOCUSOURCE BUSINESS SOLUTIONS LLC", + "business_categories": [ + "us_owned_business", + "special_designations", + "small_business", + "category_business" + ], + "location": { + "location_country_code": "USA", + "country_name": "UNITED STATES", + "state_code": "CT", + "city_name": "MIDDLETOWN", + "county_name": "MIDDLESEX", + "address_line1": "299 INDUSTRIAL PARK RD", + "address_line2": null, + "address_line3": null, + "congressional_code": "03", + "zip4": "1535", + "zip5": "06457", + "foreign_postal_code": null, + "foreign_province": null } }, + "place_of_performance": { + "location_country_code": "USA", + "country_name": "UNITED STATES", + "county_name": "HAMPDEN", + "city_name": "SPRINGFIELD", + "state_code": "MA", + "congressional_code": "01", + "zip4": "011031422", + "zip5": "01103", + "address_line1": null, + "address_line2": null, + "address_line3": null, + "foreign_province": null, + "foreign_postal_code": null + } +} +``` + +If the `category` is `idv`, the response will look like this: + +``` +{ + "category": "idv", + "parent_award_piid": null, + "total_obligation": 0, + "type_description": "BPA", + "type": "IDV_E", + "piid": "190PRL18A0613", + "base_and_all_options_value": 0, + "description": "PROVIDING PROFESSIONAL FOREIGN LANGUAGE INTERPRETING AND TRANSLATING SERVICES", + "total_subaward_amount": null, + "base_exercised_options_val": null, + "id": 65284276, + "subaward_count": 0, + "generated_unique_award_id": "CONT_AW_1900_-NONE-_190PRL18A0613_-NONE-", + "parent_generated_unique_award_id": null, + "executive_details": { + "officers": [] + }, + "latest_transaction_contract_data": { + "small_business_competitive": false, + "program_system_or_equipmen": null, + "fair_opportunity_limi_desc": null, + "type_of_contract_pric_desc": "FIRM FIXED PRICE", + "program_acronym": null, + "product_or_service_code": "R608", + "subcontracting_plan": null, + "referenced_idv_agency_iden": null, + "commercial_item_test_desc": null, + "construction_wage_rat_desc": "NOT APPLICABLE", + "price_evaluation_adjustmen": null, + "labor_standards_descrip": "NOT APPLICABLE", + "materials_supplies_descrip": "NOT APPLICABLE", + "solicitation_identifier": null, + "clinger_cohen_act_pla_desc": "NO", + "cost_or_pricing_data_desc": null, + "sea_transportation_desc": null, + "multi_year_contract_desc": null, + "fed_biz_opps_description": "NOT APPLICABLE", + "other_than_full_and_o_desc": null, + "commercial_item_acquisitio": "A", + "idv_type_description": "BPA", + "domestic_or_foreign_e_desc": "U.S. OWNED BUSINESS", + "evaluated_preference_desc": null, + "type_set_aside_description": null, + "product_or_service_co_desc": "SUPPORT- ADMINISTRATIVE: TRANSLATION AND INTERPRETING", + "information_technolog_desc": null, + "interagency_contract_desc": "OTHER STATUTORY AUTHORITY", + "type_of_idc_description": null, + "dod_claimant_program_code": null, + "major_program": null, + "extent_competed": null, + "multiple_or_single_aw_desc": "MULTIPLE AWARD", + "solicitation_procedures": null, + "naics": "541930", + "purchase_card_as_paym_desc": null, + "foreign_funding_desc": "NOT APPLICABLE", + "consolidated_contract_desc": "NOT CONSOLIDATED", + "number_of_offers_received": null + }, "funding_agency": { + "subtier_agency": { + "name": "Department of State", + "abbreviation": "DOS", + "code": "1900" + }, "office_agency_name": null, + "id": 315, "toptier_agency": { - "abbreviation": "DOD", - "name": "Department of Defense" - }, + "name": "Department of State", + "abbreviation": "DOS", + "code": "1900" + } + }, + "awarding_agency": { "subtier_agency": { - "abbreviation": "DLA", - "name": "Defense Logistics Agency" + "name": "Department of State", + "abbreviation": "DOS", + "code": "1900" + }, + "office_agency_name": null, + "id": 315, + "toptier_agency": { + "name": "Department of State", + "abbreviation": "DOS", + "code": "1900" } }, + "idv_dates": { + "start_date": "2017-10-01", + "last_modified_date": "2018-04-04 15:21:28", + "end_date": "2018-09-30" + }, "recipient": { - "recipient_parent_name": null, - "parent_recipient_unique_id": "520233955", - "recipient_name": "EBREX FOOD SERVICES SARL", + "recipient_hash": "d818c3f3-b5a2-add8-8365-79c1213e2385", + "recipient_name": "DUCELUS, ROOSEVELT", + "recipient_unique_id": "962788902", + "parent_recipient_unique_id": "962788902", + "parent_recipient_name": "DUCELUS ROOSEVELT", "business_categories": [ "other_than_small_business", "sole_proprietorship", "category_business", - "foreign_owned_and_located_business", + "black_american_owned_business", + "minority_owned_business", + "us_owned_business", "special_designations" ], - "recipient_unique_id": "480520290", "location": { - "location_country_code": "CHE", - "country_name": "SWITZERLAND", - "state_code": null, - "city_name": "GENEVE", - "county_name": null, - "address_line1": "RUE MAUNOIR 16", + "location_country_code": "USA", + "country_name": "UNITED STATES", + "state_code": "GA", + "city_name": "BETHLEHEM", + "county_name": "BARROW", + "address_line1": "1900 NATALIE DR", "address_line2": null, "address_line3": null, - "zip4": "1207", - "congressional_code": null, - "zip5": null, + "congressional_code": "10", + "zip4": "2570", + "zip5": "30620", "foreign_postal_code": null, "foreign_province": null } }, "place_of_performance": { - "location_country_code": "CHE", - "country_name": "SWITZERLAND", - "state_code": null, - "city_name": null, + "location_country_code": null, + "country_name": null, "county_name": null, + "city_name": null, + "state_code": null, + "congressional_code": null, + "zip4": null, + "zip5": null, "address_line1": null, "address_line2": null, "address_line3": null, - "zip4": null, - "congressional_code": null, - "zip5": null, - "foreign_postal_code": null, - "foreign_province": null + "foreign_province": null, + "foreign_postal_code": null } } - ``` -If the `category` of the award is anything other than `contract`, the response will instead look like this: +If the `category` of the award is any assistance type, the response will instead look like this: ``` - { - "period_of_performance": { - "period_of_performance_start_date": "2009-09-21", - "period_of_performance_current_end_date": "2021-07-31" - }, - "executive_details": { - "officers": [ - { - "name": "Jeannie K Peters", - "amount": "145000.00" - }, - { - "name": "James Statler", - "amount": "139639.00" - }, - { - "name": "Amy Smith", - "amount": "127991.00" - }, - { - "name": "Lisa Beran", - "amount": "127991.00" - }, - { - "name": "Kevin Field", - "amount": "120746.00" - } - ] - }, - "cfda_objectives": "To provide rental assistance to very low income individuals and families enabling them to live in affordable decent, safe and sanitary housing.", - "cfda_title": "SECTION 8 HOUSING ASSISTANCE PAYMENTS PROGRAM", - "cfda_number": "14.195", - "type": "06", - "type_description": "DIRECT PAYMENT FOR SPECIFIED USE", - "category": "direct payment", - "piid": null, - "description": "CONT RENEWALS ALL TYPES", - "total_subaward_amount": "0.00", - "subaward_count": 0, + "non_federal_funding": null, "awarding_agency": { - "office_agency_name": null, - "toptier_agency": { - "abbreviation": "HUD", - "name": "Department of Housing and Urban Development" - }, "subtier_agency": { + "name": "Under Secretary for Health/Veterans Health Administration", "abbreviation": "", - "name": "Assistant Secretary for Housing--Federal Housing Commissioner" + "code": "3620" + }, + "office_agency_name": null, + "id": 580, + "toptier_agency": { + "name": "Department of Veterans Affairs", + "abbreviation": "VA", + "code": "3600" } }, - "funding_agency": null, + "category": "other", + "total_funding": 133004, + "base_and_all_options_value": null, + "description": "VHA MEDICAL PROCESS", + "base_exercised_options": null, + "total_subaward_amount": null, "recipient": { - "recipient_parent_name": null, + "recipient_hash": "005727d6-ca1c-fc53-add0-3b03fb8b6933", + "recipient_name": "MULTIPLE RECIPIENTS", + "recipient_unique_id": null, "parent_recipient_unique_id": null, - "recipient_name": "KENTUCKY HOUSING CORPORATION", - "business_categories": [ - "other_than_small_business", - "category_business" - ], - "recipient_unique_id": "082316696", + "parent_recipient_name": null, + "business_categories": [], "location": { "location_country_code": "USA", "country_name": "UNITED STATES", - "state_code": "KY", - "city_name": "FRANKFORT", - "county_name": "FRANKLIN", - "address_line1": "1231 LOUISVILLE ROAD", + "state_code": "PA", + "city_name": null, + "county_name": "NORTHUMBERLAND", + "address_line1": null, "address_line2": null, "address_line3": null, + "congressional_code": "90", "zip4": null, - "congressional_code": "06", - "zip5": "40601", + "zip5": null, "foreign_postal_code": null, "foreign_province": null } }, + "id": 68463998, + "generated_unique_award_id": "ASST_AW_3620_-NONE-_18090913.0121844", + "total_subsidy_cost": null, + "uri": "18090913.0121844", + "total_loan_value": null, + "total_obligation": 133004, + "type_description": "OTHER REIMBURSABLE, CONTINGENT, INTANGIBLE, OR INDIRECT FINANCIAL ASSISTANCE", + "type": "11", + "funding_agency": { + "subtier_agency": { + "name": "Department of Veterans Affairs", + "abbreviation": "VA", + "code": "3600" + }, + "office_agency_name": null, + "id": 561, + "toptier_agency": { + "name": "Department of Veterans Affairs", + "abbreviation": "VA", + "code": "3600" + } + }, + "fain": null, + "subaward_count": 0, + "cfda_number": "64.012", + "cfda_title": "VETERANS PRESCRIPTION SERVICE", + "cfda_objectives": "To provide eligible veterans and certain dependents and survivors of veterans with prescription drugs and expendable medical supplies from VA pharmacies upon presentation of prescription(s) from a VA provider or VA authorized provider.", + "period_of_performance": { + "period_of_performance_start_date": null, + "period_of_performance_current_end_date": null + }, "place_of_performance": { "location_country_code": "USA", "country_name": "UNITED STATES", - "state_code": "KY", - "city_name": "FRANKFORT", - "county_name": "FRANKLIN", + "county_name": "NORTHUMBERLAND", + "city_name": null, + "state_code": "PA", + "congressional_code": "90", + "zip4": null, + "zip5": null, "address_line1": null, "address_line2": null, "address_line3": null, - "zip4": null, - "congressional_code": "06", - "zip5": "40601", - "foreign_postal_code": null, - "foreign_province": null + "foreign_province": null, + "foreign_postal_code": null } } - - +``` diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index de44a57af0..5b67940c0b 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -87,109 +87,71 @@ def awards_and_transactions(db): "place_of_performance_forei": None, } cont_data = { - "pk": 2, - "transaction": TransactionNormalized.objects.get(pk=2), - "idv_type_description": "", - "type_of_idc_description": "", - "referenced_idv_agency_iden": "", - "multiple_or_single_aw_desc": "", - "solicitation_identifier": "", - "solicitation_procedures": "", - "number_of_offers_received": "", - "extent_competed": "", - "other_than_full_and_o_desc": "", - "type_set_aside_description": "", - "commercial_item_acquisitio": "", - "commercial_item_test_desc": "", - "evaluated_preference_desc": "", - "fed_biz_opps_description": "", - "small_business_competitive": "", - "fair_opportunity_limi_desc": "", - "product_or_service_code": "", - "product_or_service_co_desc": None, - "naics": "", - "dod_claimant_program_code": "", - "program_system_or_equipmen": "", - "information_technolog_desc": "", - "sea_transportation_desc": "", - "clinger_cohen_act_pla_desc": "", - "construction_wage_rat_desc": "", - "labor_standards_descrip": "", - "materials_supplies_descrip": "", - "cost_or_pricing_data_desc": "", - "domestic_or_foreign_e_desc": "", - "foreign_funding_desc": "", - "interagency_contract_desc": "", - "major_program": "", - "price_evaluation_adjustmen": "", - "program_acronym": "", - "subcontracting_plan": "", - "multi_year_contract_desc": "", - "purchase_card_as_paym_desc": "", - "consolidated_contract_desc": "", - "type_of_contract_pric_desc": "", "awardee_or_recipient_legal": "John's Pizza", "awardee_or_recipient_uniqu": "456", - "ultimate_parent_legal_enti": None, - "ultimate_parent_unique_ide": "123", - "legal_entity_country_code": "USA", - "legal_entity_country_name": "UNITED STATES", - "legal_entity_state_code": "NC", - "legal_entity_city_name": "Charlotte", - "legal_entity_county_name": "BUNCOMBE", - "legal_entity_address_line1": "123 main st", - "legal_entity_address_line2": None, - "legal_entity_address_line3": None, - "legal_entity_congressional": "90", - "legal_entity_zip_last4": "5312", - "legal_entity_zip5": "12204", - "place_of_perform_country_c": "USA", - "place_of_perf_country_desc": "UNITED STATES", - "place_of_performance_state": "NC", - "place_of_perform_city_name": "Charlotte", - "place_of_perform_county_na": "BUNCOMBE", - "place_of_performance_zip4a": "5312", - "place_of_performance_congr": "90", - "place_of_performance_zip5": "12204", - "type_of_contract_pric_desc": "FIRM FIXED PRICE", - "naics": "333911", - "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", - "referenced_idv_agency_iden": "9700", - "idv_type_description": None, - "multiple_or_single_aw_desc": None, - "type_of_idc_description": None, - "dod_claimant_program_code": "C9E", "clinger_cohen_act_pla_desc": "NO", "commercial_item_acquisitio": "A", "commercial_item_test_desc": "NO", "consolidated_contract_desc": "NOT CONSOLIDATED", - "cost_or_pricing_data_desc": "NO", "construction_wage_rat_desc": "NO", + "cost_or_pricing_data_desc": "NO", + "dod_claimant_program_code": "C9E", + "domestic_or_foreign_e_desc": "U.S. OWNED BUSINESS", "evaluated_preference_desc": "NO PREFERENCE USED", "extent_competed": "D", + "fair_opportunity_limi_desc": None, "fed_biz_opps_description": "YES", "foreign_funding_desc": "NOT APPLICABLE", + "idv_type_description": None, "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", "interagency_contract_desc": "NOT APPLICABLE", + "labor_standards_descrip": "NO", + "legal_entity_address_line1": "123 main st", + "legal_entity_address_line2": None, + "legal_entity_address_line3": None, + "legal_entity_city_name": "Charlotte", + "legal_entity_congressional": "90", + "legal_entity_country_code": "USA", + "legal_entity_country_name": "UNITED STATES", + "legal_entity_county_name": "BUNCOMBE", + "legal_entity_state_code": "NC", + "legal_entity_zip5": "12204", + "legal_entity_zip_last4": "5312", "major_program": None, - "purchase_card_as_paym_desc": "NO", + "materials_supplies_descrip": "NO", "multi_year_contract_desc": "NO", + "multiple_or_single_aw_desc": None, + "naics": "333911", + "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", "number_of_offers_received": None, + "other_than_full_and_o_desc": None, + "pk": 2, + "place_of_perf_country_desc": "UNITED STATES", + "place_of_perform_city_name": "Charlotte", + "place_of_perform_country_c": "USA", + "place_of_perform_county_na": "BUNCOMBE", + "place_of_performance_congr": "90", + "place_of_performance_state": "NC", + "place_of_performance_zip4a": "5312", + "place_of_performance_zip5": "12204", "price_evaluation_adjustmen": None, + "product_or_service_co_desc": None, "product_or_service_code": "4730", "program_acronym": None, - "other_than_full_and_o_desc": None, + "program_system_or_equipmen": "000", + "purchase_card_as_paym_desc": "NO", + "referenced_idv_agency_iden": "9700", "sea_transportation_desc": "NO", - "labor_standards_descrip": "NO", "small_business_competitive": "False", "solicitation_identifier": None, "solicitation_procedures": "NP", - "fair_opportunity_limi_desc": None, "subcontracting_plan": "B", - "program_system_or_equipmen": "000", + "transaction": TransactionNormalized.objects.get(pk=2), + "type_of_contract_pric_desc": "FIRM FIXED PRICE", + "type_of_idc_description": None, "type_set_aside_description": None, - "materials_supplies_descrip": "NO", - "domestic_or_foreign_e_desc": "U.S. OWNED BUSINESS", + "ultimate_parent_legal_enti": None, + "ultimate_parent_unique_ide": "123", } mommy.make("awards.TransactionFABS", **asst_data) mommy.make("awards.TransactionFPDS", **cont_data) @@ -466,45 +428,45 @@ def test_idv_award_amount_endpoint(client): "congressional_code": "90", }, "latest_transaction_contract_data": { - "idv_type_description": None, - "type_of_idc_description": None, - "referenced_idv_agency_iden": "9700", - "multiple_or_single_aw_desc": None, - "solicitation_identifier": None, - "solicitation_procedures": "NP", - "number_of_offers_received": None, - "extent_competed": "D", - "other_than_full_and_o_desc": None, - "type_set_aside_description": None, + "clinger_cohen_act_pla_desc": "NO", "commercial_item_acquisitio": "A", "commercial_item_test_desc": "NO", - "evaluated_preference_desc": "NO PREFERENCE USED", - "fed_biz_opps_description": "YES", - "small_business_competitive": "False", - "fair_opportunity_limi_desc": None, - "product_or_service_code": "4730", - "product_or_service_co_desc": None, - "naics": "333911", - "dod_claimant_program_code": "C9E", - "program_system_or_equipmen": "000", - "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", - "sea_transportation_desc": "NO", - "clinger_cohen_act_pla_desc": "NO", + "consolidated_contract_desc": "NOT CONSOLIDATED", "construction_wage_rat_desc": "NO", - "labor_standards_descrip": "NO", - "materials_supplies_descrip": "NO", "cost_or_pricing_data_desc": "NO", + "dod_claimant_program_code": "C9E", "domestic_or_foreign_e_desc": "U.S. OWNED BUSINESS", + "evaluated_preference_desc": "NO PREFERENCE USED", + "extent_competed": "D", + "fair_opportunity_limi_desc": None, + "fed_biz_opps_description": "YES", "foreign_funding_desc": "NOT APPLICABLE", + "idv_type_description": None, + "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", "interagency_contract_desc": "NOT APPLICABLE", + "labor_standards_descrip": "NO", "major_program": None, + "materials_supplies_descrip": "NO", + "multi_year_contract_desc": "NO", + "multiple_or_single_aw_desc": None, + "naics": "333911", + "number_of_offers_received": None, + "other_than_full_and_o_desc": None, "price_evaluation_adjustmen": None, + "product_or_service_co_desc": None, + "product_or_service_code": "4730", "program_acronym": None, - "subcontracting_plan": "B", - "multi_year_contract_desc": "NO", + "program_system_or_equipmen": "000", "purchase_card_as_paym_desc": "NO", - "consolidated_contract_desc": "NOT CONSOLIDATED", + "referenced_idv_agency_iden": "9700", + "sea_transportation_desc": "NO", + "small_business_competitive": "False", + "solicitation_identifier": None, + "solicitation_procedures": "NP", + "subcontracting_plan": "B", "type_of_contract_pric_desc": "FIRM FIXED PRICE", + "type_of_idc_description": None, + "type_set_aside_description": None, }, "subaward_count": 10, "total_subaward_amount": 12345.0, From 8ddbfb7bdd2fcd1d87a5b8a0e12d14c794bb28d1 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Mon, 17 Dec 2018 11:32:17 -0500 Subject: [PATCH 11/52] Added missing naics_description --- .../awards/v2/data_layer/orm_mappers.py | 1 + .../awards/v2/data_layer/orm_utils.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index 23d8fcfca6..0ef31290c3 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -121,6 +121,7 @@ ("product_or_service_code", "product_or_service_code"), ("product_or_service_co_desc", "product_or_service_co_desc"), ("naics", "naics"), + ("naics_description", "naics_description"), ("dod_claimant_program_code", "dod_claimant_program_code"), ("program_system_or_equipmen", "program_system_or_equipmen"), ("information_technolog_desc", "information_technolog_desc"), diff --git a/usaspending_api/awards/v2/data_layer/orm_utils.py b/usaspending_api/awards/v2/data_layer/orm_utils.py index e63e9da399..d0bfb44cf0 100644 --- a/usaspending_api/awards/v2/data_layer/orm_utils.py +++ b/usaspending_api/awards/v2/data_layer/orm_utils.py @@ -15,6 +15,23 @@ def delete_keys_from_dict(dictionary): def split_mapper_into_qs(mapper): + """ + Django ORM has trouble using .annotate() when the destination field conflicts with an + existing model field, even if it's the same source field (no renaming occuring) + + Assuming there is a dictionary with model fieldnames as keys and the target field as the value, + Split that into two objects: + values_list: a list of fields which you wish to retrive without renaming + aka when `key` == `value` + annotate_dict: a dictionary/OrderedDict of target and source fields to rename + + parameters + - mapper: dictionary/OrderedDict + + return: + - values_list: list + -annotate_dict: dictionary/OrderedDict + """ values_list = [k for k, v in mapper.items() if k == v] annotate_dict = OrderedDict([(v, F(k)) for k, v in mapper.items() if k != v]) From b7eb00d38945788c34e0517430aec5c3a7144bc8 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Mon, 17 Dec 2018 13:37:13 -0500 Subject: [PATCH 12/52] fixed tests after merging dev --- usaspending_api/awards/tests/test_awards_idv_v2.py | 3 ++- usaspending_api/awards/tests/test_awards_v2.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index 94b3077ea8..6c106fba10 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -299,11 +299,12 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "commercial_item_test_desc": "NO", "evaluated_preference_desc": "NO PREFERENCE USED", "fed_biz_opps_description": "YES", - "small_business_competitive": "False", + "small_business_competitive": False, "fair_opportunity_limi_desc": None, "product_or_service_code": "4730", "product_or_service_co_desc": None, "naics": "333911", + "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", "dod_claimant_program_code": "C9E", "program_system_or_equipmen": "000", "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 5b67940c0b..501f77788e 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -142,7 +142,7 @@ def awards_and_transactions(db): "purchase_card_as_paym_desc": "NO", "referenced_idv_agency_iden": "9700", "sea_transportation_desc": "NO", - "small_business_competitive": "False", + "small_business_competitive": False, "solicitation_identifier": None, "solicitation_procedures": "NP", "subcontracting_plan": "B", @@ -450,6 +450,7 @@ def test_idv_award_amount_endpoint(client): "multi_year_contract_desc": "NO", "multiple_or_single_aw_desc": None, "naics": "333911", + "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", "number_of_offers_received": None, "other_than_full_and_o_desc": None, "price_evaluation_adjustmen": None, @@ -460,7 +461,7 @@ def test_idv_award_amount_endpoint(client): "purchase_card_as_paym_desc": "NO", "referenced_idv_agency_iden": "9700", "sea_transportation_desc": "NO", - "small_business_competitive": "False", + "small_business_competitive": False, "solicitation_identifier": None, "solicitation_procedures": "NP", "subcontracting_plan": "B", From fe41def75de65fe312675890a97273f214441024 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Mon, 17 Dec 2018 16:32:51 -0500 Subject: [PATCH 13/52] Updating response to meet updated API contract --- .../awards/tests/test_awards_idv_v2.py | 1 + usaspending_api/awards/v2/data_layer/orm.py | 42 ++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index 6c106fba10..f9a0b64ba0 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -229,6 +229,7 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "type_description": "GWAC", "piid": "5678", "parent_award_piid": "1234", + "parent_award": None, "description": "lorem ipsum", "idv_dates": {"end_date": "2025-06-30", "last_modified_date": "2018-08-24", "start_date": None}, "awarding_agency": { diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 6b12fc03ec..6f18f81f7f 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -109,7 +109,9 @@ def construct_idv_response(requested_award_dict): return None response.update(award) - response["parent_generated_unique_award_id"] = fetch_parent_award_id(award["generated_unique_award_id"]) + parent_award = fetch_parent_award_details(award["generated_unique_award_id"]) + response["parent_award"] = parent_award + response["parent_generated_unique_award_id"] = parent_award["generated_unique_award_id"] if parent_award else None response["executive_details"] = fetch_officers_by_legal_entity_id(award["_lei"]) response["latest_transaction_contract_data"] = fetch_fpds_details_by_pk(award["_trx"], mapper) response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) @@ -191,14 +193,44 @@ def fetch_award_details(filter_q, mapper_fields): return Award.objects.filter(**filter_q).values(*vals).annotate(**ann).first() -def fetch_parent_award_id(guai): - parent_award = ( +def fetch_parent_award_details(guai): + parent_award_ids = ( ParentAward.objects.filter(generated_unique_award_id=guai) - .values("parent_award__generated_unique_award_id") + .values("parent_award__award_id", "parent_award__generated_unique_award_id") + .first() + ) + + if not parent_award_ids: + return None + + parent_award = ( + Award.objects.filter(id=parent_award_ids["parent_award__award_id"]) + .values( + "latest_transaction__contract_data__agency_id", + "latest_transaction__contract_data__idv_type_description", + "latest_transaction__contract_data__multiple_or_single_aw_desc", + "latest_transaction__contract_data__piid", + "latest_transaction__contract_data__type_of_idc_description", + ) .first() ) - return parent_award.get("generated_unique_award_id") if parent_award else None + parent_object = OrderedDict( + [ + ("agency_id", parent_award["latest_transaction__contract_data__agency_id"]), + ("award_id", parent_award_ids["parent_award__award_id"]), + ("generated_unique_award_id", parent_award_ids["parent_award__generated_unique_award_id"]), + ("idv_type_description", parent_award["latest_transaction__contract_data__idv_type_description"]), + ( + "multiple_or_single_aw_desc", + parent_award["latest_transaction__contract_data__multiple_or_single_aw_desc"], + ), + ("piid", parent_award["latest_transaction__contract_data__piid"]), + ("type_of_idc_description", parent_award["latest_transaction__contract_data__type_of_idc_description"]), + ] + ) + + return parent_object def fetch_fabs_details_by_pk(primary_key, mapper): From 6a4005c1cef9d784208ca3a3f919fd61e4762335 Mon Sep 17 00:00:00 2001 From: Justin Le Date: Wed, 19 Dec 2018 13:19:49 -0500 Subject: [PATCH 14/52] Working Implementation --- .../references/management/commands/load_rosetta.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/usaspending_api/references/management/commands/load_rosetta.py b/usaspending_api/references/management/commands/load_rosetta.py index b20701bb55..04b4c0514b 100644 --- a/usaspending_api/references/management/commands/load_rosetta.py +++ b/usaspending_api/references/management/commands/load_rosetta.py @@ -1,6 +1,7 @@ import logging import os.path import boto3 +import json from collections import OrderedDict from django.conf import settings @@ -19,13 +20,18 @@ # DB contains a section field which is actually a XLSX header over that column ("element", "Element"), ("definition", "Definition"), - ("fpds_element", "FPDS Element"), + ("fpds_element", "FPDS Data Dictionary Element"), + + # "USA Spending Downloads" ("award_file", "Award File"), ("award_element", "Award Element"), ("subaward_file", "Subaward File"), ("subaward_element", "Subaward Element"), ("account_file", "Account File"), ("account_element", "Account Element"), + + # "Legacy USA Spending" + ("legacy_award_file", "Award File"), ("legacy_award_element", "Award Element"), ("legacy_subaward_element", "Subaward Element"), ] @@ -72,7 +78,6 @@ def extract_data_from_source_file(path: str = None) -> dict: sheet = wb["Public"] last_column = get_column_letter(sheet.max_column) cell_range = "A2:{}2".format(last_column) - # print(f"cell range {cell_range}") headers = [{"column": cell.column, "header": cell.value} for cell in sheet[cell_range][0]] sections = [] @@ -120,6 +125,6 @@ def load_xlsx_data_to_model(rosetta_object: dict): "headers": list({"display": pretty, "raw": raw} for raw, pretty in DB_TO_XLSX_MAPPING.items()), "rows": list(row for row in rosetta_object["data"].values()), } - # print(json.dumps(json_doc)) + print(json.dumps(json_doc)) rosetta = Rosetta(document_name="api_response", document=json_doc) rosetta.save() From 03424e4b47d3cbb12d24d02fa9f827789671c49f Mon Sep 17 00:00:00 2001 From: Justin Le Date: Wed, 19 Dec 2018 15:29:53 -0500 Subject: [PATCH 15/52] Updated tests (incl. test file), removed debugging lines --- .../management/commands/load_rosetta.py | 3 +-- .../tests/data/20181219rosetta-test-file.xlsx | Bin 0 -> 11059 bytes .../tests/integration/test_load_rosetta.py | 12 +++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 usaspending_api/references/tests/data/20181219rosetta-test-file.xlsx diff --git a/usaspending_api/references/management/commands/load_rosetta.py b/usaspending_api/references/management/commands/load_rosetta.py index 04b4c0514b..f2ce214205 100644 --- a/usaspending_api/references/management/commands/load_rosetta.py +++ b/usaspending_api/references/management/commands/load_rosetta.py @@ -1,7 +1,6 @@ import logging import os.path import boto3 -import json from collections import OrderedDict from django.conf import settings @@ -125,6 +124,6 @@ def load_xlsx_data_to_model(rosetta_object: dict): "headers": list({"display": pretty, "raw": raw} for raw, pretty in DB_TO_XLSX_MAPPING.items()), "rows": list(row for row in rosetta_object["data"].values()), } - print(json.dumps(json_doc)) + # print(json.dumps(json_doc)) rosetta = Rosetta(document_name="api_response", document=json_doc) rosetta.save() diff --git a/usaspending_api/references/tests/data/20181219rosetta-test-file.xlsx b/usaspending_api/references/tests/data/20181219rosetta-test-file.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..16a958e3f31d1dc2bdfd418be4877009d74e62c3 GIT binary patch literal 11059 zcmeHNby!@ zVA3n`w{?E#E;QCjwY0oX4Y`E~XW&?IR$va#d$Ar5T2$KE)&>T5Hz}ir1@yM5=|AFR z?&{g2YQ@8ka?;j1OvipC_ypg;;H_B_5bs#mq0A<_A|zc*V1a|n@4uGvrAHlOsk;HS zP&=QicdmU|UVRbcS%1MeWNj2{!pf4)OaF9@NeVWM+-n}!O+QDGOjCuwTi4sOmG)iOnc__&2abgS zlTNB&09oMJ44`NoR*hT2YB%iDDS(qD`}yb->B2453Kn7K(j$|`)QCtE`G@4TjLSJa zsiF#8ob-XddA)u0&)O9BjS0Q&az_!?Va2ZQu`e*T8&BJWA$@IRls<3vmwUH)y!J*m zbY31D$8P(v-oXRF{XGnz_&2kxQDdPvgX)?r)ap>6W@+GLYU9lO_{Zme&GA3j2LE#F z#Zht!9W1Cphmzmk^j=ObL_&mRUIX6?TeD2xEn`T1bS+xSaHYVl}-$|$@9sJ1PN&mTDRs1`l9;$ z44J+q3i0t{(F(}$V+{f%tQ^81T)u<=?QS`(MWd@y(4?@+eo0Vy9cTK^hv6i@>Db)u z2Li!dvU_6*IK9qB=Ch^Vy*6a$*92-R7F?F)MyXERjR&;7Kk*^WP_cyw|ibrkg;^#z8J-3p4qi+Aca` zUp3tvHQ7_&v!zG$je43>GR!dcK^}*LJ#LrmXsa+-=j{ynp_s6qST=_|aQ1_zgD{9j z`z6GI5^z|Udpo;D`zUCM&R7lnQN4_**v~evXsPhuWaW~Jo;8dGu~`kVxUyO~j}Y}k zb>?&Ry8q29mA#U{>bg48L^`vL`v#G@3;47^50cSSd?YE}2Ohxa*9kI)t9G#q#FLNT z%;Y?VtlP70P&WvRWQYsd`MiT%NZ<5IkoeI>Id5bpG^rh~xCr0&)3wf>Fn&HeG+XcM znOuU-^gmT54{N7+4yrEcP_K&)<$$WpUzH_WZOQ%%3&c0A`kt)H{eir)q@`%Swy%Zc z!mFtoA5rQjokhmxwJ@q`_m?SH!Wkc3Y+-AM^Stj44tVnX8U`Dxa;2Zr8Lijxtt-Co zjSf)@Kk!wUCw8hPBd5|uCdi^G9Nk#njk^;s?0Fp>fd(KFWevoWf6}DE9}Kbk9Eg0x znA0BuQYk5aj7?QAM9a?A6R`FA8I5c@0h#h)d*o*-nXi)6rt+@~gs?X{0iTo#VNMH= z6XN>h{GLh&tOPuu!Z0}@4;h=w15-{zn7(ouaxrzttpnEzxobfX8A*mM{gCApOIxrp zRt+MU&Q@5x;70*sujM{6y0m|T+Yq9%SS*5Q;k&ss~6jJ1sBTtQ3^Hh1oujEC% zuM`R%U?s0&=&7KU_i0;eQ~yCXv{|DPO!2eb$_d3gvdX_$YmROugD~2 zGsZ$$h?cA0Do4cGEL4s2;MW-@IJAiylk3k{hVe|Fr?Cg=%d`fat$AQJ( zJI|C2XM@g*T`9-QDY`eH#!3kU5h&x8+{{f*N&U>O}2NhhZheV z_rIB0cr$Y-dyOwVZ5%C=uE?$OF(i*kqL0oqTd3gWh*=BBp$=0TISu^2_@ZAOzx@2A z6;@+=W#}kIlG30VIKui31;>@YylWb`#L7){5~j+jT?>p?*KBQT=ChvPo0^eZV4zm& z!IUFIZDHkwAc$5>G~xYOY%x_Ke*X=Ig@q7X4O0xncdkvu0$)bOs+H72G3~CzbrLyq zN{->{8jL2S|M#<1*Tn*W2fy2avx|p~sq+t) zIiYD_zxoW~C$QZjvVO^iBuAV6N{viZ+tS`u*=wvhu@huX#N#CU=)z^`d$&C43bICQ zj@KMN_HbwFwa1;u-jRqN%Wg<-C^9kGVYe9WNH)v3xQTI{a9ZQ4jv{mzF-L-NuK)tgN?Sr4f&9j zmJyW^)llj69r$%_@?lG9{ZB+Z#t3;Sl#$*El*|tE?d^-!9vPHN@7F`DE;B6SQA+6< za2}PCufr(X2KM1SPW8wX2zWE6HC}#W#rGVZ5S?qB1xHp=CSbSCDX7CyC=uUz;H$DS zA5#yU>0A)s2l&-M|zilcD4|DC!HUs zNI;Q2rTE1yVN$ySMO=b`Ys7JrTktg3aOl<=aZtzpc#h-{9;YOS4b)k0n)_Djqz zUu_p-CS65I(AWIn2DF4Cg(D$fEt*&?xXQTh8(H3Uty&|JZVo-|e7^ZDe%_z=d)nAd zwnQskonI8~Zo+YTCvg9^Xw7wCro&Is-0UfOa`8|xsr z8ljnZJsoED5;L|XSo~|1Z@nR(&F2TEp(hjz2)zS4rD;#5MRQ2VWb>+~!M=Tx?>J@o z4g|#WAKiP?qgvup2c^$Ef1SJCWkKs@Z2|DVxJJ9lWlIm-=GessiLOJm^970g`pEM3-e_L z*xQ){O%Xo44;wE7w!ZY0?9SdnBKAg)ILbe?J?a$kt3`}Tf5TUQ)ZXdJTjt?8y8JRE z6YhERp?A%xVj7A(_(XtXWsi5oW!Pxk@0U(xJC~~73>sKf9{o5=_$|1)SeV+HGXM7Z zqJfl-rxB;GLNFlH za#SW@(UbNIyrjSWSe|22rd+^B;Wz8j+S;l%H1IcgL^e#9Yu|<)f*L*dh5ARxFrrS1 zaa5T#z@n9r5^P~Q?J2&8W1jxmpR``#4@)w{L-2LyyziTf+(oD;sSq#BonYz$MGL%* zO~r!B!#7M*KcFr?XppQ3l)xwx1(SdlqDzX^>EFg+k24D5%LnZ3uS5pHp0oEEI!k~2 zy!=E(nfaA0p~YC|?DO)`{6Y*Tavs@qkpXScQOK228m}VANR!@u6q$Y#tJp?gul-RM zb#Ck))BbLPLPgmOM+?;> zH?~VV^w4<}#vh)w*dl0xl17A382g0-Hw4=n-u>nB*rO)w{?}B4D1C|}x;265&ZziL z@-BC-FUwK|#Gr33->bc=6@{J%Vj_WWqhn4S2g{6)?Y*z}&Ha48Ke<`k=42-ETj}}C zdN)nNf4#9a{9Uj&y+M?@#rtHgiiWx6=Irv2*lSq_jNX65FBc+O*NgFGi|(r#>YB7X z{Hiz44N~q9S~6gZzFjI!xLkr{#;==H-`&KJ?tbMz-E&E#g^u||eWK%9Jc_$bZ%8W% zr3_?ep-+W!*`u{*{TSL@Vu}%!hKe3iF;J9;BhiBP|W}odhFbOV2joTaM zYv|VnE*oHz>7PMgM-;}+;w$w&(mOB<_JtQ%wkc>j zR9>yWJrj0#)BR5J6d1Md&dmQJ|6R`VqS0u8R(0(~m%AsoZTslDOh^hAl{@}{fu?)i z@v}J!$Zih!xbz>7_+_1Lo794Kkx`%;HQ4i6z+FwAO87KVGdwRc3IrP@O$9 zL3t~mLj~WvKA(Vbs2E8PF>n#Cc-|jF?=xBzk{RWgTb=Xi1XJSq>qbfHXXLLVsu#of z3khiCj7e<<6>z3VcX<}w4W6P5~4{blP5!r3&k9l~J6KSc-%g1J9 zsgD$wO~GKVe#Ak2Ui`Yu5=}p@@wkIAs3}7c>Cr_VbrsTd1834*dGRN?*EP3+pX7;p zU#re(Qgg5k6p_Pfnguyf+*=V1cq$>!NczFhO-+$KyyKv2dsk}R=O8Jp=H_PEHLvtg zP)Yh)6T>l5RKlXI+lb`lt!sWDKSqoL&vWRRkbe=It?g+)(~#I$40Ac^P^ob&O+9TL zw^&|+%?fpdIcq!(LkR}D%W1z|G*2r%^#-$KjMKyWVb#$sBwf5>u`e)W^XtgrE-kg- z$Eu$)kT;MXc!3``$nN`F1ng?MKM;bu?AvT5eRI<~$R*Q}%`WR>mDVk(_$)$|9C7w# zrAg)?w$bG3UL~cLP_L+}lx~7f1;Tn{c}r=)1D{$1+5kn)$!uB}_!q2q^IPOE;VbN% z?g~;O2Gqf(zp3;Wfj;N5X!h_hCL>}gw~`VTYVpputkgG zm%I-+jR=jLC80SB__B4YsDtrs%5aVWR>`N2gIE$zQim8}rG z!_tpEpRWcDD*rX9(8t)Pxyxnkl4~ z-d|Iy#UyvLRe=Nv2?DD!>#Gt%MRIf2;L^Gkd_=1E{iIR1IyH+e?89Vlmduj1Sdk@b zm z@3Fu)i|gao-ex$Z)C+33OXRPvawT|Ol{w}#YPNJ(8~6OI8!l zi#!C2aMadz94&)EPXJ!b|q+Be!y0=YjD59>ZF9* zNZFzgWj{1qqJCPl;94K5z0_LDjw7BrB%E4&nCZd)4p)0Sx0J&l za=e-&rAoz4*{=DtalSJ)u8A*uA3Tbaua&R5kI9N&NvUt#_l#`Xn)J6vR{I`Xd(gp1 zKofS~UG+^#qGT;u?DWX;y*NEV4m}^ds%&cvEz;`w8-k}-j$WTTgv`idisVzuMRe9YR&C6>CRty16yOp35`3@q&EM$^wI_2`xse9Y>X?Uv-r^kD5wV zXy|ur@NmvG*=HN}Qk9;}s7|cqlBdjS>?@9D6YnO{+|Dehju@-0oTYmvA$t}xBy&bI z;^O&?8>@X7F5;S*m{@b;A4-snQ#PGA&n_%Xa3Hr(r3p<<=A=?J4dy8*+@n4y*es;X zu#L4SSLhMMiyACuX7=+O-hR^9$x2+~WfHqGzWZ z+B+N3Y(<90^5sJ|Z>{z0vo6G~)r#F|)VFHYUx&XC^YL_0>z!w!l2uv8y&Xr9CLH!TrKfq_1TzEN8C0?7M>BC3;h^YN}f~x zhMUq%J-Y!LS3lm4-ZDaxf9c)OCC`@k`gVD>TdvpoX^r2c?^KFqV}-kc_F0(ek{UiI z3bkp#yQ&MJTYi4FqIotHehw6Vb`-lLxU|pUy70;U6W=BS4rd2RK{@^JC62?g#Fy}6^>IdJ#1$G5Ger`6 zlN1*_7+&YW&7Ph5d`4xdZTfrx&7NSdbJ&X=q`gV3nIf8U4F$tL2=j9f*P8Q;%TnJp zdvagwY|j)yJH(d;mQxsY8VWu7d_-R7F8h!TKUr+*W)bQJJDNIm_0P7-YWNw-bUUlI442YXPA9mLl?D8rd zosi8oRR$me0kBSpbx;x_nCoipV;VrjWi|g>v-o?{yU>^hUh9$*8^rwgTtBo+v%^7&XSipgokzto{5CKMrza}YA zDF~Pa03%>fJ{;H(1$G%15qJ#o*ChoS1p&7J;0O%*3PnH(c!&TC#NUt<_#_Af1pp8j z5G_2|5EXX$AtJyE@i!p_`UQd50DuJsWC0Ieh7$14Ha#v3uwjo-QK8?>%={~0S~`)J z@gKH5W$`yR1cuvy)Bq4K42(NG{BJxn3z~yBCvDZjr9|pF#P+^9mqG5jZG& zuns-o4)4aA^_!v!neN*YC7?ftSz+2Iaab=ezv|orr&P{~_@557Cf_P(fY> z@JTsh+c-HSB218%8GPJ^*oj2}i3kUNfrJj_?*sTl{w2w&fpk(e2d>|IV4Cc{l;16EJS+5vzVj*e4yCBBA3>ugii4sJ8aY3i%~U!ihnUcSho- ze1aBTQ<}*;X*^G>J@3I)1Ac>C3O--1OsDG>QC&S=R7v9rmR#ZcXU@1=SKLPWc%vrk zLsO^tFF$kp#lO4EpBl&1oU>4P;{R^`to@mdyoyuR02>Lvfq`BiC9h*&O1*la`lm4@6vLX-a17Q+ zbXM)@rfxnl4q_f9D_YaY)l6;f?=0sbfH+m4VIEg{i6;YP?qSV#MO*+QZdk zHRiONo&$IKB=Hf2GiBEW!QP`0m0yzNlk%F2BHTNl;PUZqGDHswW8RuH@=Je_Wk)CZ21-GM`U}WkRq(lk z!+$jps(SyrlA{z9)>t52hYb5Kg1z(Pn-ELrdk}FT2#^@Gmw`Pq((Ox6F$^f1fgEqI7wJZN%Lt2H)O2_F9fCb*j$XPb|np*wPzSF&HMjexlW{ zz%=v$rISCt4lJXn+{8eRs7}Y!@~i0pPBmE_k@?)W;c6R?V_DD4odV5NJsB!V1D&E?iB` zg%u3qnH@Q+v5$=M4cUHR4G2?dv7p{M6%$P#=qh-D@bW$l83HR^bgN(9BX}3bQjZD* z=NYf2K3^tsh;7o4zd$hXGv(h-x@<~VuaQ!8n(wiRfN|}qpjt{` zw8G+0^tJMR+{o!i%wLIoa=b z@bl{BX9<4~qJQE6K#u|d{t;4thX36G{uRDV^%wZR+@Ycj9CUU802%u7hequb+8@3D E0iZ@Gv;Y7A literal 0 HcmV?d00001 diff --git a/usaspending_api/references/tests/integration/test_load_rosetta.py b/usaspending_api/references/tests/integration/test_load_rosetta.py index 7aadd12450..8967d437b7 100644 --- a/usaspending_api/references/tests/integration/test_load_rosetta.py +++ b/usaspending_api/references/tests/integration/test_load_rosetta.py @@ -13,7 +13,7 @@ @pytest.mark.django_db def test_rosetta_fresh_load(): - test_file_path = os.path.abspath("usaspending_api/references/tests/data/data_transparency_crosswalk_test_file.xlsx") + test_file_path = os.path.abspath("usaspending_api/references/tests/data/20181219rosetta-test-file.xlsx") all_rows = Rosetta.objects.count() assert all_rows == 0, "Table is not empty before testing the loader script. Results will be unexpected" @@ -34,6 +34,7 @@ def test_rosetta_fresh_load(): None, None, None, + "Contracts", "is1862landgrantcollege", None, ] @@ -41,26 +42,27 @@ def test_rosetta_fresh_load(): "headers": [ {"raw": "element", "display": "Element"}, {"raw": "definition", "display": "Definition"}, - {"raw": "fpds_element", "display": "FPDS Element"}, + {"raw": "fpds_element", "display": "FPDS Data Dictionary Element"}, {"raw": "award_file", "display": "Award File"}, {"raw": "award_element", "display": "Award Element"}, {"raw": "subaward_file", "display": "Subaward File"}, {"raw": "subaward_element", "display": "Subaward Element"}, {"raw": "account_file", "display": "Account File"}, {"raw": "account_element", "display": "Account Element"}, + {"raw": "legacy_award_file", "display": "Award File"}, {"raw": "legacy_award_element", "display": "Award Element"}, {"raw": "legacy_subaward_element", "display": "Subaward Element"}, ], "metadata": { "total_rows": 1, - "total_size": "94.56KB", - "total_columns": 11, + "total_size": "10.80KB", + "total_columns": 12, "download_location": None, }, "sections": [ {"colspan": 3, "section": "Schema Data Label & Description"}, {"colspan": 6, "section": "USA Spending Downloads"}, - {"colspan": 2, "section": "Legacy USA Spending"}, + {"colspan": 3, "section": "Legacy USA Spending"}, ], } From 1362799b522e4c6dd564c26cee9b8a7fd35201fa Mon Sep 17 00:00:00 2001 From: Justin Le Date: Wed, 19 Dec 2018 15:55:18 -0500 Subject: [PATCH 16/52] forgot to include API doc updates, also merged updated dev --- .../references/data_dictionary.md | 135 ++++++------------ 1 file changed, 41 insertions(+), 94 deletions(-) diff --git a/usaspending_api/api_docs/api_documentation/references/data_dictionary.md b/usaspending_api/api_docs/api_documentation/references/data_dictionary.md index f69b4997a6..f26ea9d458 100644 --- a/usaspending_api/api_docs/api_documentation/references/data_dictionary.md +++ b/usaspending_api/api_docs/api_documentation/references/data_dictionary.md @@ -12,101 +12,48 @@ This route takes no parameters and returns a JSON structure of the Schema team's ``` { - "rows": [ - [ - "Interstate Entity", - "https://www.sam.gov", - "Interstate Entity", - "all_contracts_prime_awards_1.csv,\nall_contracts_prime_transactions_1.csv", - "interstate_entity", - null, - null, - null, - null, - "isinterstateentity", - null - ], - [ - "Joint Venture Economically Disadvantaged Women Owned Small Business", - "https://www.sam.gov OR List characteristic of the contractor such as whether the selected contractor is an Economically Disadvantaged Woman Owned Small Business or not. It can be derived from the SAM data element, 'Business Types'.", - "Joint Venture Economically Disadvantaged Women Owned Small Business", - "all_contracts_prime_awards_1.csv,\nall_contracts_prime_transactions_1.csv", - "joint_venture_economic_disadvantaged_women_owned_small_bus", - null, - null, - null, - null, - "isjointventureecondisadvwomenownedsmallbusiness", - null - ] - ], - "headers": [ - { - "raw": "element", - "display": "Element" - }, - { - "raw": "definition", - "display": "Definition" - }, - { - "raw": "fpds_element", - "display": "FPDS Element" - }, - { - "raw": "award_file", - "display": "Award File" - }, - { - "raw": "award_element", - "display": "Award Element" - }, - { - "raw": "subaward_file", - "display": "Subaward File" - }, - { - "raw": "subaward_element", - "display": "Subaward Element" - }, - { - "raw": "account_file", - "display": "Account File" - }, - { - "raw": "account_element", - "display": "Account Element" - }, - { - "raw": "legacy_award_element", - "display": "Award Element" - }, - { - "raw": "legacy_subaward_element", - "display": "Subaward Element" + "rows": [ + [ + "1862 Land Grant College", + "https://www.sam.gov", + "1862 Land Grant College", + "all_contracts_prime_awards_1.csv,\nall_contracts_prime_transactions_1.csv", + "1862_land_grant_college", + None, + None, + None, + None, + "Contracts", + "is1862landgrantcollege", + None, + ] + ], + "headers": [ + {"raw": "element", "display": "Element"}, + {"raw": "definition", "display": "Definition"}, + {"raw": "fpds_element", "display": "FPDS Data Dictionary Element"}, + {"raw": "award_file", "display": "Award File"}, + {"raw": "award_element", "display": "Award Element"}, + {"raw": "subaward_file", "display": "Subaward File"}, + {"raw": "subaward_element", "display": "Subaward Element"}, + {"raw": "account_file", "display": "Account File"}, + {"raw": "account_element", "display": "Account Element"}, + {"raw": "legacy_award_file", "display": "Award File"}, + {"raw": "legacy_award_element", "display": "Award Element"}, + {"raw": "legacy_subaward_element", "display": "Subaward Element"}, + ], + "metadata": { + "total_rows": 1, + "total_size": "10.80KB", + "total_columns": 12, + "download_location": None, + }, + "sections": [ + {"colspan": 3, "section": "Schema Data Label & Description"}, + {"colspan": 6, "section": "USA Spending Downloads"}, + {"colspan": 3, "section": "Legacy USA Spending"}, + ], } - ], - "metadata": { - "download_location": "http://files.usaspending.gov/docs/DATA+Transparency+Crosswalk.xlsx", - "total_rows": 393, - "total_size": "116.52KB", - "total_columns": 11 - }, - "sections": [ - { - "colspan": 3, - "section": "Schema Data Label & Description" - }, - { - "colspan": 6, - "section": "USA Spending Downloads" - }, - { - "colspan": 2, - "section": "Legacy USA Spending" - } - ] -} ``` From 5b45c68dc35f16b3902fdd688597d2777b0d997d Mon Sep 17 00:00:00 2001 From: Justin Le Date: Wed, 19 Dec 2018 15:58:40 -0500 Subject: [PATCH 17/52] Removed outdated test file --- .../data_transparency_crosswalk_test_file.xlsx | Bin 96830 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 usaspending_api/references/tests/data/data_transparency_crosswalk_test_file.xlsx diff --git a/usaspending_api/references/tests/data/data_transparency_crosswalk_test_file.xlsx b/usaspending_api/references/tests/data/data_transparency_crosswalk_test_file.xlsx deleted file mode 100644 index d6cbbf81d7d887ff0154da520fe9280f93fe67b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96830 zcmeFXQ@3bAkgd6G+qP}n_CDLTZQDBAwr$(CZQFgS?CN7elWb@L(DL&nQmV}3Cs zV}*h=FbE0&7ytwS001GtENl9nKOg`=Hy8i_G5`dSwy>S8vx%*J%^3tB&WcPUqr0|z9&FVA3PFko$2$Xsl?~|n49L6e`ulnUUx`<$h zF>uclyI`3e?3OX=NQnt|`>HCz{b?C~n*5(luBq4P7I{rc5mW*jhPsO2OwM@0Lr4TF zTv0}9kR~l#USEf%M-FE#FiL)UD13>9Ze%#CW&;k7@;CzMJ-$mUwsRrwUCZF=45ELUgJGxP5AtdrK)VsX;EjGT9^UuTn zDcM` zFmfAstKisb(RoSLaIPz&aaVK+9PIiL)*?9`-QnXE7T5wU-Ic5hv>WXGIO$FDIoNI$ zL~pO5T_}PhMg)+w-o;jogkPH0DUL{Bk#xC&rC;#{n>kc;h2hwe-{nz5G{qc9Fh2jDx z$L3`{AfK9+L~TRxM=rKD%(_YkvS>(@)7X%(cF;OC_`Rj3AIhIBz88s!>ekwu0X|AD zG&Hp^Ta3w4@6-5LlZ5ZDyUE`*@J4N<&^OBdIdICrTUbuW4@X=g)t=w!;ON6gO^K5K9`{82a8{6cDrt}sbpH%vc!PNy~e-dQRL{`(sKlsQUC^~gwe8*|E_MKss}F@K~AFl(q;hF z;5H|O0iR`wJ^LeHCMlWkKI_155k1SQWX&oDVGBg<1{g7N(Lid6g3472CGC2NmZA0t zo6-2;ZaNBurtb#{5$N@D+NTk46Ynj(UyI*; z3~u}c8ad)m8lp|57e53;pG*5g2r1b$r{Jg^{)sQRYc7#a?=qxA;z(%qUhGUl$uYi0 zBJU|;`+n>eL|iB@FVX4BT1pJMW~w^Tgdrw;>FrobqO`*8n6Z@-T_Mq=$Tmq2YJMzdvQ~{{Q>7`+s7svqp#bm7ChKAt>xqohMdAFXPH`uAc)etJ{#OMcXflF&SV+*? zn3slGwm-(Fjfjo1rs1j_rC8Rvk(9mFETdnyLt@+<-;KXA%j@Ik#})|LKQ36#_2M|L ze$OWq4mOY3;h(%e?&EmMV@FW&FS|ed!0FS~IL6V%5EoZRt>@2YM}8h9=b3oDop|!T zKzTJ-Cosr}@+kFb3%`V^l$M60M(O&v`e6z9q^V?K3t-E;`9E&|E!l~=yS}XdOd|>w z0086vBiqT`#KhT&{y!fq|KS~3>YKI~94J2c7C(d&+#uwHpddJ?3r}p!Tp2UR>mb02 zNq-$+Y-Z=*Uo~5V{YjV$BxvC>yVF*C8ZiAlW|gdFMH2434U=qVTQu+w#?2Eq!3;>~`u8eTks9MiJEM;m{2@}6Ab*C**3rP5XnH{C z3J+XbuZMUxfeaIhBEIJGT2h7!M_pz>b&Bpjv)@=+c0hmre0hagHFY^Crw#?8dy{qw zxWb&4d}Qbv1x!%U)?SdA?YiajiT*st@c$ zCYebu3`0w6(qP^Yv^>W=UXhkie#)ZiLutYnS4X<`zgIH(1slS@T3gIi#ih|I@z?J+ zA8qQbNzG_NRcbSo*2}zHQ7@6b1WJ!HKeyXLL(WUx+0m>{u>cNh801i6BLI7{2jBB= zQ1c9smOh+6z3=H2X`fZrp*86{<@{vR9E))pYfHBjbgJNm1l{yGdEq&HE{F9#wt}Zt zvERiCR6x5neL-j<76u^%)l_l}@eGH!H#dmL0=W_a*9b_! z6-iM52bxV3}FScFA`JY_fp<3t8mQI}uk1 z-0R92mxT;`^d$Rxwt<_q$}9+Jg^(pqcH~s9ifSDx(`>4_%$_R;Uv7yCzZw>TQ;`C) zv!_9SovtQp#U<8iD?yqa3+mU%Fw|er2hgWZgh^>z@yO$Dj#Ydy{A(`yln2zVN>L|V zVxX%pcU@|W@H9H3tD|E%Ema+0%WcALCo!NDD3eVDJCksO<*F#`xEs&ILJ|i%o|Nzw zF5u!f9doKo3~o?m7?G7Vj6~=_$ehm-qzL}ifkEvIP*6*4DoEQ$OVvC!t*sJYzR#(k zVVVD3*(q_|(8-pa>6BF_TRL*q^KU!N7X`1fipJY)w#>?rs$SG0Wz9Zi%O);uEHYYR z$13|h2UDVupyO(XYP)T3erlmW%*~>nrWb_LPX6$%9@Pr4lv2c2prw69KjZn6_vjul zqfCA73H-|+KVGx$I8|1}JzhwXB9pwC&8>v0L>rMKZyV&na4FIXTzmL>>Xt1SoaVpx zP4iS>!0?tIsyi~2QQllpXV|0s86BpnDOtIV=yb2LIVQ|B&}BK1TW4k2?i8#Zb&&Bb zHOw^*O+5@&rdEnQ1!s^-{Rp;Mh$msH$kGVO?};JSsH?4(HE8iD<`bd9Us&?{@2d7+ zi-<~q1@6j!C)NM1dH;u0!}Oo2M%BhHlL6s}&*HcL;dMwrDU~SiBC0}lqsX#07uN}- z(m(yW%VxuGcb%n3mZ_Xp3a9Qi$Lr>%Jzm#3BcnKbVzx-4(sw2I68zDT8G9}WLGFkKFq zzDrrTkpjYS#eRJOk8qy8U%`YDbc>?iI@tce)?;E|=2W{&6gS9~3t{7ppC;h%DFH~TOHAdxe7Xyec3!IFX|%KC~= zkmByp4ly2?cxY;M`#4j~Y-%gj!|M6nJ=ahFF~v3ih8z8#dqh~j>zD+mtumCtt&+qPqm zisg(vImAAFQu~WR5JB&^H~xLb8mD$=KFn-GY$nS(?n<7>*%{>)M+dx^8ad42e*Gba z9|0iX-i+YKaSAm{0z|NTz%Zjsj-0F#{|>$)28W=-9~!vLy2_wO4>`if!#n>$$-*-h zNSgCXNse4%q)!r`QTr`94BW&s((TH^L+H1uXQ7tRB@RhBKm5A24t4Wif8{gKCHSn~ z>bquuvIPC;kzys+*F6@FF7t=~zDiyRxI$6-CvYIp|D(W}|5M;9+E$wkD8Bet{1Dyb z4cU`O$feG(CLypBHM5c4I8hgdB*!iDzdgjAuG|B8OC$|+;=7`!W0jRjYDWMw(pFFz zvB*+=5ZY3W)Ms(Gt%abZeD=zYCa@?^Fgd!_yS>`D)n0e30tDgu-h!718Ci_=wffBL~wAj0a z7WY^j)7yhRn2?p&!D~{agYBTjB2izqW>tsUAwBZv$z>Oiv+|JQzXkG!oVL|I;m~dpNd!WJ zu@aeAfOmgT0|%O z{bk{>iMlUd9Vl% z_AS+|T|9Pu&|LuL&RL=%15_kKspgpUOe^&Ks3@{{+&z>J5#8c(-TrMcE}vS!*m!TA z$5#pZq(uJWweGcxID%Rq?|x?q?6&Q(UQk@aL3CxKkNj1Q@7C>xVPlUd!JJ(%8RWtU z!aqMK+=8&X{pdHY3+@FByz>v>JdmrnQtW+8Eg79OE!$YVto{v&j9}_3p}q(<@Orh+ zdjD-rrw(9vYMuZ9;F${qfbzd;Iwx}jM-yXZXGaTLGpGO1;Fgb-6ZYu*jhpH@zf8Fj zxeBTsfT)z?RTpStLBZ=-iO@p3w-9XrXggvZ^es?9y8JtK=|vWQnx13MP5L^n^{cLg zbn2tpaD7C!mAt(C?&s$3hX*5j?m?GC!&9eO(Dt_(nH>2#i=xw4-1cH`|nlUF4{PL zug>?q4_k!V{Zf~7@3!{${U-0nvM=XMW1NHT?=y>*=!s*>vBURdTpF=c1n2_T^x0tN z%I)aoySP)vcJU)jWi7i8{;o)p7jmqI-GeC}{eiRU?>p38?5_Cg{g=KT4Q~#qMZg1AFXl zUTK%>H1F@{N!5~U?1lKQQ?I4yvzu^J>^`OJytK2k>|d2TciZRZi{$8OX+VXg?9|?W zX>$4cT{GiUdm&Au5-#D4)ZBjFk_mVM^94h1odq-A98zQ%r;@q{n!B2?0m(o6r5K#ef@31>BHn~K6>UlZc zBhvk4wI566*}~ycT^!%hdqm+cpWogw%+a|p{Nuf=H&aoYEObOf`2?3DiSpG=k#4h$ z&#B_b0!>|6*nKxXnmN2l$7W|HiJjOy-3-K%P`?Z3Hq5^s%z2^`y`zig_@y%4+(phb zsiwAVWJuQbe(ws_pRqT`j~wcA7B^WZ8$%+Lm^T>d<6dSM*k<|1(I(wN1Ai&mL|bOa zpBQK#t&z}&Yus_b44M6^e+QPFWQePre0^GS`O*(8`Yw5Jqv8tOUVA)X-gAx=wbpoZ zeK=56XftL<@mN6B(r64j{~GeFWuxTQNmN5B^v7Lby=X9yWkiafhIJ<_sPeA|1-Pe* zLoNJWZ2NmOo)|y)N0gm6)wJY{Nq4+uU*XNVrRH{8$Jwp{(+D+SDO#;BuaP5%Z`5vn zVu+vqbgux)wm!TtSz*+L009BZN2;W+o8*}7f@~(yoex!7{mxIu>K547^0cqO=2?O0 zKKNbWdpY)M)S;s1oS$iyqnCZntV7BavW8e^keVTDn5%ab_ppBY`1=5h{%d9V?4dn< zk9u1YfN?S@lO*O12GHrki>afHW!R@a620?;Nqf(ej)BJWz}3i!Y{CebD+kHEaZARw z@CodfQ|HtfC>5%z|6tyLq;g;?NwX`alYh9i$ljX{B3LJV`!C>!nOle;@Le?ljT18| zUFtX(Z2ENRG13$j^D5(rEWe@dSN%Fm%x%Ro6pK3=5JK`R7MY@U5;YD@ss!6NEi zn6KHEGyI~9%vi?gBu#0hJvu5GWhCp5(=5ukY~-)~>%I*bAS0<@XdN%pY6NO-K_<@$ zdMNi{@>xW}CqAy8ZRR524<^vy(pU;aoq}hu<91Kz`Qr(A7GRO%>rmPB z=aq$@95t>F`{!3A?B`Q|J3I8)wo6kUD7~v&!)xRlaH4mZ8qtGvAz#Hh$uZoho1qX7 zQ15yvy4`NY4A2mFED;4#Yv{W<-oT^sO95}Uf&Q2@WF4B3L4@?* zh1TREokUrV3jjjq26#SIe9 z8F@eK$GDT1{%W98dp1Q@UX3ozC%Rw!MKBjwVQ;N$q&X!Q!z8Ex!>Dbz@4zzTNf-*$ zTT^--OiS?SUb6&{ZTd5!C}-T~e@gg$IC~O}5&3uyBTUw8K z0}uBrAa9#9#H?I}9tFWe`xv~n0uQZf=-T@EG`>hqJ{27ZIn!rGR|tY*DJd4gN|x#d zG|$j#C@7K)mTZ)YGUT{pln|>ZI>=4LZP5ah1^|Q(fiy9uk8jSNqzX2jiJ9yOF%2?8 zh@oc>p~L{4F7BY2F?+nh^!L!ptIa50gi9v?>$=MV`|{(|1;myzSDNpwzjC4OY|mUn z@9N&u6S+2)a8le#saVpUCeay&w6m?UDmpJMWQGSJj);fjw=dzadP|T$Jm$CQAPyO~ z^d2dbX1yDG4VVDkWX_SqU8f_2dSLjeWR3N-QB@|8TmN7P$UX)S_%Ab2TTvwB0=_j0094uWLH;c+l!)rJu=k>865aV(lq(&jP$vXb0hgC(I#R@ zbBqR5E%X=H3eyF8Q82Kb8p&J;_kxT!vUE}+mQCiNjT6xfEeTe!xsnvz8)@f{IE=}D z@DwRYqxIPIQ1i@4P!RCfC>7PnBh6=qPk)i!ZoRi4EbBn(08QC6GTRcBA(l5=s~hAXIHDMhgU@ z`r2=De+pfGQ!m;iU3oL5t3Ub!mU4^Fxia*Iau1wo@y&iXW}a)KiMN2@p1 zxG9ww`*^8EB05jv6Mp7s!F9;vZKD% zM|co@^=Vd!2-9qJ$8ll`djqJ;@hSX>D^hO#FfTylPX&?%>&x3$TZ9<8V|Mt8{)RgM z9D(%nYGmmbzDthjeHq?Ltl7%aBd|8`O6;9Wrm$2s#nfSTp4|e7kM5}~U{_D!o2MZ` zmq=+n**;z;TQGMnher-A(Gda~id$Ah6#68gP0lhqX^H1PTx@y?`~3mcyCc9J_}$Of z4PMCNo>^W`J5x4n)wGv84r1Q2bI{t~^XCy)vh5NDXcMXylb1)Mb$!Ah$YloxtP9zL z+IG+eF%I)Dc^4I>ie07Q=1}GMjT5kA>^wNe_6_~kCR?wu{vv^~ossArS36;Q{4(@V zDg#7NcG_xQtjM0?uEf_D3aLQ5F&tqS9-OKPa#MYW#k9dbE(>A)RD}5Ix>jp7COyWm zAr&>Kkf=PA#7q+CS80hq)|0n2wVYDfFU)#4@bn(O6fZ2vQ#4l_yCq&NfXU;P-RuxG z4~j^5b`X#jbqyJ{wl>NXap^fshP#+ad+3p|&4n}jb88a%+^C4Uo+g6<<(AvmN;@aa zQiG~eWR&>51H4&~tUPAbP2l3Sp0ef=Rc-9YOb{}Et_OHc)LUtDZzDsyR^kXB$qaiD zpA@HZT+85K+VhR~(+Lmzt=v9CDFkq;Um!)c41Wx(MF3X-QJp&axI?~n+OmIRzE``c ztjYP`{zI+PEX;8H1XUpgtr4^$A}f^eeHs?k;8$L9nXdc7?D zYYxZGSjOr?xTrKBuG09^a7#<7@$%1-^M{{-#Od3pIY6KstDMo1B%{%k*YeuYbbP_( zunlJ5yCf5!K;E1rC<~b!zRFJrtJ$=H+ggx-3kek#B@jYjr zgah(i$5Y#7)Q!mMI^y zb6SZaE^cDDPG_Lj!+F_mV>8-D7grC!Te{EAcYQscoLb{^pw15E6m|%MXVec(ZFRoR zRdl_?mKIz%aHUQm2!qx~H;ogmQ?f;!?*766u|{4D)1QU~$~ZtM8&MYta~oZWV%UB_ zaGPfzD*0nAj^`XI>e#-cuoBw4zk9Cx5XLq-D~n9_u7Ok_Q)cJ9lv_PBd8g1{P_uNo zm(H2VjuX!$NqrxlfTh%P$7+5XV!1F@Ky?cJ_I--%TjroIm|!g$6?-TG@6QPzE9(Z~ zX+yKf!EFpQgTFOK>FPVez)tRXOn!^0z2S+p5wYP0jG&V#u0^VL}MLItTf#2(Gy&=yjQg zwOBURB0^se%qdr9To&%GR1#|h_qjB$au+fY0)`l>1HJBz=d#XylXu~#u-gKzm^((r zt6__9_4ME?qs+;s?t<6G|I%zbqy7ORv$CJx#R1Rt@~w9hJ#`46IHf^q zMq;IAEr*vp);Nfm!y)Tv4`{SGI~eLw_<|ZzHi!0$CErWXFb-3UN(pg;+<~0dBT-19 z&=uXr+jsSQIkxT+?uQdDsd{NV93P@Utv$)*BR9X8X~QPe#QoV|b++P!tTh}+*ic)@ zEROJVD4c+!H%7?q6ho`17TJuc(1E^qWxXr*<#EZtq-hUeV~L&%if-~RJp7}=r-=US z@P;5(vzNOhY)tIi?q~wUSxxM^c#QW$mDUN;tWkOpyFQwN7oUIa8V|&%ErHM!_ZE|1 z=HNN)Rrg1wH#Wd}jS$}@T!58uu^lZLWzaVizz7Y~(0OefyoLJ5&Cl|HOtEZS5gEEI zps`w{*cV{nSST@74_tc_xH_B=S$efgg*S}|tJT0@L!hb$BIvdKjlm`it9Y|b2(HyL zeshnktWYaX{uLm+^)Z2?cA6zwS9~-+Lj2sBB{$az!RbEDoG!aS!V!;n%t8i@8Yv~e z8q04%D-p&x$Y45^InKWt3WHpPZ~As0-M}b2wT@ZnLU>IOlVx@=quQ8#r&&!7ZL{*P zC&dY-4=ZsMZe}ja0e`I7{F>nbEe`j``WU6 zu+BG!STY-En|AfjyEV$+-eJm*5hJT-Fm#^Sw(dO|3Zyt3AkMQ8qYGCKuuvZkT&R!v zF4*j>4xvM@X=xO>*1MKN#S7Z}t6*9mylqVdxRMQOXQk!mgc;E&Ka zk-P7EoW_ceqf*XX+W zt5_DFoqgY?yeD<_btqFgQi|&hx3WJ^4p>Xa=wQV<0}kOr%3+^+L=@FqrIfh0HWKl# zb&iZQ%j#(VerN7?0rm5Q>ld1DT6brNzv3K%g(+1sG~rQ`dLM~eglIuvJ^rVaxzu) zNy@};Gf8yDhRXKKljg)yf5@(bypci4Sn^1f=o{b`AF=CmMKDEVqA;7;dCa7XFP^lFSOBb%E5I>W2&btt=vBSI$r~X#iHbBTk_~@Cm{x z{SdAF^}G`r`qqftBP=1KtO$~kRVx~XO~NN4WqW|K1z&Uwg9)p4e>Fva@#fGb&WC_$ z-^4lnTZ|A9KY>)U@^*~NG$5o1hlhw+qxPzV!{a`W01g|X1@BCNE2)<(u$etgJGH{; zRnJY0gLJ*NcaEm=9wO0D4`!w0Nm9@@IP72j$?r}$%;viY{MJG^3+9g>BY;ud<@bW7 zdBqv^09p@>dGKOKq0JDH!&cho8m=W$vF<)G&;or)k8&Jn@>eLY%nA<=!l(cqhj_JW zfYf{{SUc-gm$WFzDAMUY%#^IFtDE zQ90NR2y=)7n54$_S_DnGKK2>duw0E&V0XBFQK=f^>3*)VYuYCTIkDQmj7LKnFe-Peo*`*d*+}F8>? zF0xOx7b9WSeViL_p9Gvud@F55VLnvcfA8QjQyN3T!%OxB@Gz8f3DwYQpD1UwPk;#% z2$tNRP|r>9ZPEsP*b)FzL=qv5raZ1lwCJXlcD17^^&uRU?OL6fw{zaiPz$Gv3p?#a zL}14W^bl;m6a(S+SD}+JaYkkohY~U}zvW+lnzD=#cC?Tc=0z)hx4&}r^IPrHvv<|f zi?wjMXXo}$wS6#Vm+yDMs^xsmtmP-`pyhm>RdX!}SyyDVG-H4k(wc8H=OOxWEyJ$3^hy(Q1o!tnIA*4+J;@)-h};mUOvZ|~1fs^GPmK)%mcrFL5JVYi+) zpbQ{armL%?D`%FK#(H_v#WF%v?%ApSz^un$W(i><^b}4TNhyL3r?g}I?_plS*hh~M zi%Xx2O=XCcto)J+sDLeCjYK5f<&31Zh-I8s>p=eZBb%|Q9X_!+1Wo$=AqW@jMcw{x zIMy3hf95TF6;Qc4m6v`+R6^9FRw#`ridmlO;9mjf^-~_mbP$0v77z95OpC4l&<00f z^{P#g6u$){s6~s|ak8Dx$fZvciDOcvD6$Jj_Bs(8#x?Yfd*m`XSj5ft@<5p3OWA$sAT#83%d#!EsPl{m|L$IKr{<*JO^ZKtKNMA05@Xap96fehVOqjX%Rd#-F z(m*w@fX>R$u@frJ4knMPp6f>x#Pe2K*3V`r$L1=D%YAp1Yo#-Z9u)If8VeOOfPAZa zAU6Px;14BZ9MDk^V#1Jy2yV?6EaDWX<$=5I3)<&I0a36RJ0Myt*+InN;&Djm^**Cq zkFtzFA2RzJ-|Z%#0k<|eq~Sr}PJwi8=5%6qC>%60Z;vq1{iP>T(hY5POR}oBciU zsh@-br^QWH5dm<~uw93>-qck|s9K>iPiPC_66Lh=)B5 z8-b%cBRLZFs9WHZCOQK@Q5$?mPI<~m#zhpjbNx5_PA`2B_UH{StT6nc$J)p+t&Dh6 zD{U*SDgfc>+((^4EpCsUh1Y}M-`76r^)0@6sNqEYb1lu+4u}@!qoAw(es%OyPA(E* zr(b!}t&V0tn;p*BoAY{o8=wbCgoR3eG~$dUaMVlsdK!n2cNDcyU{A>dLgZYI4GVGV@r-;{MgeO89f(UC3nHiqA+x49X$s z$Dx2Jve#ugh6>(~ee$aQj`h_+EqyA3M~5Fg>5VE(7sro}m#~vOK9hztm6+k_UIF+S3wx*((%cLszD&880GdFqy!=<$}4J zu!3FAc<*;FUr;`i78W2WU^n)i<+3PBj@ZPdHk6Xy=P90Ch(wG7=N2}&Ka=3GQuxT+ zICyX?D~2jUd0pINbJi1>LeNfX0$Yqk(k49C-Qt7;m?D{xcEQ*&%N5+Dyrlhs;|o-x zoPi4$^f!13C=NrhyO4(_u0m>^D&&fzgipP>=e>4P<7p7&gh9n|%^_I_M&#Pu$LanL z9IJRhoP{efUUgr3O7EE~aj-MRb(Z&S*c%$nj#ab%S7({Xqq!4Uu`Ur`)pd zghSJz`Im zN+veGas&|UgA4G><=+JeJdi9xAi)UyM;hZ@cV@rxpT>ddFFJg58`s?G%tP&xTn;~0 zP}h}%lz0J8HtKlV^fT?_kT7@RIanwGfqZkeBKm^3yYU*KwG+st2osl;x$oQw{58io zHYC)~0}7B_`c&_ApJoSqfq%K`Y8QuX7|vVe3w4Bpipg5b4FZOw7Di3Y6Z;@k-k=>F zDodf&I{Wk%$rwb~J7?M5v`lTQNj}64z)mb=^}9yP$T=fnN$i#)UrvIa-Y9xTVv8e1qqAD za(p%?_++?rR36w4uQ;5CrPZ;DT82Jycvk+V^g-(Kpnctij#zSTH8VCF9)1}F+~Bb*-P zg(o-eB?!&3X!KHfVJ9iF7MOt!QAaKE69!TjF|u8xFb4YB(`MT;{D$e}dpv%zS4pJn z%0P3&U7wL@45NvHIAj)D&LF;vjnXFv*WpYR)cYP`^Vp6)PAgR}%*BD8b~DV^q!`yP z!e6!onD%;4Y1bS$4UkMXCh!JZn#^r%JEPbueRB9>Z{6cb6)zh|2o#$XIZB$WU58YH zrnGhL$S(#@H%ANH+J7KX4a}8T3Z%=w$Q^-Svco1(`FfsYmm*^jTS3x7g5e`k;PW#~)bLt1NBTw9mF? zn^Mp9<&H8<46DY*h)TQYzkR&Jv5B&QjkJvw6;~P7(HCfToi}cc0>AWKmPQ2^*K(50INi0;vl;tU8quGbg1J=l&bU;q&Rf$xrC)HM%1mQTGlzH^xr+vQm7OwJmR1xWof)`>lpIh!NbMMUNg$Qm_?EM zV{^dEP#)?gE0g5N2V3iSDCx?0Vk3^i5e!l7e*g`1N`9ed2V&jcb!Fe; zK&gF}e@Cw(Pi;jR$W@Kz0hDDdD_u9Da$Lhw6);h4G!2{OTVL^_X{)?@tZ>M%0k}*H zye0~4NbzN}`koTv9}CM`mj!i&UDtV-%!QLiCIF4Pi1P$uOZ@;+oElh<_$~v{IzBBj zY&OSg+*!hg;zPD!je$;Uq-rU!_bCKjY_Wxa6Bw^kOH>w_H_;CrL&Nt%Z|Q=r*l&be zYISKobojVsBe^-RfV@FU;JI(*BdCXOU^Tif5*Wa>vGJMZi++x49y=sq08&iijHSvD zHgM-iJ;AR9Y8kFJMC8zEy|Qpoe839riXkan2gXq=zt1DQ<{I%wd`_20AC=LGs3Kd_ zoFs5CvTs`fKM?EKupGETro;A-?vHrJluOb0A0+fIa@mqRj#`CuSn8o6cx{8_(te`; zBol;~CWo+ERBTsvfKK|DZ6ZLEKi(O4H9ajY`J*jZ65X!49e$RtEDjgK@`H$fp9n7tQXf&{lqLR%`4+!l9~L8)u*|7R}bp%-&dmtdmBv;kgCvszV^QnKj}Foy|p?y~c)yfGqt3wxcuuKdcX$45fcQK=z9<0yDI-1Cp@ zI5maw9N%L(F*%UDl4hj4Hd?mVu%6ZxI7}N;<`xy{^wR2<&@8#1oDG9x=CHtl`_a!O^u?8UZPM0jl zuv#~M4BAe%58Ul^MZqvm&D43bcCdJvJixj)6-?0(p}jTQ$Nmn<=Ss zp0;l`>0%3bgY&~QhVSY2zgzYX1jX%pH@o~nde>%lMIbXMqL=_7JpvHCta??G3!Fs@ z69OQ(FT^3nY2%9(YrV9=O9nV_qE(spmu;)rH{mY8-#xtYDrf{gI2AwDiyR*I6ii5E zFb;UAEl)f3oq@_9<;#s1DhB@`2`$;QF&>VpK@33C93Bt*R#5$ zP*I8l-}xAru+EG5Gm>sE?z4q5VYImHUGm-(gf{_WRC)?Qy8&||woogYcE?uypp=yR z7tT*7z1jj8hlq3yKG$tX3ZA5&7&EXU(TV#>@#Z*cF+bvRzS_6Xl{Puqz+!N#L@0>{ z7l@JsX4W%MA%JO5ecDnKdZ2R>c2d?n6>`n(%Nub-+3X5V1b`3N zFvJvPBeIwM`x>OFS{q5xsw0Jo`L>EJ*|O4u7R4Sm`!=|ahZ~xKMMrIPv+QUo_4yEC zB3>z?&GO;R0%e)|f*~Fz9@XgK1AG9zNXsh*T>K0ejd{kTiU>M_7P;klbjNcl$sf&H zb2}FrgoD3wj3LbzFZf!^KKnmaW_uzDx!5M`K`Jg4MmJXZw-iov5xi~(M+F+PBS+N; z*1uLpCX0Y<`NcN94$-jYpvmue6>S!(Gu`*#SZ2=Yag_E4pv1U7ne-97GFTNTz^u;dSX`~qL07CR!G z^F|oAYftc~e|+2}hPDEU;Kw(kgS`kjP&uV7B(!E-CPBT&75hv_EBusP5P|@gpUc>N zfIrEKB}#qy9}$-rU@{Mw!<` z2L*8UGUySZntsk8{1jYYL8t6LjF&-9WCCFAA|{WrwF7Xgq{Nydky~Iu>)s4!x5bT) zd!nCjJ%OHN5u{s8>_U`y;_|mn1)_M{1%q*aD~x)5BGLZ-(?v{c~vG zkZVP=7kBNXoB*FqmV;fSKn9Cu6}oOC^p1hxIj1+N`jkF%1Zombdy54orB>qMz&;~) zV4+(qUXx?hVDD`4iJR5a{-9HutMQFxh=Ie!0tHBEl@u5+Q+M2rhsYcZZ=6>WDb<$! zaV`PO=*oVcI~rUhjSUc8Zl{$6RtYK)=ZTV>j>+ixE;-uP#+$s85-g1!@bG*N#0DZbz(optP?r7@1uAV?K*Zq}R>nUSy#hQk4di z8`-v_-xNaCni=+e@=Dl%TSdWC)vViDU5u24vnXi?Wm4rB+Gpr=6i>Wla?`il0fwoe zg164PyiP}~8Ws$zdKgk@zoc7)SFK3fKxS_F8msxYA*8se!Cv$`^NhmpRd=eL;K2h% zn5I6%?O$OXYZm%IZi1^Nr%vgo^{ju*LEFWqoFE?$3pVz(2>NDLZyiS!?ao*RY-%`0 zfjKY}JKBBLx@df8xEq*q!7R}&GzgDe(Q^;E-(R#Hn7OnOr2|oX?Q~qq;6i8b#ie)% zNjYU%D^^36bw@<2vuo0xBNbCWGrmJJ;eCA=lFWZGttc@5cbB z`uR@$BaQ{oShibc32o{aH}RJ20b%B$llsOO^)G^7ze zw~|Ak&7GbJuYi)y-NE3sw8U~~412A=RP1)2eL^)FaAp(mTbupFH~X#jjlI!GIhimN zbqO<-?|lN9miL@&%JKR~4b(Xk*ORYDoi=@Gw78gVorg#;iQ6(K?DF@vct>ZPGb`x4y2&H~-6Qp(?{<%PCwkY~yf5B52^%7BZf=A0z|RPa z0l<_*NFxuimm;_Y-)SNNioceVM0{JkiqRyRfNP`TyS+(E00?(jjZa zi@GC4kYZyrd#SKhsx{b?!&BH)qOB}TK||odB9*qgE7gAeV%Z^Q%LeXMBp3 zNl{+7f#BqZmloLNYQ!~Eif`C9*LxR4)W2Kxy^lo*W+RxKJEdN>-nv>g!xV)U>TB}A zwHl?y6*IG>^8gfa>C82NSXgF5GBiQk`e4#cRJaRr%JXU@kKgHvkHj4qMg)9)q@b?}l6|CdfGqVwQ^_-g1NUdpnPx z2D-&szGCsR9tsQcSu#hkWJ%}5Qt`P>WOAw!OC~|6skD)oV|_lY8OAVMqn$IQ0QI3f z1GLtXc+n_al~NE1=LcY6IR zfn?~4Nx3E-A~PP9y;jZEp9edCejz~uC9=89C)=b=HgKeKx-1}eu9H!TThA;B&G^TocC|Chh z$)<0CA|YHJyQ(=49LjP=Ide&{3o2G#D0b2&7bWllq=h0ikKFF$VS`Bi`bu;gK2 znP*k3rib0v_@vsQzG?(P(37Ys0N6sQ>}zl`NioE#`M<8sF&{;9uGNmGUi zi97zk>qi?lQiiEFUqt_4$)N;!Bt$|_*_`d4E5r3TyxDR}X_h3`Z(ua69$ACd%K?7+ zCB_5gc;r=~L&LAy+3EeYmNL1N!&w_7+lD(fSKYV^cR13^km)SoyEu19UqV!Bi_A4; zR;{Ipu3z&nfB>Cm{ul#M?TlKW>01U(0&Lj)wGA73w&f+u$hFkNA-Rq2X5Z3NUKl zjxEfZ8^5|bwykn?q%ur7@HWC*-+LL>r7m}oQ`C}qz+(0p9pf^VFH#}Tze<2wR zUyA1>5L(XGdCJ}wq>Y0^HgX@lJ^RpQl7^%XZa+`Q`Ai6QjrJ{hsR_+Biwok{GKp{dXZ`EpOE}Pt&;BkFcD&8AqB!SLJ0ms$AK$Z2lGYXx z!N}@i>+<+=GdAjdJiEWv4u?x++v4Hu+Di;OlEAV`B3hzk6UYu6#k}MGkc|xI-yfTk zB%iEd9yX($!2qvG|`eC^O@b1$mPoI2< z?rFlprZmARS7mm{jjyz8n=Kp541vT_H0VQYS5EyBUxOGTc$7;2kaF4B)(Dom1_tq- zoNGKKTKV_C{_i4&OBZ!5)4%`wU;l4IR{|G)3;M#i=K=Om>J2m|wMu3tQE_WFry$}S z%?mi?qoGZl3)m2Nsqqfo0Ql{aCvB*f*#@XU-9dTjy&!;kEzdN*Mq{e#P7`9sI1yxj z;?EplO!Wnzu;g4YGbHQ9G!!5n&Q3{#@W06HBE$IKC zlu~HlFPEL*D1t|~b^DFCf{(AAY&9QW$Hk}}Uk9$WufK_&QDV3b?E4XB3x`Dx|LE=R zje=yr^+?!>fnj6z45kozo=r!?P5kZZ-ExX**06v?6zx#xJjrlrod?;u53Qu&zT(LE zr{w0ALVjGikb&hw>$l(*tvK|@mvk94I0&*xKui;{Eb#BP3>3Q>y6S0QI+26g>W{%! zfTi@NE{e8Xkl0GIffHlb@U&oS7Wla`n6w~1Zpw*7n44lXa~#9d28b|;xx!oMb7Q$D zE?8*UuNl^p-I0xvN>$qeXi19yq>xXrNktr+!Ao;;c43CY*SjzFhm+y+Z%lz)o#1t8 zxa%w>c?X=Ct%pcCdRE!wBiKCp7o3D>Aji4cpTHPnb%~t;`G0R$_)NgSI^Yt7b+}Z; zg{!GX5gNozLdXW!OKL;FXNe+;=r;k6j0)u73BidKv}VpYiS=9~LE{d1a;mggj*jmT z1!82fq~gMGxF@JmFPm@>++vE1;HZ0Hi18)$B;`EBJ}r+Ob`dP!8@L|T(FN%#ZQQR7 z7jv`KTfIl@$=n*=9QFyJ%a;afLbum8abGl* z{(kQ3^L2 zaPcrW!woFEk0BZWE4Z2~g&Sdeh^UduS|#8v%p`?lq_jx5Xfj_7|lp_pk=(IPDXq%66gKuWS%VTlY=wqyDQ z0Ro34${@hTMKb+VzcCRr&+|4P(VtRZGHdOfnJ3T5eJ+qC%QdZ+DvQX>lh>U)_hqd< z;$7{lCD|QCQA_TkU#)Vt|EjCp#kXv^YxSy4R`tyo1f9zEGt}=atNI|(vSpRg>xQGv zC2n8i-HvvDRClzEzBM0$lmWbQ2nHPw!Jy?3)cyV(B;@UfV9@;#47wkJqNsHU(681* zFz9>;@U4!AAe5}Xum?4VfK%DN+XkHv!JzvgD2f{QoAj$q&<%Dw1cUB}peSnJ1Sxh@ z9|C-<;~@wo>kq+E%^~1awjY9{&WGTr`ynWb8izpo)w$~w{Hpzy7UqteN$NQ|qFopM9%HsrRd56h)xlJrn#B_s= zjU2%!iulV(k(8u-;I!8{kM8>xI%OijWo@1PMCoGnV@4_|NbD06(0F-iVL0}B_0z9> zZZxMhz>S8EsFjoX77mRwwL-aNN=jNOGl(m3%NmL|pqV~TBL1*0fG%<9xX1G`jJGE+ z59hG1Vu^oT5P}2-w1hkbUk&v}m#ha{r_}ocHj~3+hBf!+h{x}>SFVirlR1u{|)mwy4haJ{}ayAIA@(2sn&a*gh_w5 zT;Cz1bDq7Lr{Ezs?#Gd}RX#x~Zs-h-?_!CNmQ#pw+znvM#+D(^sYWX?^yOmsIwQHN z4~lx6Jq-JinfdF7$)vf_m@=}TN|)u9Df`)QySagsq31h0Cxq4HtG7TNH4KGXVKgo( zj5I(e?TIvPlpJl$rXeY{@u+QW8VPr1jowqk40rnCr;&wPFq^(JK?~?N@%RYHBF+&F z04OTNj5>6upYog81h5I0xj5v|{38OpF)Pl>F#*!kPnoO@y<+n%^6!ekr2U^E{7N({ zK(0j?7I;VZzWsT40+8C*+xcd;@3PALuaC1c@u67#D_S^x!3I2p$OSn$?z0O5m~9FB z@ap0MKOZCbYNBwrvhYAyQ$Nh!@fksK=~cruv!H+Oe2a)*L~|j2*p*smd&dB|O`L3g za?QZT{?jC8*Dq3AXl`g%(l#yhn>Z^PDE!$T7UeyH)SS~+^74iozzw|0^XVf^Ys`Oz zn-aIMcxUOy;!L;-k}(<5jT}l{=&l)cTkfQy6je`yRmKGK=_k^QsqK&tJdXh{4+~3( z|pcb5A3$_1k=MvsliT*Z0Km zgo+DiGZ5ebAd#KGhYr#W!2cKcCsvL-fD_>BbyY{R&Fu)e&z~1OY}XOmJE)kZ5kRGN znoKOQC=5VKmgggv36ZI6%apG+q$^X~T)J~YI6~;~eX%x>ptnp*=b|R}a{@H8;~X~h zrI@{Y4y9T$27&dk!i;N`IPtJ0ODO8zsar%}6yr5`oKwkYEvhU& zK6N8MZ|8fcn$jU zEc8C=jC5tEZZm|XT49jl8MMw-tJo=ZSEuZAOP!YGM^Je_Pq9($iMKLD%#wb;aN^qp zOs}6jvrdV>vi};8pBf}jjEi}+f78WB`O&Q}FHjeA*)iQqt^}_?DUW*$ZHJ~?_^ngA zCmD(+f{}edbhucNPNbfEFC+zq3Q&)$eJc)MY`BRNylAiiqn1$KD8&KZ5*GpG)GPBl z^g*@%-7A>-UNlsneS@j-T7B}7erh>uo!#~mi`5-pPatS9)sv09LO%DAl`aA0p8Aim zV{F7oYv0sp2BCedO1#0`;S0wC6R3N z8b&$kh!m(Hl5^jeA~qHiX{u7s_FtB0I{Qg;Vcg;SoupNNWs@VY*fG6k6f-ncWrAI@ z3y)JO`J^rP(Vgj>I5#QOdU=DXV%ZINA4zTNM#YFuiG_ zbjqq^Ae!ste*vlSNetnSb z>&y3g33d1VY;z;pnHt9PAda$XqYOWWno2VQ@P222{R!3vAj_s^02d(~2Y=mUKF}DqKy9AA#FY9{G{hpEczow_A^B$1D0lu~ER6t3Y8R@*M2jg)2mbNHYsx+`tEbyLTKrBmjtq?>nwt}N_$;Ix_yO@I^Kf}6WMhj-W`>Ki z&nC>+5Hsf$VOaT0M{>qY5&tr;rYpG^;ZT#!D=8(VE!VjJqOBsl9K|xD+7J1ajt$3Q zhY_MChY@h!>PF6tmb)3|XB3K8hr^jSqNgg8Z$+r5lWzsW#{erb^{^>xT8kJ+IGIT2 zfp!u4DmWd|IFL5f4W5msQ)FNxcVZ3ag8)7z5=T>gVv*8OK%@&4|3%NcKbar4WSkcs zgJie5A7pW=|6N_E%34};4IRoQTj_dRW%w=Guce*Qp*3UU9Q2Z4lxW{LP*3jf*6rhQ zJ=weJhtN)dvkjAg^!Ua6{+kla_b1(6rNxW8YsL`OANoXA#+xE;7mL*TnPN}07VSSv zp109j(Rz|o!ZtHa%&&vA)i);P=;yU7Sc9-l38fX(?mJDeaZM@%ZKC1or;3;(_W3># zK%4*`dbu*;Wj(7;V2gl!*TP8$9sf^S($F5DMQZU-+Ba(y?xi^RYNixj_5xnf;Tlfd zq(d}isv^72VQ#X@{CorIlt3>e^t56Pf?x-*D&BSey|a$9M`@;%3>zfP{S}gc?90G* z5(=WCjcJazq0XvLT?O+ZdjX?`b8RA6NH~_b)qN7D+(FKlADpH?5cqwQy*;2&@d_tv zD!knC)gRy&2e68;r;=4zMldWmx$rK`_73|3Gkq>*lrJo+C8a4#1uirM-G z*z62@xQ2SN08-RabP?t8%dPnmaglR+(8QDW| z*aP974PL;`iT@J4nSlTgd0#)&zdj_u2^)jTqhyy;77>IjmwL<(Je1uAwVb^T+f5px z8V^b-u3^yJ;Eg}1_LJZAy?RJ@48eHs1hdPN{G^@%nbOJzHkC;!$hpK+n%jKPOAPcTsv*pNl_);1 z5mDh@m)1+=7C*xrB=25xUoj4cg$NVG$FV+9Le{c+Y5i{~rgUcM!@GBs7$shd;B;6k z)$xi{CCfB{SgJp=Q_jl+@Lxz`1f?=X9Z@_ytYasWUG13IP-`6s2L}1y>LZIN0dQoU z?4pDSX{;H8g0j8*;QHWkcKY`0{DM;E@&}*zU==GBz+1KO67G#pN zX$a!yx65T}-bA)`CS~JuM^&q~q+9Y+leN)ZF_sruAn3yBt7a_vt?KT;=J^@3cnN zqQ2*cM>q0e%X(b@>Gi09F|>jxq-8dIZ<>=0q2aby+rDhO$c!$3R$KE&-7>aax^

z?zrkU@a1>jUqtq+R_Yxk7wGHRg5j_CO&J1^{Zhu~1z6GkpCDe?>e?p?+x8=j5){lh z;-mB$@$;tyCUqGuVG5PL#9OzLRduaNDPPFyNk4hgi|VbGf8`=Z?#*J#U7(!A)lVq| zyxRD+0<&*e!%_u;HX?VWKpF~6jm8F(A7drd79<&WIVizVNyl~{j5EbK}v@7 z`t5}vcO_e81H@7C?^b`1(*EU(SlJwLxsG0#D3=oFUBEoOR>m&`*$SQ{{GEIxbK4{< z(K9;7nC*;2ZoxmA^pizKBxwuI(Q1)lb}O;pEL6bOD$LHf_=A6i0+&{Uvx)8G6d=ll zfYQW*w_r4)=Xa6e6;5Q(1#py#KLVfGtFHVi9@7`h87u4%gxPfHs*=W$g?)@N2nxr$ zf4M*mR|xJ=peVoDUE^rCW^j0~yI5=WPXlUL0xiifqaW1ROcx{{foiQKbU~~G(UFo- zR8!&j2uNJ=PZhayKS9 zs7gf;ctBX3Y1kodktN{G8h%c(C}^5sTfrx%V5zb)sGz#!FZn$!J?!(ADP#yx$I*3$ z1GK$)Paa-l!!R^#sz1;uQQA!6J`g?)v*P9UKHe}|7rTo<^|O`B5@0;65%>B?-af$o zEr^BKXZ8iO!Te8531GJn^bJE7adnl4n{z0H@9{p&##c0Oap?t;B$bBgX)@oGVg=#3 z^g2)V%=N!cRe|&{w`zDscRs6%C*v*)B->CS`=BhAOizDZxh7+9O6A#PDl-Kpb}&&t zl~vS}(N{~_Pdk6!c=#RI8Q!NPg~8nn!G$IBb~IQ8)o!<5BJl}{Vm!*ogn@N3B!#E| z(v#{X@LE8>AaP+`CKfe=-ktb}0`Sm=LZk^K^24_)SQ3fc4469yVmzboFC4KbJ_j}N z!X^Bk%ECf$b3|c;G(#+`-f@TfqaO6V=>_)MCijYPh zIWz!IL!3`=$D@=jRLmVWeb;?=BFoHrefmjVB3YL4H( z2;S`)nPAZWP7sW<4o{2h;vT-D+iXw1{zxbip>(goMybSqlsd7n6+J3|U9neVl8`J; zn7!6v*3Pq)d^1RW7;nZax_95@Z{Z+K548Ic`ETj3Cq8Bo4+>#yKmSp}Hd&4?p}aPoa3$v+7p-1UBF1 zu6`oxZ~cfZAf81X5YHe}NcWwNBcVKFYp z*+LQuPqlIB0w0A zp=>oE=i|*t&%Ecn7d4G+hWh3rAiPJPSTA9o%=gv^`GLAN6@Yz-)e$#4ZUJaLz@&&Q zu_)FTNc;c%KOk{>a`5cQBfj1F$aFv&Q^zeQxK^7P>Wlby4sYKk>L^)~Fv;Or2pK<> zz$a&DAA2y12F%0T)VkjBvE7frh(FIUZr_o&hukctD#RRq5GM$J*7F@np%SsGlA1 zbZup3ZHlDM6-{=eu+j#@QMZ*gDmJXN!EDrh02ayB;49_rYwkZ(sx1k0j_;j96f^g= zi~tIj)E$AY^9~;BJ2U#BqnI0lxD%V`*X-5D7IU&!Utynt==0Y*xFkVk<1#6~!6pvM zX)Mf94$uXq-@_KdhY`~s4LjXgzKv=uDN2vlwH)p|VrD-5hkPKpn~LWi+Ot&zPMJacrRHcf}! zTHGsGdEd+2 z@E4KEi{3y?x$7-@V{|uDT20f`Q9j-2hMplO@M46_ zshcdAb5=QG^Ss-c$Z4WlaTppCsNp!v2{f?~cjm^Vp?JC?)b!%$L@bbuTk|@iCBZw0 z=cF7qDKG2#Nsw8U#78``^ZO;*mYkqkWD)+bS(<^bgVX>Y8zskc3miGN)Ip6P9&%NM z==BVUiZc*}f<1Zu{c^mxKxFf53J`}BG6He)QVeks;|0=nq#%50X-S$>;2oSXgJ6op z&BgV;B*3z)4-|@s-X$7wikASgPT4jeq4NLNTV%t+g@}An34<--o_PXR`SJ=RrQ`Wz ztDA!a^`g(acn25eBuw>4U;ULp&hjiS?=ki&<*f{(Z5k*GaVAFBTXeKK*+)mF*2S(L zZ>QJDyjtq&iof-r9aa^2N{HO^J}olm?A%r{eJk>KsWOHZRl$AW47@IWSL|5`43#iuv6*2LRN`{6 zt@z~PywgNNJR%Jalt-w$De3)icsiVpFx|9UiW`K=h;u4A^y zY%LA@(q>X?FEzhw*WbXs#5)x>lQ#E_b|3zz5uUhA7)1s3Wfe+jePyKy?(FJPUB<0MDPGj_jH-sfQ#qT%1LiHLi#o+ya03fZo0bwc~g| zxs3z5$Y81gQaB2`h_9)d9rex*R0*lMfIB{PuNIB8_jQ6{p?#7p3eq%5u-X|!4q?pi za>OnfaV$E`tfPEQIK!hQ>Wq?zPjw(LHf{{M3(cR1;*dq#1A;$fiIYRp5t2YD12#Jn zB6|sAuwAXUxcVqEUJZZ9o*oSBxfof!`;ZY8brqnDK*xTC6)E6{(@X5)~ICP{+*#a6&g+_lCBinbqs%ws5&stNWW!}}v~ziC^%%9+wc;qYnI zmGkkvnrMBx&wPfMdwml@Aq~EyY)?g`oL@c&FdVH)tznv9BAVXEU=~^jV)ia^q9yx= zQMkxnpcVG-fBRpafE{-01#*y}<^26`|2vDr74ax?%jQBN=8XGCZ6pQ2-O4P4@%n~& zM$spjC59wyyaiDZiZve@02~pMEONq`PXM?$+RNilJq~PR^3+*V1R;k@2Zq8Tzwkkf z7%KumNBLQz5nalJI4!TB`k_yZM2x>q^<017lF%1>k;scZD<1#zi{d*zUk z2`=WtrCyN59-WbnM$ZalxrrMr5uAg=Bx0fVV4NrX^=_A&CnJ4uwI7X&58)NUJ25?y zhlr_`$K#h1!fo*7ghxvJuZHq`8;sXcJ`c`%E8&@SR|d6@s<1g|Kt_8Nwx$j3BdzVdYB06nT*ZAp9A zO+_==I6FTcQVR~qAZ7d$8i@v9A3S~1BTUEn@x}0)Y<%FXE0+$xrj%XD>jFDlURW9w z7#ZW>exooAuiV=l{4)rTs`$^`hnWPLo&@HZfRAYV7%Ob#B2iOdyaexZAJ;{V^}Gk|vRjS5#Ls37;mPs%s( z;nRcOlfm9tE8|~>PkQ**qih0r6h$^cUwM|CS&naG#G-ATdx7r_Mf9%x;9|<~$k&(g z81Ir_^Xb0+Avia{`UG@C<=ZHrK|&HMPtFt}RRnU$5rJkWz?Oz{_~h$H8Jx8)ybPU> zU{>I7fZ#-oHh>4{8B-}GSgVR&C={BJ;tVo|B~}aZVGgd-@p zTqUN0+JlV5h22koT*CKChYa+wYYX0l$oGt zEud*}?X=Qu@B=Ayw#JK`T}4AU(MZ*r7!nscf_#yE!>Rg7s*`c*&H)CFD{VaQbnZrXzIcj4`{|;$1XK(z!SHbg80<%w z#S%A`?=L{fi=oKXEC^h<^?B)gXfA}cS5C5kr|DtFb^hfr2c14%=$dFf;({@fsk|kV zyEw6!8k-QV{nf^b9JCJ-bY@fPe;R2oDHV;P9%+M6^WLRNLG|4Su{Wl!n?U1mwS29# zVv!I3ao5f?e#veOH>b{#6I(JShJ;$vFgQg1T6J#DPfuSY^JP5EdHn-a2q>g6;*Wcg zRV*H5XwyW?75hZ>oBkGS%H?Y@DtJ52h^7Tj<_b?dZ^lqlkM)Z&*pE}w89AywvFsjS zCx8Lx43_2VC_-CL)`?|!Pe&-~)iOWi0gfrA24*NKo}+w#>bn#U&5*2F#LPj;M{P4| zhV&rfY2I$+Kl$+b@!;Y>g5i)@wVKKUIe_sT10XWMX9&^98V_CC5Bf)-LejSwF!m5H znY+tHp2m#-&^TEjK)hpctjke0)FxvZ?Dq}}F0UG9Yw#q=@wFDB^+h8+ioWxiA2h~Z zvO`8BM7y%1?DvZ`=WX~ZgY?pZ9YxaW5EgX{E2(ANazz-5Q&VKNragx3)R*ZE5+BMr zA04&C*Gzh^k95N)@q2wlSZc@g8lL2J{WP9$^Rp`vddaB{x}fHYAaY`>Vk~)*Nfy7$ zrV}R}STN=c2lqAkZC)TX@`$?0h}qDnAO{+SRLt%r zH;57SR6l%u@snU|Zv15NMOUG|{NRbD0pOZMXt>C^lI38i{DI%YNSfgD1VIr$5ruFt zx!A$FvZB$^(-G3>%Ipro6i1k~S`gL5mrs|l4>A%lYB$&jB@08+%CW$(N(OD z1Uo9FJ;#x{ACs%nUV z`c(?_mWj|WSafM+M}KzV*{sfV!Fh>>I#Fs;H?G}VaSJ6SIsip#2duSiD~}F#+u*fR zCn9cHGL9;-XMtnsx|c*;@7OtnR^A~es(!FIH{YdivD$M3@~B$nzhtB{3J!_$1C!pw zv15Hw$=kd;b{U4ghP^Y?*5|9c%4k2$Y^#?}F+ITxW(CetDtcQ~v1WCFSGHvX#O>7e;%<32nz#pi`5+})X665|dRZX!2M^ank9AK{|fb=6(BjDu0RHME~5UpTp=B@*8c z60PyJi2hmK`yB;!Lklv+%d1GZx$hApRZS}8z0t8ZR;5I#u^tIE8jXI|;%ctlBuVCG ziSFBN7&y-rMmsjk$ZCG(y=3xZyUDkHYY_;#K^>2G<1MS-u#~7^%>bBpO{OTs)OpW|kPZ+@eTkPQg4uSv=iL~DLPDuii)Tqg*iB0-Lzz?F9%(%vp$6h@J6x8vK#08;}6N4{FXl?39A#W*_!B!RG& zoo|6Sx6FQ+L6)i)=cm~Us2je7bI%kU3o--YbMP(k9+-Fo9okolo$vwnUPGND7I`|g zW|I}*an~FNqw&sxg|L0(ux3CZ!1ZWd4rDD$RG|Xu(H@U;wcEjw@pQR$Mny^5A;|P6 z)k^>PFfF*D4|oVgpVIwj-CqqxJHs80+wR3q`i~sHPWPSm7J|R+zB7)}AAa|R&bHcC z+$vyPVZC=&b&Id*>!ol_SF~1Bd^Kstc7^iZ5pQ_u?PJ!$)_5flx7GEly_Wy|^!#6w z#BWn9|2DfjO!S~1OJW&~>FZ|iinI$;AOLOFRALu+g604_h-qFT+^K&(bY>s|PIK)nNI++VYjiT>-37wyAK41sof=CICN@j3 zmg91S{7RGXv(pzLqn(X8AhSIN{ZNm2H9UQjQnkmj<_aLI)g9FmFjJ=OO(Sv4yq`3t zas|kM4Pr|c68*vK40IUU58Pv^yw<`aOawzkn6QIoz2SH*Kcjtw|2RMt@ebQu!Iv+9oyNwjl;LH_N5OrfJK`hPtMM%oL(HFfFxEGW;89|tRoC`G_VU*;1D1*XLg2nUJ2`v(8YZ;I+3DM}bNt`M%b(Aavw$tduGMt> z=;@yI$aPf^+6qBcBN5zywM?x%*pWm6X;2>*qDkF$LjT9I;@CzjF^aQZV`>g!vX$mq zF1zxdsyq0H{3@)wj`z94>x?X}XRNr?-EoJ*cPz zY=dD^u(d2U+6*XOHrb$P2NKWQoLmT@EB;UdgV}QcHA!~`7d77O2aqZbY&;-dr-JIP_68>WDh5c>WYshC_WmmkvOH3|849>vjs-SB3wg(j^$39@T40s%NX&Q>c4b zaDr^%o#SXY6__RdpuQ8zeF_^Z^#J)Tmrlx#b122lRrb@vRT{3{8re(Kf0pWO&5PWs zyRo_s4l_K`HT=v9T+4{;yi#lP+KOE5HrFL;Me%!%tII&>6ZWpV&HDKMLhxygBEp+> zw;@!SV%XNBFF6X95*%D63Y&u)caO--(P1<47KUo~s)dS1CP4Kp&yb->1HIw~^7P;z z2)ze)8_yVYjly9XEXDz_jx}9pgr+4Vs-qun$e9kf5fX~L4}OM%21(&?g*T{T62rLw zn7R#e=~j>q-(-78e$YO1SosWdD>uYm{}F_Y2|~Ol#~*lYb8?&z7(`O;Lw%r!I3&cP zMPzWqIL3l;ERRLC+=~QpBu(KrpV;ShtwOuPov=ody?Ej{&&BdUzTGX676_L`0ev01 zp##EorkpJvk(f}7h(nJ$2zLNQoEu}QdEb~|I zBlN({_}82Slc8raXt*fxe1MKfhyqd;lZ6Mfn=cf!fVk~2(!j7FN+~cRnQRdF&pen5 z(6#Y^&mEQd0XC4b{6LQ_EePVF#gr+S8UX`BdUCyj12B{Z_Mp8J7^$yt)uL`hFod4m z$fsR84^4pZ5;gsj*G#@%?-R6L&&%qUe6!E33C!IoUQ#9-7q)nIu}XrR28Sv%QsAjn zYNTZhu~Caj^^YhwIsj#T5g%W~#~k9~|8wy%IAZ=uqjDisPzgM6BqD?brw$VMA3{7} zh+lLGTr*#E32H9?WpxQJ44RaN8e+JrIQYj_7bp+D=m>mOEcjsiA{PE@i-p6)73>}n z`V6m{^}H-li$nES_jGrT1Xq@}Lhl51=7!f`3A({k@-2xcwutMKkSA`(#LDU3K#yr2 z$Z&QO|JJhvLO=#Uusmb~!Ldom1seXoCEHZOMK7I1VHnd~;Sq^IV23q_ZkB9lHnfUH z(WX)I$naf!pI;Noh#-kgS;4PnU`xTn8414Lcf*`rMK;91W_NsF{{3(N6P_Tp2m^>d zL_>0L5wZi50TZ*0JP5dc2t%UX@ez|VSj6{)tsaa^Fpn2rrVBN-)6yg17W8X7m>Pfg6v!`8^Y9FJB96LYv z$nZegiSu9t=uwJlfa1uHsP(N31B zUtOl8aN>!I)}WLkSeJHK8c!)5&k6`C6(B~nW*tJyC{T*6Rf>;;(lp|HEPxt1_OO~l zU>FFK#PGUUF7_qY(M_w7Xj05+Ig_w1ORs*k)uk=9habCVD&ib=>rB*nCMCWDbPj`y zYI6~*4c84aacGtH1X4`w+{4P;NIBE-B3^kPy6%$asAzWH0*_&l_ueQ;NrF{S^H^~^ zO1637X>*V(0^wn+t~0ITf&#Rw zfQcZ^n`?9%1Hxf1qJLT@s(BU$x3epF{@vze!#{aqeE&NlGRC zO_9{6d0rF%?N=Wx4>ASmtE6GlUk#5D9QdODID1up#gqmzH*f3GqP*R z;6xG(?A>h2AWv1;vOOU*@MNuNih?_JJw16Zjl;Jyf>R~my$1WKF)JQ%6O%0wYjMKC z^14_n@$-kJ%NMI`?Zzla11g>q;E@cl>ii@x7j7=>*yiS2TB7)9kn&pX3>t-)07 zWCrf6^$<0n92nEF$$#C{-3AD(ic{KQn^R`lfXX<&%;)l+ryQ3N7AfYS%b}GA;EYH+ zLcVvGOPWDUdb53yURB)@4TuI6SZHjBIvguw&Rn}a0lJ4I66t&Su{&QzySOTEK1fQ7!=7gMPbHN@J+75zj} zseA;y1agA!*YGzW`As=c$mM~}9dWw35bo#ul4|VR0_hWwMihAI#g;t6)9iRPo;o1Y zy%+z2T%mtEeuOQ41rP=lga!rQ0~>$wA@9J0B&4PX5eNc_|Ype6g=rGe(xngA{q1t=~wxw9R#O8+Ot0|#t*L2}v<&w<2 z>d=w`vT#MRe)ehB@M@L=)>_!Ufc$$2^pal+*F)37?SgRiAsYXO;9!-uLHmqX^&713 zk#4Z0Zh3=QX*8u0sZt`xyJe4jkNlw|)w#M)M$9rwO6g`nthJ&wAW$5j4m9g4=)=c5 z^X)sOUb*AhG<$E4(O4d>u4`mBRbD85ZFFb@lO<~mdja0oZxOd`3<1B47}8z+NnAl? z$Zd%m2U1YP`G|j91LpIZY4_+M8Y5pH6~FZFOq(!&lB;kbxJd3v9|oHdcVPdAe9HW_ zY5J(wZ^{5_Cc|)M7xNM6%5TTl4YSjh)EqUGZN}Hx9)d*PBbm#EG`twT9U_Zx1C?O; zOTGeZ6s;fmuK+y(J%%U&Af=S6%+-(^P3&19FY~cpMaS*j?RYjCl+xa_L=R)j%+Z)4 z(gBuykWj6^x&Tumx;IujQ)_ah2AD)#jnP_aB-$e|o307OBFvAniPFzyCrF`~A$&vO zv!FhGh5hmpPX{*7d30#{2*cpNtQHXb!t5QYbuZ6{`&bkGF0i6JZOH_fQKSGBGAft0 z9j4dExmYTg&Q3Dc-Dk(wFt=+Mm0yJu?QeBT`_B_W*}j5piErFJSW%8~?Ur)!dGp>n zjLA%Wb0Yz=dbBsq)HkzaY2UY<{O1YqD?GY5Nw5h|fu-`3CU@RPF-Pf1^5vo0zZ+qL zFDCc+&op(ocok{l8=epdp!vn6{8CB~-hKbgx0?7?>PD2)M$%nqE{ZC;2X>EJ03A{! zEkMhSBq3@}kG^4|mBHHtnFInn&65*V2^3G@rH;&GbzIg+3Eoa__~@zSvSzxmGAb7F zB0KR~8%MWpaw%=n=VgiCJ1j+Y$*@lPTMDkzKQe+5tqozl=L3<)G~Noc(=h!fiXV9y{$dFv6>3O{b8Hv9K-T=5TU8Kr1-4->f z-TIj%#|5dd*u&9%6m3|-fdJRz1^(yN1-y61 zXG2MCKR(C*y9{c+<&A8;b8ux%*Eby7nAo;$+n#V@+qP{d6Ki5;V%s@!CYacEzH?po z^S*UIZ+(C4N>%Dquio9Oe^`5W?`<*xRPEzV^7hDq6CV9OZep3O(m-*d>#UR2ENiaN z=u#@huMt}%72G))xAw@mU@F|t#204-C<#%ULYGU|y`cucUXT+M!uwBYCIHqas3Q^A z3m1NFcuo;0B*?8GR;1()?Z3< z7#thvtL)Vj!X&IC+CO8?D9{uw>RWdOhUm5DSWkLjP`k2B{fyzG!Zb}s;949k?pWrC zIbRqyo%N=%cO;7~`}0RSw+lw_!U$^N0N?!hOr~Okj&7@P5 zDYri1y*cMbrWULv#>vZ8tx>}!ANcM32zbFa;)1m1Q&$JYNvHUGd@QSPfj$jtpS-Y{ z`0&<1%eAFhSEtMy(s`n--6}8>SvmMvK&cY_OR7enqdOU1Wj5ywrtT*yq3(zec;NVe zl6m)VsD*!fKaY3XzRXL%Qp0U&nsi(z)^{2A<0*h;8O1P#JWrVrZUFjB`hr#!51oG3 zfsB*sDx$1OGa^i>!S`^8s-$c-|1jo;bvnRS`vmv|T@5$VQc=vW>VzK##*o!X2h*dn zTyNk1IAeBPz6qtou#Vip0xk&aJWF0;h}SZF&IL^k`DbPhcC8T$EkRE~aj&&@hczqu znD99@TA?e03KB+b+6QM6_k?+Hwk~|y-EX=)kt}N=zI6JcU~xg%(P1FK16d606g{Tb zY`Z?Wo%EH8eGiqK;iOi{$xeKyW0=><^%eE8e*j#}2?G1o_9gxlmb>NXI3bHzF%X6> zlTD_S&Z2OHYX;_m2!Xa5cZhjyosx+x!QvCf`C@K9|LJ$DoU2k1&2eNIB0Lu-_4IUM zG1s)|l$_6pdh4T~)OFKd!jz&rB(zWj1MBCwaetQH$u!V~#Zm8>8veNW>zscXEpZC|OjJWUQk9z6%DmYoi=lp=pn1<%bCCv;+W>13;HA}s< z4mF8&WO15bcr|wT;7g^`aW#&pgUCfkEX(p~EsjJq?5RG!=&ZD7d@fysa4A3gaoa_4 zv!@19@|(Rl*XVBq(X-qX{#v^K(>*Q}=Pb{{<4dq~1zA%gXvL@33B~QD-yS#1iF`1Xh=s74GkWY~7FwVw{E`Llu3q!S5+c zC8q>l8fa;`zs`v3J$KBLb?0oGRQzr#V$Gt}x3FQ4<_Ofdn(-5dY#lFz+!5Rt^-X^< zDqR<$)%S^x{nJi-66u>F(t|9-Fi<=5$;z`x1zEWh=gkzwslKQWIdANcuZ$9cLv-_n z6@BqTut8S1IDv%%`*+PCOa3l$C<^&{n^w$W6Y*Wr=UI3{Vx8ArC>gcBsEIg)>Dx+z zAEOkFe{9I$moye-r0_?r7pyEt1o~;v>QH5`x_h9;j$$ur= zJh7cA*S2v)t1U^DLoPE&U z+q1zZ#SgLHp1ysm0*AeCkz}g=@b1Kv$mJ7h%n^{!4kcLJjN@-KRpM8`IEmS{AqL}< zd_BeZ6(oUwe99mYFLN-LKaypx;`f( zbjlTXAvfh#L0`Hu0N{U0NFVp>!b=(V?8-IQKX=r-qkA&BqUQ| z1m|3fVHfU`>{MjE0#PImB2?8nADOU&iu6%P8oReCO+m&(NKHf5$%x8|xdvgMACf2+ za~ytPNqjC0HE2Cp^{OkW#{zQ5?t)NbgPuVa$jYD8n90aT`$^hl)^Z3BVTVjqqwJ*7 z8ouB+c~CK-N)R& z-Bn9h)##)hdFCGG?vuI8d?h6*jk6%#P+(u>9w$4DQTYycdUqc|eGZ-yH1HBtKTfpe+#zJZ6Z=lQsn3 zGr}b;-VyFeCuJb*JeWTK8f-5~7XvZR$Z z-^hc)DM=y|ec)k91YUaefjhyrlOmvbWO01-Ma@?r`znat())cS({M z)zxEN^wg%9W5g97R560;g-%8rZ@O3pAn1+P7E-_Mw6siCBs}Qj^UH8R_#l0MkpD%w zc;i)!W%F&`9U(`}y0hYxQV8z3<@cVHln+(kyuwd5rbIV+h}2zoa7psQmTu8lnIIB> z(O33WE2l6d2Uy7)cUWA#YLcN38L{F92dg1R%mqiKg=gojR1yRgTesK`7hJSB=K7EM z2>5807yp&7V|8(e`N#cP(KZ74PQ+ry?7}AtKIYG&GH%&aTW|nVq&+I|!(!t1r@Sye zlJUYa>TK;j+qT!A;O=!$(|Nml?ag8F^e-MSg@= z01(kP(v9Coi{|A~H+}b}gWd@ET@4~MB413KZwfcr(zZioLn0IK`2J>(gR6Nzre%s? zBOo;r0i^c}$bMjyqU;XV9wWUf$Qd`bx3M?ooVrf!O$PWNSag;#kFQ}CzQ|PWqhT_` zU(b_;Bf0ZSwrmfIQX1aFWr}M+2EM(VV?F&|d7b%NN<`_wf-M$yEw@Vaj;5}IR`2lR zhnPIW-f~ArUB+Z3)G|+zK)YIh6!0iN{a=3->eG$C61r_R?DnOLdRbAxv_~eArF9z8 zK;WrwcwzqxZI5kBK&{zLT=D&e(C~J#hr6OQdK0nYaxMPI=jIa~`fr#E&fmYj zD&ZkDB{_?N^l3%p^P@XO!5LhmWi>2QbhoOCp4A$7-49V@%hYJ;vQPrB$SBwZ(WGSd z1rqa^#Brfmt5WOfu_|dE>q-t^>!I`O&2?e4z8A7X@`==7Lbd{>Ci@Za3~`@r0L&e8 z{64y2Y<|3~w?Z23Q&jPc#94`5(z^g3vv%SzjU#-T%_*fXjbrt7N$SD$794>`v_Fvs zECbiHDYTYSJ_^#n0{(sfkMno!{2p9TwO6R1qno;pj?8OR>=a*0Hj?6OFm6br%9u); zB@{0L5S_Q%YWnnWdQ8u9MHXG5Ca|i}2Rzg8(TLG8OQ&z$w4E-@XblFz2^nX#S(jTl zq2jprrBE@-F2Pnaum&St)f~O}c&pR|_{m;jLSug!p%XrcRNy)gUNO=m)s9By>y2r?dIUKz4nnMYmbH+!Ca3b8yECD}bi^$k1*Tk@CbVL2E_tuvHlS8b9X zx=Lft^`zsa$D;? z^?*o>Wyyz<{8XO1rFX+}CFBf8R;}jCuUI^FhtMyj>5RTpD zDy|eyGFLmOnhuKpla@M9@k=T2-fmBap1kSf0U$KOZFnzv`)X2Wp`hQy`pLf69gt`Q zQb<#2uE0GHsz(>J@;%SJ6n{*j-bHQ+7u_GX7&*QCniNEmuTGIUz(p8g0UP3LGtiBg zS3t`%R>N{D0xRsudv?DN#9)XQ+!)$t&yyv5QR?d>^o8iQHv=Y-RdXHf91&OaLI1ba zSA2yi7ixW!R0G4HW3gwfkC8KW4jhbyg%~Af58A~v2v?+PpeMKdXv7=rbE1Fps&ohM z@i13s?urs3`v&zz>n^6q)%k1J?rew=O$*DA;(ocCa}`&y3Gh=(Ld{(7+}m%Zz2#%W z0#_N5NjmV}+vgYRpEP%r7Zf8JxQbWQ?u5&1A`h=n+y?`uyGfuh!bMVrxtmd>gp&MV zEpgU+*>1hMb_IWO;r4&cKE5@49(;1Ro%N(HCH5bw5(plb##aI#9anSEqL0ChzV{_% zdFHXIWDVh$HSVanY@sEb2qZKHw~NH5+5P2U4P=vsv`r3jYDD-0I|Dh%RfyI*oD>HG z{J`F}x}ky*%&u^7oUmlN7vvo_4_bu>{7TL=v9Vw*rE2MsH|H15XJ-OU>ZAx0Z^Dq$ z6BvO6_eo2*sU#|`EXLv=y#icMWMkWo{aU(9rjSk2>gaH!b8@V9(L{85$<2+_Bzq2CH^qHA=m`ub|J_oVb44q04uCa6J~hD z#FneLYUFkd{9nwd?Y8H?0-qM1AzLH&J(qmQ?FdN>E6fdHz=#4zMTuXm6wYQimjX%* zG!8?&5Nn-~xuB~H5>r!t4L#5Qn)QC8cK3WFab4;>OX7uXTY^2ySoUqwCGT3&W0x~z zX-lP4#bcyAhrx0a6n%wy_k&T=!Cg$W8o-i9KU6n@-Hm;(LEhQt^lo)fLsha`uKpb8 z!O9QpscrO=(z0&Al}mz?6i&)EDVCn;nd~ms$J<{vVPIr?LYo|4b-2Y}7U=h0;upS+ zcr~VBReWhZ3u3fjm?qVG|A1x=>7Wbs+eoM9-5A~z_EwNyhTD5Tj6WBMTbSM%03*c! zZbjfTTnho6;fXMCPiXy<{`R5$v1B#T`Zrq)rpP5L=6r(%49yi8k16na;^#`dCWmg+ zr3}RD_uIbEtbOm;xZ}G{Q=%{a&M0#}OS0))r_4?%NSpmOBgynror;#~G4%-^W1T(q zC}ePi3iA8Q{U^-9i&)EC)>DvWr1~P3#$#VK9_(!7T?F`hGDTmns6xJY==5CZ-SDcC zm{N7wHk3u{)X8}i?`uR?q=r*EBe0MxQe4RZl&Xpb=IxU9ia42skg6g0MYiC2(n%bM zl7<)&hbP+`c(v#2SPyq{LMz5nAxXB0Vk|6sj40gw?PF&0@oCncDiO+Q=%Kp6_X9ej z!FUZx2ok@T-vT#Ekv!-DDZSRFuYRd;B&7@2T@bbV1wIj|&I-LyS=2iLS@L6Ub{+E_ zS4#E=haN2XQZ2PU1ys4Y_YB>VH#BbJzk_~2kH?2A(t?tDp0m_?pYn8U>eS}>aEeDJ z=S-1_3jwrIhy|19iIt)7y+b2#TC{5lSU;oL9Oi}+*-(L()tXsHxb+^ytEMQ? z&_H0nhv-yRL$tG(InMdlO!%{cMWn8G&8)W3`v(0E1a9)1SH)r>44RICmP#~p3;6q4 z_L9OTERSKN+GIqnB1sU=!jN`ezby}f`;&b9yIx<2hIg^t5hVC%BW&!xwI6LmA#7X@ zoL|~RUd=DZ&W^AHw?vu>S%!olZyj@cOFNWoQxjT_IbFGv)6So#H&bZJSme}&PQ|RI zHYf=CZv8YrxMQp*-FCFW7$bV)nKCkT?VsZ@BjmtW5sK)w(vnS_0*_mON6#Gx>*9$` zX@h!ZQYCy(yH)1pIf867M=-x%k-U*i()!-%3g((UB+8fd)lTK$>&3@b+RG- z0gAa*K~H`cuKaGU_pc;bz~fZ8jEstwA~mH=wexxEA4?|)f$Ph_Bq=V+2T#`lH|0Zn zqGz4L^oM|ZlK}2{8f+;#g@2Y*{F{T@Ef%Xp0WE6*J;^J4yPHWL|V>^$T$r&s%t)IP{}Y6iGd#5U{H?M&HVVq0ccA61?uRsX;~rU7u3YsF$7ogfLHUB@z4 za`M9yD)S!W1`6rS+zOTv_g8B6tW4}S&0dO#&*Svibn2_+?XxFGrzpg1me_R9Q-K7h zKHRMsR65aZ*NhT^BxJh6 z&(6(54EiLnAv)!BEdOl-&ypRXbf_W7G%kSqt@!p z0FR`+j%rrm7MZHBn+}5c-cQzRsK8?RnlH2=i@kjz6YkBj^~`nj{X5Xe`Ru!IDJ#;y zEU*@*)#N~@tz2yfsM6w_>_M8*p3Vi2zu8u}p_wNq{Ur|On>2+2exD&Kf){ODf;plv zxde+(&~2a05)1ab9g9;<%24~2M_P7wX95Q<6K=mRY=OibHr8{J#6otJgcl&V#w)o= zp->aDf)HlTPdVl~hauS?BZ%D;jp6)zu0&k$b{)6mM<=q^QcyY-8g;D6lA7Od0bhR4 zmqo345fA6v>FV;?ryv2`IrWq(&SDpm9FJA51=Qa;{`jK3RNu83zg1g!;N zag|rfx!JitV|b+7-=Q)@Va1}fsDN1q70FXHpu;dAmeBBu=6<@=BUzRr>&Z(F6iJLS zE2;SHtie}b5LX6~p>>icmj=NO&+Eul2`;K2dbjO;U)kdzTqM5Yvy8ZRee(XftKO$fh|~(8YRIFu?&60yTME5D__3we%F;d zK601n?J2KO%+{FoL=0F6Hj=Wa6E3|({+_-{5X>s15x+bOsD^K@r`? z#}e|p#satXVPV{F9yfTHuO{fv3-0D>y(t$6{9*tJ}kabV^i)#{J zIj>QDbBbdCYw`{+a1kckZ8ym7`WWH0!OS(C8-F&Wu5yYw^GL#dNGGk-Sc4&ION^hF)WY&DL(h0|f z6sEryHaLQ}B+HF~TpnBV_cHT6r zt?Cw$X*P4u%eq#TdB)hIj3<<-1QybA7A`S3KK)aM*-S?#n1>#T)t{w3e%8CfVZh=7ZEjM=)zKQ49YkiAZ}X&xYK{ojDw;1X>=BYj4Y z>$;F#d6?;_S1f^Qw=M(h%PY<&SSdzc8O*A>9zONX3%+!Bi*8Se^{>>wA@9u=pz!__prplRaOwA_^{!v?6> zJrM$KFy%hx#S2g?K+L(wBq>if3EvYmF@am3Tn^f2Zw%3v5~-+&C84}}_uiKc)uet3 zm&Jm=Oj>t+asOeKOg?^VI5esUgW)F9Yd@58g0>^#cb`vYK46hR^(tj{`7j*UOUCyl z;T2JEU209G#aX1e6(C;$yN0t)WgZ?M)>7>6aRluRBOBPB$DUH;=;VD9r-F(s3VGA! zJ-ihUX3XJURts*FoGQIL3yX}g532#R?rV+_E{^|7bX#^OQl5~hHaK1Zjq1qV+Hk;T zVL*_YDknCG&eM0FT#(&ZRCijanLZ4#KYv?YJrO&w6f)=?3E%hjTbP90hiZh?GUY4^ zX#T?IqAfuG^7i7u*^ZqF$khK}?-Ye)KomX#4}A64&j?Zl5YE=g{OL|zNRd){M?_=y z{gZ?6*{%tCDD|gMDVP^;b~Zg@f+}!un8sDXpX>wi7$&K8$lLfJ@Dr|bXRx%G+pE$) zAtfmtRyWqsevIY1u>X3A-ThdqP%(#UnW_m@?)q zGg%K2&AlM0aaqH8h$w1O$ieLws=#6yE|j*8!l*wBPwXG?YXMsCVfWT%&0@&WU|qB0 zjeWaD95%Mcd+X9MH|)Sk$d1ou90xnLZJioK<9&8KIW}pD+w7^47ws!Hkc?zMk0GnP zIi%e-4m=u%*W{<$ilz<0I_ysFiwC}fKh3>q`&7k{siGCjgFDxYb>#`1`=xc|4JSaP zmxRJ4WBR0b$ujKCYt+#A)_{cMjO|AOdUlE~)TvhR%upnnG*v;=b%C(|D}oAJ2DcV) zm+t@}p*(AHad|jQPh;lg@U%Xiim5Zs=CyQsVgEq&*RBQexf4%woazKu8mT5*DEip! z_ZsXx-R=(!Ld`w&y_YFYHBCpec^jALBZ<1im@75+C)AT|YxO^IvkG zG4Q{gaU>`A`A;QYBe#n+Y#B~j_Te-6qF$Xs*N<{C4V@-QrfRoKVD%I}5bHEb5*G8V z0;pkJUTn0uCgElCdM^ufs~yI=CO!D)RrBaPhr`pbggg#pW0wz_?q(q+9B)o8 z5i#khge?6QN^m~6TQfikn7&R~k7Nq(Jv^2P(ip4*WlNeIpmck;-PVis*1Wab%dk>k z>VB#L*iL$X9fGq|pU{%5d?omnFoV4a@P$+Bvk~)Cv(v|ofS+7Rt$CfC-ZtRtcE6F1 ze=$)%#7Rfpij-OR6Hy+b)`8H5iffQPDEfcTA7lG55%D*R9?v}Wjv81W58%`Fh3Ej3 zT%RCkNsYw3Zgg0@zNz4@3t12cvk@w`@#Glx9- zibx~Es8h(IS%B0VL zRgn2RDd+dFEb8-N3Q=vXAY;$!MpsYIHl)%GqjBQ$8BsOjtZnh?hxwt$?+OcTN)jq- z!-99R>E%ixdE;n`%8{w5rXY+)TSe>S6)y5v1Cl2OG!U*FO>f8PFfp^ZHB#E8Y|YV< zPch(2p1Akcp1j%G7j1!F)h{$!S~q{W^Fg0Bb|_UP+g?t)Qua@WWPSVCJMxWdXNt2$ zYaG-i7|wr=of>{YgMfT}frBW^fkR+`K!L!3fPj#I2$61?0op)7umzw%uz~;av}N*e zaJF%=G&6H`VfyzE3!|r_K+ypD*T`7ujiv9C8kf$_t%U4#S8#+ zHxUxD0l)vt_5Ix_Q$T>9@=V>Av-|bl#?zCbzyH_C$>jc7NPvJp-*ZC2=hK(>Lqd7o z*UJT9bo7debRff_4%z?X^P`Q(u;=}87ce?|)%$UMzpq{qp*J^6^yh zwNXbD@YWjl_yV*rM^A=dcTYQ?x4S2I4ybz$L_&V=pLIut&`g2>&o`G>hWR}`UmNAv zPeMLYS3;V{tK{ma`|u7>R7w;i{Lv=NZtovsWG9hZ5!= z@Om-5&$PotC^Re?Ai(Q+^EvpG5WxQ}s9W%PWAW)qsX*)NQ|H^xez9SX;0ItoNZ1h!wKQH8y4QN} z_k6{G-7f&nfL@f)RlM{_kXIA317QJ>i+ng z!)x|c=)Mqac%5YQ*f&_(xJ0b(wdLgbB5CSq1WqRu-ZL$jJ(yIAvR5NuCf7{qH ze#q#~&skpHA>gDPjoT-J>I%_WM78K*d!XY}6Y98dcR1BTBGEpbd;q#0nQ(B5Oi0P!PN=4?2EQPtwAa76oO`2^+U7o}4SsjSq-c$qA3fYDLjSl? z`#LyyUY-;xqVQc+9^_aS3J~0@ZSAU8Dk|oe;@xD?%2Cq1u*G|;CwyKX*woJ4n5JSG z($6#yn98$b{~mC=`8OY$cbk}H^et$m_;-WxCBvoozT!iOag)NaONOPg(*4T73}>a7 zZC70UDPDKe>EWY{RRO z&y?3ty_zWQl8X_i?^KL=+|}q>HbuM_M~xA@^C@}HP$+964ghI<)Tb^-!^a;|rhUa6 zsNF7Z>&>dWo_8`2UFYL7kKx!^=RQW{FGI;n7b(VfGVcvUW#_sMK}qUOh^hki_n}@= zvA!R71WkE$$*sxydqLQ1G~e*b+@+Jx<%}PltQ(_~vTNdJ{Z|Wc%o7f)g^TFdbRq)B zX1qZo&ucOQ$NW87BsI(V-oR_mJpe;vNG@Wr=ZJ0pG>~+)ACLWVDTP<&Eq^lUA+*iB zY%(HD`GUy8I{7p-#;#PDA2fBBik)?2JgA-^1dd6`QZ>cV#4E7z#F=&EF*L?MTWOwk zgtmu2&M$;QcK3S~*uX*3)LnAs2S1=0H{0Xa=SI1HHFX_w74!s|U0~ptmOE&quL-bM zSniOKzJ>rB(|Q|zuKGCNfVO!2Nq`5y;0vZ`42vI1yl00iFf#UX!zQa)DyvnBrgf~9 zrh9ojPlQka2SVIE-iuEajPz|*jIR+y+~V5C0RVPsT| z(1u{7T}NqTQnYm(s!19Cj_}J>nS~fh{V*s={n%)gnII}GEdW-l1XjBQRokSzOxHv6 zM5$qjNZ+VC_Yw&4;iY(b;7>RtE2C&;B6v|2X=U~0-5a1HA8Cb~wLpuyt>ezqoDnx7 zJMBT{4gQW*PE>b)s_?Ofi#JwxgNkE#B1cj?E)bfn6PB)$@_C)S&+?Bo7*w^80b93~ zB3w_B;A^xMx4w;@3`II;9iGeiQO6tXu+;zkpFFw)TjI^ly%-)>3Ta<880Bw6TziTT~ye|ztKpsn;y z9E~4-xq+^*UM#;+j1f3?b7|+3ibL$pS1d5`n}7y%{w+T6*iK4n#>~P1tZ^sH1}!#6 zgh1L6vy`#lWz8-;Fp`xKB&quy4Q1`679`1U&pS??{CFc1f`CU2wiO^`Ej1FK2|*Hl zUjK9vMR}o8cA)~7iIrvq3?i-qD~i$mb2UDdxy##cC7zx*sls&)yOgp&4Y3FuQ81!1 zrrx9C=>v6v?+fOnJ0SG_Qb^SMEdX5Oz_MV|m{{eLK(y009cL4p7@Y)46~|mQ+muv+ zEeS4gMm;V|m@QP?{tARQgUKV`NH*IJ(=1rWXufLlP}uE9t!CfX*Q}D3BE?C(fW81B zsp&w8G^Kp7d?rK|GZ#|e2Vy|Qt@x~l6FKGbkhCPWjC(ALaJjccWRQH7dh+>gihK!; zC|c#JNSEB;G}YVw9|W+q;UB?SRGF;({N$Mu!8)95Ziuzt=J@9RcoXy-${e3)7$X<< z7*LUxXhp-EwCq7uZCW%a*%K$i+v`6H&g2c_m{k#iAE*}bEZ}`*D#SCQhq~(Up#wws z(43@B5hu!xkCsjGqn67vBg2{}8KqCl>&sR@Tp*P+>rmEpWKPjaC*pT9X+G-O#rnB9 zkd#2xBC4k?@VY;gEHrhbx>obW@K2}pxK$TQ8@)G`*VX}Z8^UZOpf^u-6$7WFh(>)FJD&!-ZJ`Pj(EW)KJ( zF+nRf9wf)l^;lS1QD>6#=)B3bC``|CXe-IbIe#XzFR-h51SR5A@0gQLgE(Kbb_pCt zahnBuNi)YlG})S;uIvBG^(fRg47@^IGHARN$aY^g5>52pWJFa>ffYVvU1MGzIhI)8 zQi`+{texVP#IcM`tQ8|R6%At&K5|cvVxPt^ASQ>NZOf8o%XW0DF$J4uQ$J6_Ta0A_5ES_eLE@E;W=0CvVd@0Xfik9? zl;yWm>#g39p3}#m?S@E;cJEM?x2o;JUSVnXPPyQC`Re?cma3Wfe}goITBSkJ?PKp6 znNi{I8!1#LUXC@(a;y>L|0fVU*Dp%zY!PyRHR&9`nupAqd^IdUfpn%Z3s@oii+n6x zWzTj@u#ra`e6ENh;!YJw+=UJ?*T-l`a674GWLp`My!O||*tU2Xe8pY3NfSQNdwxgx zpLY?rEEW`4lghHBhmqEIG`=I3cCPG(+|ez%neTXs6O1kVg6>9d*{{i; z1Q1+`Bl5shQbZ5~&&8SHJI<~&$QsZr6Jh?Y4PDwl!7a(Hje3U-Z9{;u;00ZpO}+)O zxE2Q-H!w0CXa#ZLy<8PW3ExeH0tb!Pa-bn(oP4g4M@ON$x}1n_?$ialP`(y;Ev>xq{S;@4Z88k|Gz2Zof`#xu zX;neq4KodqZBdxqSV@mQMcPlFlhsMv<#b-NO;9|of&~{2GfoIhLW2-e@|`VA;aXX? z^Eu6RksznMh1g>b>Uc3eYLQp+N4!w^O7#@?Ir)DloXiS(T0DC`G)EmhB*&sLEp@u& zbnU*xD`>nqgp;b;rUlPp_H;|3OyjH&LI;{+jF(#Ym3|y^*_9Gt1i-6U7hP^~23F#* zT_V_s-4a;mu!)lJaN_x%7Q{&k3ot|#FGo8N1e2}^L`kPnfg{Hv zT+5s7phh74ia!HUuF6Ei7DDGfXW({-Ibd7HTcumblTNNIqh?iQ(?-C6=JXGOxNfb6 zX}+_c+3qpVR@>(m zzL9nEQ~4=%N>t^9Pgd!aYu(6!6lfb4g5AYS4xdu%WVFWGsse=zZu(6Cw^!$$b&WDh zdQrZ_82TctBfq_hd4quciZ~0$0$o1Kh!E}n&Na*Z{nif36+s4InUp-D9X{0bsYhHo z-boAOgyqwFP<>XgAIcQXbtGS4Ce|c1XavN|23WI%u42v+R1QlN&o(E;1bKMZq{U>s z6DW3l?c45}(whO|2w!~9HxRjG@N+N{D?+)@$A!x37Cbqn9Rj3}d>E(5M3@^e64X5k z(MCCa(1SL$=>;*}tncXQ`TqhHLSi*0ZR4HxYrK5TMNnB*6?~v@byTuByFVYGRJai^ z^)HOfxS{*!i{}E-7GTBYpWm}V_xhMfE&{wn-D}}bUfpY}C6mq+&aIIjg#GF8# zv|C>xP#QIm>QA3(p+zX=Y4xnLg%t$c@a5TxffN48zf!}CGFHP2Yo2d30dbdT^bOX0 zZE6V<5T`-qFDFvR_~yq`h=AI-QGqdTUyNqjrm`IF(oIuX)gHyC)I#Mg}HRM0Db|n1r;cS9cICX|t03MGD`WX)28mLSmkVDzRVlDIp0DYUaPh z9Zk@@n5@tjJ3_g0(FMUa8hj9_2Ig7@aL*Es5to=u7k1-u+fT)5cPgx-A`}a~mqZZt zCW9jBMirauJ22JEnl|?d+FGabcpRrzd&DNrK%Xwz6zdtab)L#1CjTGoo0KWHlQd!D zY1xoF)J~Kxg&6xQj!qNwEvhPm0dx89Qe2kV1H4}2`W!$mi$uhFwNqHSt`B8_jycdf zqGi1D(dIkU2 zJZR~0QDz3|!WK$^l~nIwVaV(#yT~L8urvf!MXPCzAVN3~i>!6N`bA(?!cwxevIrN- zWy>y7U@~$j)KdZpHMX`-x@=Nhfi3c&)0V4l3f<|ylE`f>PQ@JLFcaLn%0BWrf?dg2Ao}aZB$aMqoS(-D9PIX4l%iw;1kSv))G0oRd$p*N{RnV z8_f9dL@cq8o5IvHKifWqu>UFC+RQAZwr>TbjTwKHtgcZY{sM`VQWZ3jX-bWp zV|^CJxkGigleHl02F)f}Yz$nqOsnEV4|ffX>;g}xyWf4k2q!S6e{%K@{XExU3lmo}UoA}32S>P!^w0Ic({D6ba%vS~Ds)r@!1Dv8JNYRd5{M7$)E z^ctdY3M|Zji;rQ*g;if6l4TjwEEv4G4ZH7`Q0o{t{B0xcA(u`AhzPM&GaE+gycusMkaD-N1Gz16Wi27tbH$gm9Q_A zb*Lxo^sxN*Hm0M#gN=eb=+`G+&OyK{@9;zZ6G)E~6J}iS;~U9Zxdl5Uqc(1yU}o3} z1TE+-o%RGJUq4vWxL~iiW$!HD^-P7!ApeN@fo@gA@G+f4w5Sk1cq!aHD)F2_3nZ(~ zcToMc+Rqu@>AVy_t4a#TQhsfTFCqw{bJjHHf28^OQ!Q(dImp9(-yI{mReq3cw!=db zRVF0mqUJh&1>=;xo`{rvKlU+rG;o2hpox0hz>;*gKqSI1%8TUuqAeqi+#e{pky|FK z`5tdzGGRUd;SuNko$ZeMrzx6cHmv%2VV=X-p%YY)uoHU{tJK|;cse_$n%*7*cc%Q+ zA&|Q1YVr2xl5(h_tVi%k$q|9rzV??9HCg{JTw%UI8A75!nS89ETiE!$Ehk$ms;IL) z2h^}j5Q+M|L&s8ReAC_ks>{P7DsPW<-@E~5jp>5Om2iP77YfGS!Au0busc)sc zUBFv-GaC>~jZ~w|jDTp8W}Y-JtL?Yv7niyW#>Ss~;uM$e+=LW$mhUB(-YE}&M6sOX z7q8E33<1O5&iH#B+K;+c#KOB)BpV%P97d{WGW~zr zPa~J=|Nm>AV|6AqO7H*rqL3|ZzxclDLZ8b-Vz)9w;*x|Q65s_2fy-WvYlZIE#baa4Ku)E0FHlQCVP=8{+Ff$w%v#y zlJZxNu_^*a=c-70r3NxkC>Mk5;@FEd|I)H`BOLrCWz&FD}xknB3o zpJMlqE&NxigdFPj+cqBk9<)8Bo)o4~D6Nh72Wix#bX*EMk-Bbb@!wQGlLT3TSzb2> zC>LUWYYj#5+Id=be#T?N<0Ucv;HH0A8pwF^-|R{ z{=2YES*Mm3Xbn>hozZYA_kwZxhMYZ+l!e3o@$EgAr{#6lV*mOr|jx$~T(nXx9BW5_s+br(y& zV*lj_PXk#?3BZ;vsDYN&=ou0YxHbB4;v!4~qW@ctly`(rWMbuFH`{7#z>Dvc^sI@AIG?6R*RiRy%mr23q1{M07sdWLyD!In<^S- zviN^l&j-qtxVVE=teb>;5A<-D%2FJX7IyF@zJ((b8sL*30A5)Y8g^8ke6X1aky$Jmy0R9A^iG~Bh z4LM-(%{+*xjRw+5HCVyWgFYnUte8~_d*RXSn8A<(>Y4u#zc^wV$V;&&%$i@CxWYf; zuIw74^HfBpQXW#XPXGgaL((gps zcLip6S##tePUtG2t<9N?+Q5uI57G{8HX$t`Kqv7dPA6Wh>K-y{()f!+J>+Dz@cgg~ zn3@|&RZvomV$r9h%_av^VAH??d8rNrHm#mK&dSp2MX3v{U8F~Gb^pf((snhB=Jw6f z5}cX9!i?kV0YRDKpUbu|`ejP6qhEN1d3n*vB1kf;aTI89yB$!H-!z`GT>qnX8$#SI zN;tvUU`)BB{~yAQsl-c|gk43u7da#mb++sg2Kv4Xm?-x;APmW?msY%taCWsQU7n|? zl%3xotusKcQawG64+&pWj3CbL>ixSXZ z*M#tFNcPrM+d3hu?mvZpE^|i?YCx;~1D2oqM&5@z)Bi`7MVmsE$(P`)m;$b&^eysb zv`ESM`oTqB5r7W2V@#Lyp6Xj!iv4e#*&hdr?U#&Vp-TgyiQh9(RN>Ku{oZ!(5!Eqk zUZL&dI{qP%#!)`Yo%~kGKtG=aCCb+2B|)0=ls3&h1ljw?RL?BQ|3-RmpVJX=CsBX! zhZZu^3_^@?Mgfq8Bk@||ybUWZWjBZN)jEUEo5ku3B7g6P-C+O#x()j~Vk@`azupV< z_7Q>-#z84+Ja6vL`AxF?7No@OV`_y}IERjpta!Es924wO*$DoDqHtz`T$}xL#R~%@|%Zsw2;c zmxI9!j)^&$Vd>xM9Gixz?7tPKwP@u5^`QS$w0?EuyPH&bB9WCRAFb=5E*wTDa%u&} zKlIPhMVEZvVouwRevzoo?3vj#8)2CcHkl}sYa6OiWgDlA5?+)q1S2-=1eTmsWY)EA z@RB6CfYMKdb_t@X&vacQ2YfkfJ+62ULo@H4_s&N!>ed}y-O*HD$2C6H_xF4w1Ks1sq4exn^(zm_s-{siL6 zA2b74{2tg^7G-_MtxO8VEqW(kYIF{AX`*1@QrcMxNt_zM6lUap5ZRp5^j5==szwXN znTHOP;zrWvAs!=#S(6Wril$7RBF}_TKL@*@u)IS3X-X*~TNZDaD-|RgEFYYx@DJyX zI=D1;LUc7~=MF6RDiQR;RPJMuxV@8>#2W_c1-}->|HupWYn+Y$TZ<5>$qGWEsQW#P+1CylS!Z-U}81>C$2u|XfoD}>nA%syQg(alr2S%y) zc=yk?Y043*RId%nktW8S-g7(5)D1oLg_z&Ry%=3^gjKjj8LN)VFqH#AAZh!rHd2mm zObl?9=@w%6m~7f6Tq6H#9l!jjs-xMX?gb~&-qXj_SNZy@^V@j;v$>q`qKfe{MW3kbRVn3D)4>;+t|jjtrJ zgjx(JFE-9U{l3mj!*rzAL$MtSX284-qNI0Yv6DcLnAnQT)6d&|L(1k@xZiT(y215) z5lc9JI%|L*)-Tng{>@dI z@=YHuNB;_!g1@tbVh+utrp>9g+pBllWW3!(+A?Dfh%?`OR_Z3iyx+xE_S|Ig z#!Qh^6jwx&5_F~WKrF&!b+IW|058rYe5H;^m8@E|nD<0c~mqr1tN zIpF(V%D^R*V&63kXyy;P6v^N2PUBG{o z(_c@-HcgJ|ZZ}G!6Z{6%;JRzF5M8itGE&lX0>t2!7YzcI1O!py4`i-Z9=Cfh$hg`Ij$bv3 zyRz+{&PGu|pyju{eWf4l=AAQQG4t%IWBs$M$!ei8U6ldO>3mFFPW9MfEo~;GOO)Of z;lRkFC~0|Gurz5R99VU8{q~^1O+fy+g;{R@+b!-PXmMLCVk5toD)YA3w7ATRIXPC^*y%0PltvupVkcyq zYWLZE_d+O@4Ipzdz>|ZS`TPvA2Fu(zcOks2DiU(_)w$#`f3Km1U^UK3g7kOj&Bv<7 z@o9?-CAxwBQv{~aLi)*;kHdzV)^o8u({7vjW<`tl0Q7pp<*?05PTT?9P4L*cyMdTp z&*fU&%m&s_2LO+iL!TJf1gwD&=$NkPulXiHO%?d^#TQQs#Jy%M-N0dy$2R3I>2sdu zs)`2e=o-Gc+*=d_ksdT@LokQqM#e-7UaYTH0Jd0 zQ&v%r6sxJU0sO#bBX5xf*g)Rwgy8Ee@9M#PpM`!#k z@zu=@xObgl5ORQbA$41tY4B|&1N0y~5JMHYYk9~05;GtlN?C_=lGAD8+YKjK!m}X9 z^k6#E&VUf~H35N=p;(G-KYnSGbV_!&8ZqMDPC)ZU$P&fQI80gLK{??m|4`(+=vXqYmb} z^P^*f85zpC$pB#%Qqy$f?WuI57IzA55YXW>P zlX_{SRse{_0c5bFCGpqMIoKJp_fPpZ7QCnhkA_}SY*z@uLxdC4o+Vydi;~u)@hX*f zf9y*MG_Wd!o@y3c;53y308Ed?&ryPG>kE>G>=RPaNDtOzpe)u^fFsaoe{Ba#Nzhwev+D zLZ0*h0zZ8n&_O?y(6!{^k#o_h8rq}LTA6+R!BGZQ?f^5m%lz$)iHs=zFglpS0OGC$ zcun}yl{<$yHdROH>l9w{-jBa~BcugZ*(dB}CxPVZ&f$YX^K!%#kiWUhGF1G>sTedj z&S(_|yL9Ody4*2nCy`_4N}U`FTP@mZ7xLPIKWX{&USj|%FU$j97ZtAD3FvS1$mA57 zY|N4T^wB;bAEl^t5%3~#^_}8Xqgd6%__RawM5izozEEbMOtM>6AZL7dtEw-sR%uEZYZ0XLR@J1F z4!-mLV!s90w*g|2BQLvqUf(=#hy6xQb14IbgYYM5i?l_gxy8aj<$i!)2o}hoec@)V-RTB`Y0=R z6fK9xKx7nt-pu{m57RC)ReY-fQOW`1zbR^?em(*kdm|DKorv!9tp^hUtFINWYR0Hy z;?wfWP#|R^aQ{1Y?!A38|7DHzrBlXmF0SYKWe{@ZI9>6s7+yhB!0XPDIT!7q+1w;< z?ynld+RRFvcKS#JO2F>|_K+GqP$DJ@ct!B+Q{GLkGy1Zo9o@7SN|#E;#OzQ?s`c%c z0OAkN1Bl;T=l&1O)-pQ zL95f-+)+JUJnl|NHgWu9h8}S(v|WIT*)u%mW+@&F8>QOz66$bNNj0Mz6T zp07^(aOUrX78mKYMt*82lVGt&LPT-d-w)}@kTR~-$k{=BSuCi?{q|M=rp-;a6!nT12PpHgHq`oLvO$ea33#yub=G(tUzl(F~W_Bu~p`xpEIPA%DL$Y zl>-DGEBVSyJ6q`szrOAkt)!GCVfYHmWK7J`g%5-@7iH#2&ls(oMfUKJCtq<2jms9= zgW)zA(6Fm8)4GD*LCR^_G;Y~DMJM5IZ*PQo{LFki^qScVlW6=w#5DZX6ZF97L7e0E zhue6}>dh@I4Eda1$b?bkN^7a`C0Snj9;HnFTbrK88!>9YWUk80`O*$#gBtu}y}ANp z-oqLlny)nT=t^d^Hm@LgeLCz1*AcZg&kRqLv9CDpxjz4>@ckWPLfeF3SUEj|z%5_A zgoMW`REV+(f^nL9@qn_r@$oDx3n&)fm+JFlj~*DMlC5|?f#-cOj<4sY@@>+bQ2s!m zcfDc8=30GYdEB3?<)V}N=B(!6BTZer4+V6emLDw8w)D=NWz*&DPKQ&~>I*DwIX@wr~0LoaTk?^_B19^a4RKD=`qSs`$v&;mD$xH@uKX$f* zCu-Nu2)M(IZvW=IK8|Pp0o8B=z79?@&Rk>|jG-bEOuRZ^vAR4{ip{C z_+dDgTH&_WTF4Uikl2`)?~~sPnwoFPt{wuxSx4*pJ{K7PzNhwZ#pD=9hcP9VbFN%X zz|JSXzj|z9iFOp)F69mm3;EbT&Pz)JW?Xk|)H=B>W)g1knOVrhOGw@fdVs3&YnEdw zPO!@Z!d zi}F!w?lKdGocjs!mfkVq^U#O92U5fIQYMq0D>2zZ+c$jS`nX-Y3B&wL4Aq2r%Gz+| zETa9w{Q;DD_^BRFjA#t^DKpNXmNPM@Xi_y>;70<>@~#-<7RaA4763f}!@Eq=jfA9b z#xn8~L~6OIz$mrzRhg^|){Vz9l0bftXkb}jF-e>*j*Cn)!H`rB)re%ADQEmLU9720 z*7@X~v{a5;8Kv+%^yyp=z%|n8QqurB_Vl}{f>EMAWeHaN=;_Uw|jG(Jk>zweW7kxwb=;b8I{lrDk{9xb^~FY&^8!k*crZ0y|P` zA^mgtRU(tWEFWkYErX^43nov*=TBxW^l;QF-9e|DsZ)_cgtl7EG9D!b*|g<(#GsjO z(DXzp_mJrk)}Dhi4H7VQ@ETwtv%s{wnU#6WpNF3?JH)^CoTrOr1dxtaW)8}p|NE!} z(ApD`SDI`i%xBE zv=_eh#~pB$y|(iYEcY4Q<#(0?l?=zl;O5dE`!Vxbqyj0ILwX!OJ6 zALPw_TRKXs8@f}nTgM7*)1zUIw^~HXn^gp%Dd+@<`1FaGcqX&>Stz5FN#O0=0d@hy z?i^y}H3m~rHeWOZ-p%pN)FmF!TNk}=?@!!gcsFiL$=4r<0d$tiUbo4P9^C%sa1Hvc zn|VlC%48{l*k z#_@YR9}f_)*b346QXMYOYyRse{m`$UltNcNc%!L0T=EPlg%0L<+ySz*QHCiUE>8Lq z$L>9`N!WX)hdZ_@YTr~MQw^e>G0byNeh>L!1hd-9^6+z@>IHJxRQkL%|ZVh|0KCDSaIo6Z2AQzTW9NJb_a9Z&>oh zSSWq3g^^P;fzeVeSsb?&o+ACLc5ACtYara8gnOi!rH}u@3yWf$dCaugVh6S9j7sPU zg+9G~i)mip_-_bxrN0s1&o=LQjb&axhRRDxy93}SW*X;~C@8L_xaJ`NPD&eR0k9QS zA-t_Up$E(rWeG_ZXWn&5sgLZisxiVHbQ((LcbK`#{2O46{kZKFMFob#Edg91OCG)K z2E^H&fi0p2x87LY)!h&YfttAd)H#o>bZdu?t}zSwa#s?yQt3LFJ%wHxRGQvm>X5mq zEdda^!DE};ghzytBV)pdR{1#5bXpI<7vO$Gvf=nm-Xu0*W*SOc5+J?o%DtJqzk0h` zOik97wW?n|4<~7zuCKc2t2Z??uelH(zVvHwzeB?Fho} z?-f;!-wQ1tzb62M*?Q(76{BuK42-Re{GZYbW+3H%*%rXA*5D@6lLm~m;efUp3(U$L zGzA0xv{O?Hx6Cgv_>DfYsyBQh$lCG+{xoh}AKDtfo}%L=kyO|+Vacf_Pqvy=g&p(@ zh7c9k0FpfgNH)y7B2n!{tJ$fHAF|6w6`o1VEk(gcV_ z7R9VGxhlSzQVQxLp1TWM)RrCjH6Zd$o-g4C(tFFRS+gB&kr`Ut`Gf4d3YmE`V%e~p z=)+Locmd@)0opfrKN;oUaLy{wom&?~1)fmi3fjF3bNIqkmZbQbCF-4SF;SJCFf~4b zldd_^V);I2;Va>VX=9n%(gb+H2V<77m+F;GddRK4k`_iptO}D^zx1jkQiI>Hzbw^L z!g?=3PT7y0_Kk5-+0zNJ1Yj3;E7+t?&jtowc6^|hN;Yq!DeJX!4lSH+YK430c+2PF$m+DONEis98V1?fe=591S`A|bGu zoAk|CIg;7R{C@`*nY_C+V2FQG8~ydXZ3gOg&F1|sfl-WLyz{z_ig!D&{L&xU=$CcZ zBYdt5@YNg(JSvk7Ut~>f)3i~cJHAX2mq9*t{Fe0@thvxh@XMmVZhRy&TRD z%A8S>J5IP=CPqM;4VJDN50+{RwNSnv{7oNWNdQU4)KP}NfI~A_9kbrtpELa^1&XXt z|JgX@we2Cq^sv%<7H(f&dLa@U(mG&lRDC za1%sM_V`=V_Nt4@!!ozKF-bBHlfZ0a)1pSFv1u-n+HG``o-_k>QlNp}v_O}>G!Unz zpC=2fk9Zm_#xRU2!0^#j2&QX`q zG(s-$cKh`~(PtiPO*IPl_@3=COjsA~qTDWLl|R&C7OGQljO?*%C|aN_81lLybk_f2 z94&{*E^ntbpnA67D}t3T(oV%>hv+9HSz)r#rQO=12%0Q4us9sC5{2m=cR;Gdfus)E zw6#C~!grg7y=7%|&F?lNv$7Wt%@G@)NS01fc(N za~hrGY3?;99mBR3)G8+td0|AbrAp=5+8?)LMbvM;Z`8yUQiW+MHLX4UZ`_6mRl&C( z@mq{Omcw_Jz>UWR(Z^gOgIc*f>)cPzAFb+`&?lNiWMof9+S}Dc7Dbh~wSU+#Zyj}S z`_c@2k^()k$TXYtws#Lu54=1q>SQKfzAi@IZXD1y;4WwNz7GTuAXx25}LWU#LzU;b7mU8&)27~aYl#^A#Kgq#e-)Rzug z5q1(V)cIu{Ptn72{5>RJxoG9bGoc8OWt$d;V&=bcd;^ReoE-9y~)VL0#?w~rld0t{bn-cE}=5P zkS_23J#RH?pYKnep;ZS$Zj!n9($(^_KNJ^8e{|fGaLJ0#YW|U4 zXP8nKhi0884Wdq}WTf~nxmvVUjZfD8e7X}F-dB72?CMY7@FvZ{@a1RHEJgwLqCdAo zf>UHAJQc$3KV#*$P>SQelNn)3``b&qQ*?Nz#yfD^7$aTrtK-*7hwEVYceVSenc3V6Y zVS?YSsycm9RN`y;-b`aM`=y1-uUei#m1k5!a#oE?nxv3t~>vy<(ExZ$BQ+CmwAj?5X&G;&RtXMDDA<2rdDIV+qwV<-@ z#?&B4*B{a#W~D$5eQ7b=nv8+KiFtoaekD&+f4;HfCkb-@VcC_;f#Ma1nOwvPYf(j- z+B0trL>78ZB8@A~s)k(rWu>UNkb55PkB9E66zC~IJwo}ViXE*pkC8T$^+OsKexoLO zY@b%ocWM;=A=F};8UCF6;qIYMwOjc|-xPWRn^ZOM~`E7-BeCx03-W6Z+_=6-SMkz*qn8RN8&Vy4T*eQC+PbW>Y1RxSr- z$lBNm<4(WTZwu)Y@!BJjjwUk0R$om`Oc3m0F|4;XH{WcGJU#VMM)n2@~=S+?{`wKCI_WL;-W#tRKjOjiU~gThK!k(*a3pX-dyJFTu1tsUbm|>TWK6($jOMIHhZ;9YgD=YmIc&l}qOT zBJ={T$419F1O1g!|J-zbaZSCzRdi>MBdP0A?{3-2I$e$Ej_S?*16G4-sF_;U;M>Gw zO-UE^(r-0G3Zs!$cGBM?1-5;dtEL3rm5IOGqz;5yBvzf0m9fvXJZeEt+;&*59V#Cy z6>%6J6R{LvoDLDi`W0hJFK3d^eO;{QcC6iU5@pR#?kQrDOxsh4jKvwuFkZQ8@O@|Y z`0JO9DyEr^X=e|rLs=8|F*Zp<&vMNqutBaOa!OimYFeU0CU&BXI9vP+643^ed|Tpf zng(zOt~Vez=VhQ`#mWeC-1r8o7ES*Zs0x+(|gMT>HJ_U;a6n*O>I< zLZDIzytE)_ZdYkQKQ1rvMw2ltNi%~WRO)XYs?uqJY@BBlVnABXbW-9uE(S4g6pQ21METJm z2BbjXzBJ$j2mzsr<_ew!5Q~`;|L=>n! z?n-eb+w2&kqi#+L|KAf~ryN6PYHK&7IpaxX++}pIf1=e=9wT`U$(MJ(~;<|MAG zj5vbB=`-jVRE$mI%BPvzl1~NL=iYjG%wNG0T|J2vcO%?$NOiEz$0gee7#VKO$pYXvY?qXGePj7wQi=+7wkfzT3G%;Z(E;(G&nxKe@JiPB?O z)9$@BW+~2`ckS_WvkhC|VcvV`p55! zUs?hDQ)p7r@G2UZhUDn0I!c@VGU79pX3tiNbJ}6YR5~|7!HqFlL*UU<~rah28)i!*e_qiQ^5C8rtVwFAU zn51mh%|r9^fiqQH*lT+>(^aE?VxvkcBx@c;qArH{wEmpXQ0Lrwe*tDHUwJE}kcuWk z+=eK?`l>v+M@$%PFKouLzH`j%=^z{?X#f^B=)vpIsu&Iwh+`|5Y4x>54SH(l7&lfk z45$9rI#Lbxf0|_rWh;q;PHz<->%s!K5dXCnC!)7m+JQzRi;-`^pw&VdjPg&;Bxe6Q z8XS_A+nqH}cn-(PlGe_pgUPB&3vMxSn7GW0-}M%~DHJWhs(qA< zU%&Gi-5^xwI?R=xW%p%IICrho)Ta)iXTf#b^YSl}xKW-KR%>sN3$bdz*qsZ~%slA+ z2drs2X{v!CK)0cM;0j;Rz&P&UxhCYTSy~N=b>uQGX!k7LX5ex5uWohM;Q|bnhIxc{ z*aA~Di7nGd*|`dEv?E~lHToz889jCkd99GEpeVKT$%+f6OU?g#e1vck6oD9UiJ^wx^aLJagRa3)^F?wNVksOw#H0 z`>s1{0D($~re`gIBsvg?!Q3^@6wzbz(OFSMFcyQ8hXM~*zC#8+0-2^2STlv?Ye{dK z6T2qHfd&yL1>yistYY=!wkaa0ISM_{!2Ox@$h*$ITtK4YdzAOt>kA%&p^4Q8#`*)v*Rff~Ni{>q zRw{g%K`6jSi%RBnHW(rji*91}iYS7&4fZB;aV71D*PpJejHE==+{P$H_+wXzd#E9LJWD^MD%WLi9f@SHwjCDOB{=|CQ&aaFwns==ekjd}(lZvren%_4E9`7f+-un$`yS?knGA!Mb~Wa9B< zts@uzOW8~EbYibq9+I;fAa)kEovry_zG~;00pr)i|77>6yA#*!B%{?eU1!ZrBX~T+`70k5^CT zE&eyMN2QM%{zZ1)m)c+e&;JYa(W&OJ=g&6(y#I63`RC-C^MAPGzfE&<5tyc7lx zz~#hNM6P?KM?Pkv>5^74OK!n4y>qHy7q#>pp#d30L!)5KD4s*4IwllWIu0PPLP=egph zf00}eJ7)v80TKO&(xU-FJh1}5-3k7!2oUUFZu&2xs-eQuOi}Kz!o~>!uT3@>YK1Bs zMUgx7Lmp&ZLWq;?mUD4jkS_Eh%X3qy=wlk`@QEi!B%a2AVl{_4zCm)hJOR^{meWIp z`>wD57{8m;_y0n;5vRGxS<31R)$iUYSLN=reE3+keeLnn2|w$LwfZPQ&(|;L4XVd9 zb9u+f503_~puhbb>la4|=cw64+0wa1pW>!BJx`UN{E6+&5s_<-yA1$uIrnS0%=j4j zsqEvw=zF&H@b^Tf5awd->`xcbqt^^WUS!=GanCIj!u;%j;eAH2t;U#(n;fC@5jA)k z0L#;TdT0Ar!J6l|ary^=j3eG5Jbo>k{y%{IY|HcKL?#a~bh)FW=*}!fv=>=#(Vb5* zVld#%_iJV?%3b)}{|T{b2#Q}%qy;#gRw}~b>7&n;ER}9Z6fo6$Ydg7f)cwUb3gdB( zLxhJc25{MpT}<3#>T}oi>;zcB0m<43ZuB{>%|!A5Tmx^#VQ{8RQ|E7d*m>4ce7Tk{h&U~OXT6XV?MXg-LOCZ zTIkSHkTDWvc#1`mf`*iuLS~a+Fgv*maGB$shTNA(pPJOs$7W9A}Gd}A8 z0g52hght!Cgz-LwCY{#QS9n{J%FVlfkF!|jnAmYwx2BcevAGe_y(o0g;>hL~mf;|M z25JJ{k5K(!o0Di>V`N)1#OLOP{6RR7&s!*F-)@bQj zTGlz*rP>H3T1qZUcSOwveFr;ybf13qadXmCDDt1~4S~fZon=wiY$xk}PlVe6_3AA_N)q!qsAJd!losl6)zh3Yf z+L6xg?@?_$!N&LGOPPGEe?QpXHC8%i%#yCQ+Ee>iA>S^Mt1)E+zuHErv5{~=KoX6Q zDm~W;&3pAQZ}h+>Id*K^nYbwhDyY@`nKH0B17z%S8S+0lg=FV`Rt@6{8HTPrwSu^H z!)^+a!$T2?F>zz1RvvP^XA#9p-x^v=25z7AqKWA; zzC%|Rrn5*N8x?BLmHCTC*S1R~XF4_nYsOY+_KG8Gn!67r;EhHsO3SrfJgO2FB=kLC z3~dnM7v$78glBE9v1716ldg&|JWx@-@+abRaD=8a?AUHTglvEEok(_8lK?sO2G@0= z?HL&qqgvJpOCk!^xdKQ`7uCHx1VvnoCX+U)aGxrb0m@NRkZhGwdptVVev;#5#LhxD z=3Fa;q3K{61Yo{jVHu77v7*3LmGzOdY6 zoX>3-qj8q0kT95BZU%)h><67XX5k$jY@j~U;seSl;GkS9KcK|n_x!Fof>M=;n|0in zCCB2%oso)hAaLMT-_2+`gTaa>M`~yVZq44(0ws{ycQbz7mmVF=g=5zZj$!S)`A^Gm zoiP_SM{Ylz7{4wj{Q#eH36~?ciXpCAO*SNe4S*cowDVw+IYi%}R_Euo#aB;BcPd-r zdAR}oq+;7c6sQKw0o8y?dd1nSJ2{LS68PY7vVB-GAUFql@*EzWrt#@qoHC&cT-uG; zm}IN9IfZ9I!5UC!X>o*N3K*Jeg#q%Wl7lK?sO9IllyBAUd{D&{5P#ozDmf-?Rrk&y zP)c4kyI&idVz&G}%~dgeJ&*z2VjE-S%?2h{;vvMQEP=QQYT$rw2jDM2EJ?qZ0MDwp zp;6%|ExxUg0_M)On*5$T@&K5Eo1pX4@R1iE@G^+IFYN)B=A*`C_5AM7Ka)S5Q%3_V&ZKXTVG=Sr1} zb`3orzjhXi77a|iaG6kll%SRcmv1zhc?6QN)rz3)SMJ(#V?m#&U;J@@ee9D@;#;~^ zx5k`ePqne{nEOTsS<5K*c{7>s>vdcL2Hz{MLkw~giI*_MS;DWC-`5`i^|~2G&gW4< zW_jP8BJ|vWWZO`!GwOu8?xp?Fhkr1V;|B_L+DnJT_smpvqJ}W4>>qcuE;8Bkftfp; zQoMRgxs+2%nLl)6aIT(^(!VC=^0k}i3#%4NF1y6$1*4MM0UUWd6wW45f=hrlvr3 zkf;}ov~@WG>S$(EnfE^ZH{yrA?{d^tb9}P>m-H*MXS-a5C|WuOKuK1R+ug7MfUg9} z)IvLkMO5jydMv)5aobhr&S_MbvKvuK!i)PQUAZ+t^a??d=a4h~L;yIk=-3NM6-l;0UW1i2Hd{@o1<^X6U6aZde?7uBbTD`g# zR!~-%EcPf)Ak8|$tMJh+(S2z&zJPN7y{nt4h!p*Nl^;mvk$7xeT_$rYJ~hT$6704G z+n3hBKg*>n-B-x;>{X8gbI#FNsl79`rJSR=te5L+Z?e?CD*sW}`X9>qCxE*B`n~P_@Z!CyW#8v#J95jOBr7vb80$=0vbHS9 z;dkA7=aHbt>k2a7_O@%Kk%P7#JRZ9q=1dVmJXI;Ie>TbYUph6wJUWCCEDvt}!$Ms9 zlY)G`((QF?uaUGSf_CYA1ZZ*$cxIw}TN_(#AafGZD;`8TpN=DWPbAR2Q*W@E7IK#< zv>}ObhO9l~@Zd?_a6M{YIIEeHvbC{w1~SmlUYEgB zUX8)Y#t~*c+tas2s3ua0C%Oj2bKWs@zu8WOe{L$<&sv8vAJE*j{=$$HCen*uPL%rT zC`@uGYw-<;Vtv`QxhKx%;f-ERyqZ(FGE>XTTWYxlGawUcdQs^+Fy$+HJ2Co|;^3sT zT>ZhUOoV7;NW=b1fz-di^bQZW~V%?}CI_k}l zRBzqo=g1-S&%Q!UY=NRhf2r{?CVb_&KU-qWBSSXV{3v+M^PV@e}i>4#@%nr0Pti{AbQWd~?a%8>bH&&zb83K#jso-?dljfAQ3|7i)`t zb7L4%p6}M!y3s&?T`h}}jp{CH%SG)Dr+hXe06VqIILcSs#!9J<0~?S`PjVVLT&{f` z2YIHPqp03;T?@%c`T<9u78xkFR#Pu>aI}g$XT&tPk5$eTAHV!=$fa99eKYxl^`9ib zxLe&%^75lbble>32DZE=j&{jgh!xI%z*hJG2*eh+`!H<=qAh@ulAJp$ zF^nPl(C<}_*(IKm2ufQbu+<2Kega+h0D4-$YT!s=rr5PsqNv5OxDfjndYb@Egl{QR#|*_r;mHE(RVW51fUdZBc6y5N_Kdr}L&7~~^~ zpvx{mY?;NEKOb>x#jR;eB9^XT*IYd_pQaily{Mwu$nBVKy{_pOL^H4;Rq~=r3gq9H z#@Yw$DERQ@cS$Q2S;ajs#O?9%M&pls^$VyG z7YEm2*JM(>MRvJu^B=X5KNXq0Kl4dvcN`tI^V_9>BT3YTc9)t?wC+C3XKVT2-)5bi z99n;n|7xEkICPQ&dg9D5T!d%z+p4V%S$S$nvvN+vYcrBlIgNHfPi+Qi0_18dUlR8N z?>b=c;sMVq(~*!R4yA2IE{=eXnK`oo4QpWex}T?x2oKLLC94WIJcX4IyO z&EV*&RUL=poLGUUnq$`beuew%M?@Y|)7^>Rb(0OG5~x0G>MPLFnk9t!m;4)IZU7;s zka^2~pIc-6tzpl10@tPLPkLB}Unft>6yIDryz|U*bIkU&S~Zke7g4-iic>)Rgb5G9 zTMQt#AEO3cY%qRYdZ?9;#aye`3UdB@HZvuCqLGzVnebq(&u-Pyye>~80Uz%4ej#EEJpy?6ohN>ylW2#iEN2FsH zMU*4q)F_8~?lbAb0VhMqwHmCWg%Ais%po)gxAx~I8MR9UN>%gwp8PWJK3T~bXKIU9 z!q|si2q$LBzd}Tg>A2Oh0;$uO@kOlK>U00HN+5o4zz0dqqDa>)fAh$}PHX^Gjr>@b zcK939{J%UIO1o;~JV7&i3$#?tRQ&Hv;l|>7KD^_oS9xhJeM@>1SID#sd`w2BU0Wx=z~_XYnH{{)*2AcVt)F{rd()(i*gmAnQgx81E%eOFKd! z7~23e-P31Q3Z~Szh56Q-d_mh%ap5v)2kbf+uBx;FsSE>8{;Th*w4mL<6jYJfK5atP zCNh7@W(Myikl=?DVtdygiLXCy#oCmwXwYKev_$=@wl2iWLO#ksn6spi>Eww<7@v`g zz8~{zqJ*S6*tZ~itO7|+e$fNYTQEzYnJ4b}1ixc(z8yoV2cW@UiJj?ps?54!GyItq zbhY{c36Udqw~D<$K9j;M9!8V4Tox};4D|)nE!4KewhN_gBh71H6*3pR&;vj!`r*SN zknj-taEzhDhpy2G_NE z(YQ*_k}U&&R&UJmfi)DlAK9%*{#hCI@+JITjMf|>ti{-dO0EtFqyrBtfDVt-@M-f^fRrxhhJT~{)-3M;y{*j#d;keS@8=irZ< zh%cQ4f(zOkdZkhpFM$FDQm`qa?puCduYJpdTcR&+?V_`TRn3h(KgpV-RdQ+ur(#a_ zw+5%;=VGz%Q5UysSDxK zvyiWKl10nab+)Ml3*V@8L$Oc-h#*n|fzU#ij(}9@5)hG+AV?QPdPiweLm+^3q7;PyihzRj z5|AcJRjEo5X`=Ky_&eu$&Ux?W-p~8TCRxhN&SrM@+h(?BtUWb9Y(WoKX1mEojq72c z!8PhKz%mL*Q_CPG2EHl#CXfMIELF9q%PyE-IO-(8v(JH1hI`V<6_xU(TuqNH>kMzHpHBt9}MfWmlW=@{^4-UERGU2?Umqzds+Vr zR(JzgAz1!;;P`ytqs-u7AG+Ag!zbVs^=66;BxcxLQtTxar&QGov8q{FZ;LMezlle8KJn-n{dqTD_X}66PtU5^mtz0pNxEnq-^A*5*AzsTr||$sC`?}+>Izo&dXK^?NI(~^$1E^? zi&{__l{co#^ERpf-1!~$DLYu27xFkmqW2LQiBa$(YHw;=Fpn%Sf#=!XKbv@=D8OCO zn|LiKq~QWS%6$(zXh1Y0JDfjrc#PCr3N1n{hZNbDe(_RN+i!Lccy$M{md_rq;M6&L z=7M_9HPqf^emKR;s7nMeBT|H@n&%V6)`{>w@U&bd+(WA;|Kh+9hFgSzQ<#VoopKIkc2A4A?hrmng}lDyt|j(> zLw`;Eyg>t0ckvrdV7acEc+>0jNo*H|^DWAy z)>=pFlS*%(=|oDU7L;?F-(Ev__<{t^t=5C_AZI+A^HPU;Z}qksM+L4=qolMSix#>L zPfUO`ZU_+3LK=MQYT5j<&J3bH*ZwPugC<

wPSeON7qPE226LSs@x__!iSpD$ zQVg!4%a+bm@@QJwz+ky0$*XqHjlgH>rRyy8%lR(6gpox|azRwG2ntYe;+F`WzsbDH zW~qML{QnMl0eDw;FJjO-*Si4k+r7grPP#FxDp0x;&@>bbI^uPSaxF|$h zkBKoO(u;-dGFEOfEd^2>I_2jvY`dJor*foU<9zml@3vVBidl>oCst;QD9eCMjt_)k z72n$ypsz2V3v6PW4pI7?Q#4@yd3ryA4}ME4^l7I;>w%l;Pl=DBo3P(bE3`Va6kE2b z%#S=;_?FnSb(Sq|rzW~2H_3ny*tOJRgC0{donbFNojng`Met}Ozdg>bEkbkU(Ri-J zzOD-Blw$LAo%cwmuS9bUF}+NdS~0fukDr;-!rpN=O)W7a-*UR!WAg5+7MA$>{K9oN zPBCMmX*|r-ixz4?01>*OZ>T!Ur#dy~wq+6)pLpvAAEV>O(zyrjKYu&x!k|>`?P^lj!Va zp?8Z=I)-^emVV7KQ~#C--xU^SXtI8XSBJxu6=L6LsjoeqMS=B#CsVo}P5Uf1ohXOV z?E5lV%rEIf8`xnZvlXX&I35vhtPC&4rq$Q0o0e^rW#rwxO18j0j!B+OYqyA= zQAru^(aNUO<<~pce!q0khTlJlI~KnLlSF{m0lCkD)OlJRE5aA8o8Bh+)8#yA->QAH z30f5t*l`(-f&{)zV2&PG@H&HiGC%PepGdEXNV0bRh`%1&>oeg~>SINjR%jp&*KN7y z4=8qz=b_}13~?TC3&-!sj4lu!c6j7}r||+QD0f--)jGSb)tO!2QV`v|18(TI;tInD z*y)3IS}*L<=jr7EtK(m|A%PC1yps3{_K$svSu>OV|3wZVS?NIi5D5+$dq%zd;GzOa zX@0VVE&iYnH4lWvFYHy!dzAe@A1rxamQt<%X$@S>rq<8rani<8F9z$E*Li)}WGslv za~Hk8)YI-z(Lljo(|y(xn?q)#L-vGG@G4YxA+y1ANCU!J@sX@c%0RRnXO{?d`#_ht zSh?u0!#0*PYAPN#^_Kj!E1`UDE2TeXyz-^%!`^I^8xvMWb}qGl&~5@io_@b;PHLyo}~qmi&}e7M2h*{&VOgz00Lzn&7x^ zJIuVCPrtW9FOmwL*7=5D#La(3*tgVk`zYX|4n$Mtf9Km z;p6uoy4tJ&YGDwKtOWP0+GN3uO6KV0BBhGtJUn8=sO<~{2 zh=LAwh+fn=(j6(b8ht*8H>v8fT=LdqFN@?k5s^CVIRWnHr81==gkq+6-lZ(S)J94u zT?=)3VObhf^uDuyQMJ|@Op1v1iR^Lyj(jlNm7LSLlGO+Smdx^!elT76(&^*m-+D0% zhm7|A%^EDs#$-py%&zU13OdeQ3KC5k&4*on-(yGF7}`pQ=`mVm820A#H$Y{`iXIfCDVDvz9RYQA;tfnQ8VD34=b zQ_Vzmm(xzkD5s4*_x5?K(+qif1qFhaqdUVVfHP>n^9u{u#vZ-VGJtje7H`2|X%O^p zm$vH}^Y%DDGt ziUOCFn_o{D+Rjp6=(%u~6cYd0q+KvmRp z_;|YB%-`4%JOjf9JO0x+ofYF7X|%wPAE=Xh;;YwtbB|AzVdnPyTw$4EYr|4Qnb??) zy=0t%d!6Fw7g$MEr(La6YeOuo;?B4^)sC3z8+F+la)-87^WGn_c+Y;PYkAXg7Jbfj z>gOlvAI5YzWh7pxNwc&+>)3L1FEr+n+F8N)M{0!*^)J@iZ_e6F#) zN^#cGe|ueDCp||7-T{ope3=$XAu37gOE%XCr ztfk^Bk{fS33+&}wHR^aH^sytvAvh8OS8DgCZd#Ttf|uf4g=e(~m&P;N`*8->1_ zZZ9T&L6{!?EQ|ZKz_?NkODs5!6Ld&x->pYHMd zbviM9e-g!9gMfA*;x#3Da~Qp7Kjx%S?R(!4QFY6)IH#0%d{3{9yQkXkNRRX0x1}#1 zezd*Ki#~YGQ**NrRU$T46ued#`%!*^<@qIY225PD`3R)xCNid3g{3?FHXx+!@it*bpN9XnmL z&e_2z_mS4zJONh~u{ys+ITSZkU0sN9b_2D+vYbHgtKGB6pU7%?_oxLvSFFa*RE=j! z?yzyZuW6ZC{u;w7N)?5(W|?|h3|&P zIWgLXjrnrx#>cto0A#GQ{Y4v=f&k^`lck%4;%H5f`>ZI->+o`O?K#EL(^djun z25S|20-?D-A2Ox|vz=L~{qp2S1)FfVrX_iNwYjZ8XeZaZRL*v#gR-aKKaFwk3K8K3 z?9VNy^NS*~E>YSSoR^j9I$v#&6ou&KIzUAyQ?YaF$p?R~Jml|lMwfSjbLh%s(EYwz z7=1{0u9`g|xdUsbxrmEqVMiM|NMLxfk$Ji2u=u&-Ob6eRAdoq|r@m1R9Y>7&zLh{$bAU7c^HdXBmbzWAdk z1plv|r^#;MYlCGagYMG3XTgnzqh6r0jZM?t&}G%|NR|~;G1&NQWi=nJ%vX+4SX6l{ z{!!V2SYV@R;xIGe_V{3YD@*F$6OGro=5*FukzvA`1AB3tbzHGGunUnllFvUEteM0+ zbrk!0Ge0+w8k*2xdS0^{cjK5^!-cwtd@>=+$qij7jZp=)SY4hu3|qOcJ&~$LU ziBOquahsfF!R*q5p^^R(I>bo*jeV`k(XZy?qdD-T?Vl>SpJ&8u2Fj=^ch$bKa1aw- zEKh7}zp)q3&c3UhQ_bs2b5(reOlMAg6lmrrSe7Sf=IcFHZfiISYN3yxYaEP ze)RZLT;JN;W(#!2WXa{mq$f*06z&<1(x&HZKoNR=FL@-ZZ&!9Ftjh9{4hfboxnEUD z+VR>qm#(BFci3%5*7A||;##VkUd-1{M#V%EG^NhRD^)r1$QO5ADRMKoK;&xz1Hrc`{(?~-j_9dUZP$7rX3>glj>U(tCTyH(@v~^as?r|QXM2vg4hK*@5 zRoOZBf~(_ki8Fu6G%qD0@p%uWtDUn|EM}b!rMtu=P8MIS2bZxGnCt+T$Zk8sw=&}Y zTq67w$@}NN5k&4*WV;^ejyAsG+sxW=@m*=+vRllMa`~m#+9g){YM0+`CgZ5*6>U5* z`rA!jMy5G!9v)5cmG%wqwA*P$inbLb&K|$9E=ivTGKY?~C)1bODSB7yO63K5raa5# z1;(8ZE((>-7LyQGgULe$Atk3j#uz-%I1kn!d=&WO+eg$w;*@B~pQ7-cHql$+cDgOA zHUjsuTh8^1AJr$i+Q)v=f)!paxfJWHQRbOB*ra>@Eq>51_u`qr53-)Rif_85`Fig$ z#8z8hEkt}uhEwEE&lf`ALGRs_g#6&-`Cw%Eu0^Om4Ntpsj{m)Z`ix>?3Hf9@W4~Yx zq{`F15OT-Ka`pt{J=T}I+S~4rV0@oBcsJkka{@-r@aU0}C+w1@+`6GGA05q6Tp#Bj zVdR6aY4o1H6bYU?dJ(LO{2?(xC6-5WKAMgyS}k;NCYy8oZ{+^KnBL5xB7n^6cV~Rm z=6O$Ucv&T%77a-$ zgtPkv0^?-2hGemR7PxQgLk@GIbypn`MgI3xSJfot6{n2LSj2m$qHB4)S%>5LZfWy1 z{>o@_KKn^Yymefxw9}qu%ICvWbbncEwrGRm&)Br58L3*M6 zt5UVtJ-{nB$e!mug0a*gMVm8TV;RL~(#4YkhK$sJu^#7b=B_7NM}=D%0ys@Fj3qNL z+sUXbuRqGu{1n)|T|ah@+}K^-{LsG{dBuH}-|~$JL{owjz>(6}U;ZTndwtJYS|4xZ*z4y%-s>4Xes+S&NG6yWyL2(m z*nBDKqX0H@Xw6RO$zy#Z_1>s^2u5tCc)?Tx#^^dOj`L$7O_if!!y^+(YwUP;lU2&N zp|O#@6vx?7z<9_%VV&)5uYlZYCTl ztkp_^@nwQ?-5Do8dw`xzLB4BiqdVKorRm!>_^CKaom(!vud{dta#M*)o$sd2Qr!iv;P#*LA&!HZjPVWuzVH9J4xml zCOG5S{P3g{iv1^(5o~ML$Gu`U? zGg3K1hN-{vsCue~7oS)Bp?w)~^Idpe?Qro#3NjPr?>94(5p`{Ze>Bn3*IeQ6DQX1` zIXQzTJFjSyT!lY?WA{tYY;`~|ayg`|v&Bbs5xUnixdeWO(p<*+(OVC9+Tj zi1(YxeTKYoiS*PrF>Ac%IM(YCoa=JRt|;>d8s~Pd(9w5qE{P`^tU1fVJGe?}ZCi;D zmmx0DQ>N^qJw;U03{sG_L&$1n2U#b#PX2mao@yx_| z!-mGBR>EVnEP19in1*Xsmz}GVno-dqi%SoID>_M|6dLRJ*!$ z&sBa}pf~=|nvhpX$$0`hV(`+t;8ucW?Z6a8(#`@;$uEba~R)mJ4+Yy?9DSh_CewUeF^%unge47b?pXuyY2~`byLZ+LRL7xm#A{z-7 z7t$;v0;W98JRap2@UXefcO=D{{kN}!NiaCzOtafMivkw`8k?P#W$WMZdl>}k8_eoI zWmWL}>`_wt&jwM5#j1a1v$oc2_1c5Fu=HZzv!k@HVqw-qIVOfA`+ zN9D#$FZ#dIej@HJ+~H+>VW3Xdh#w4l5=0evO7HslJa$?XHd;0dGxONd(ys{chlY}Q zk(bc0XwmakW8N#NCui`FlmIn5KvNZ!nIGJX$U!|P6s59el@@hOONn^weuAgN+|cAQ z7^B-=JY{1&O z-R&;Dwx1vl#qwD0`1)KDu42AXc(eVp$b1*B9 z(n{jZbXb(jh`@|4GIajgk~rI)Ve;rQaeXtmIfxkj;V-j|NRNo7Dx)PZ(mPNaI%lV_ zj6BxYe@d?_-dTN@r_2~P;B4%>OP#RV6GfyM|7uw;&o?fpS8a?#7a@ACv%B#&p1)SJ zc)%UTAOjuKau48pVT>bLck7U>u?tAnv_)iHECJ)rWZ(oa%azqHVVTjk?_4GC+P_MP z9mkT@Fp~_GV=b{ABx#hdT42wrvw-ji)0~C2i8+H@2RqBx!YJq(EiSr|9PO+X0=@Og ze$P5HMGY&ktL7uOn|Gl}#VfS+mRfa=oVR$}@tp4aQ?KH7>f`?o794ZOEP%?6Ayq0j zp4?olzQjk0l??v2Q@%f{v8Qy7y-b0<;JWjk+~=HC<;~4@WH!$ma0ll5@hh(HU1$xD z~yF{0lRZo!uY5Z>iZh0<@01~Oh` z)HBmNxltR`${>fbkof>GJ$FcPnr3Eyu27$U-4<2fbCnDcf3it>3VJiWbt-wa>uola zr7ZXd{r;hR5GYC+VsW=3&zkYY*PI@VXqi3!X30T2+yQ@+imAdmGH%1d`MIl6&aiHn zZAS_icimi#E|RFPzP8sD769El-kp3<4jkz07L_^3*bb}?qW4DZ6Kl4>bJ^~RbB_o3G!VV1C@eqV zIAF~zhi2_j)bE)$B_GMFKAe6K=At|0qtk6Kqim#D8t!tI22Y=9$NnXFsR1A?jJ$8$<8>e68oTlZbvN%;ptiaW_k0tOE@6{9yR7 zxiijE`AsjLkxqX4_HQ?SOwQXmT!)=yGFj#0^Ks36^S_$K)t|tgc5=NT%+2|w(NN?L z@4_~NFVfrM@0&^5>tCRuP}O?Vz;NxC^c+Ry{3yghbCfsTVm0Ny^#aR*G;Y&_k@Mhm zn(w3RgoyM2yzm$-69tCiioNgUuS|L?o@WjVh23@GHFb*)e~D7zNV)4G$!yi^4y&EG z;xE2qn|=`%QuoOk$cZyi)l;54_&lz`hfFBvDZ|qGszA^7vfF#{4&y&8nQpsg_Gmbc z7^Fay@qA2IAZQr|CcLJ&`4B9#JIZ!9)W%$#OkYGVEE@;;ss#(PJ7D@s)IOD@+4{f~ z{pb3Pc%^lI(nAH0XtCbv4}}Ol5%5N!=W5T@jK@m^iC$_aoaSnmIXwSfFrLYen-t1> z*?5gv_6KACU>Bms0MVjIdeK{P_N`!a0tD_@JNl5@4IKd(Qn5n{cSqEXqNpWDM4J!N zZZ#Z^m*-l&CZeyi$YL1SE?6T}!YxV$j2QX- zXso4!8rDeW2P1Y4RaRwyP?IOU$lo?*!6r)-6epaO{{^aIG=X6qQu5oQAPbHa{Mb4L zd}i+KpS>-KSLwh4&_%?%@#8vPF?KWd-Wj_8qj*5`2c@yS)vs@du^8f04LF4b59RG_!Lsv?C!Lj#2yrPx&cVc6mP>{Mm)kDlxCdR=3xG#nE>@bc@N6 z$IpjJOeFg_Qg!BRHG=S(n?iqp6J@0k_mQ8 zyTjze^uC@?$LrFu8-nRK9g)VgZ~N@8Kx0iV#`#IrJQ5SrqH{5g{T$iTF(U^XAVnH_KFf@V7WnptTladlZ!F-xUV!^P zty#VMe&uSVycoLAKCX?$(8%9x4kA>duU~LW{u^h!nZB$h_0s&~F$$9^`-_P?BSQ(S z82LF(%lCf9I7v`dv3uhA;*ce=yDHqV{nbBBv@ohGR`_nKOLn$cm+|gmh{r%OFP$sw&Xal*Z~4e9Xl_ECACnwMPQBgPbLr z|90iRB{yEC;%u$_r^n@c<1v(AohEKDmKUe}v`cD$$i%nxbei91!g!_TV0>?n@Guv3 z=-f*NtqB6H(Yep$%3}wacO2)m(Y9al-iMmNQn_{cBnn>YX(u{BdSJL@Ag0|Pxdw_r z!cwtAhPtKhGRVsic#V6araB;se;c%kTn*=wlRgtducr;PPP?*f!J03b$r?a;^l%(s zXgauK!x+YnIBx!7=>z#CdKL1Duv-=iZ>+0t`NDVQ57|M|Ev+Hy#fxWQQ(232b%5DT z0<=k<$SFPXl|7zv62(MV9dXmbd?;d#+6=A6nM2?d2a0dj zxAa;omr~cWFoDkEcJlFE|?pNghrd9tzMu!-a8lNOc8+!Gj zykE!PXCv8jc2=Vv5fLBl2C7$dZd(TgM%c!x2dzyX)az~kW?uGNzcluU1& zi->h_Lb)2+BUD^PQk~at0iJkO|1u+~zfL4emgz$Oi1#ih?%o6R^>w9FLkZQ0TtU)o zEO>)cWKE|RjF*GE%*5GM`=4k09x44|gQ%X~lA= zw9L(H8R-P~4^E%sInU-<@iRU9`c&J6Gh`fC3eLza0B&}f4Lc6Yj3MCo!K0E# z$F*R^T?AGU<1U`d(|pXs0PZg>$sciPr|Gra#ui-xWT8fo^!E)OdEz$5fa#tU>#L{J3WBU>et^1pcSt~3nTeUSk6+5*TchdEUns4C^Pqn9NY>CCfH z;<#JyHZ$pC@<+Y5qKjMC52%Kpff)-V&_=j|s@6o$^~vsI{Z;i|^aBKAqeDtJ z7-{O$N^Q^H>*<>LsmWR|NHPImPKT|8*QG&@UdY#mY>lsuKAATo;k?Gcy2KhgHX9TG ze-x*aE-nkF0y$?XSs+MKk#vFI3#42fZzeq{S>zBsJZ+JvEtRBM=x>UaGHi-ga)Om> zH+6hHgR$GUvm6Uig}zW8%8?Iul&y#X-sSu9mftB-4wZ8qnNurMkKa2}N6GZ4P zAnwC%tA zi=4f(_kuN#nksLgZ-m>VCHTG1Ik99>0&U+??}(T24UD<0B$j#en%=+8zM)?>U$0ZT zY)xr;ADpv0v^YiCmUhX6&`nV3z+-*ItG)A}(gC1Q^KHCgcj?)Rf6`|J#sT{KeJY|D z%gh24)KL%@%}$GkY}b6h;IwCbilH!Q-V7jt($rvXORk^AzEtncTMaYC%e+}u=sF_O zy;)X;We`q_s10;k>ft-Anj}M0Y{>JVN#iGVCpNh;`qALKZ(pKch^}+a7>kbT0lDN=0_)1| z@Gm@T@5$A};V`d|R7Rr2%ykKAL@-XK8y&GGR9I^&&ftxhiSligyg&3-vX>M=EiX@H7Rx}qsbvBZyn?E$ll6-}$= zPYiUsgEWvE|0e>lDCBq27sPFP8lM2OB8Vq&o&FWJ=$P)QpBDZQY4f~$H89{UPdqDD z&E(YHR5jJ$I7)C3)Rj-1tQXa?9D7!vQuUPThi(qIT)%lC4C=B`h5oo(t{(<6pey#> znVB+e{Q4Yt@n~%95{geBaPnz`6tyds>N{b~^kQ8H!W*oDq(YE3Xy79?Gd?OlwIu%$ z;IXsG5l>;4-S&zsrFOd9Tgw{Y;8Q8%2&_b0BqA0pZXs7C6?a$#>IXTdwnuzXQt$NfY|Q_Ie$`oGzlq$x!n&A(#GS06K9*>|+71u8% zIk}??L8zmjTQ@5^s?h%;7{)*qy)j@0s?)=V{QYdgm*D?9usNs;hh@2VbXpAuHK)0( zz!OzWc7C_LQ6%=A^Y1{0a*nL9Sn9vJ>^yvcXduXo9%{00L6E+inwg@Oj_twkuq~qP zo1q^2uaVZ!6((51^io;U2`AXMlqkC`;h@^1!kHF)dp8=tN<*L*bT(@X>i9t`2u1^} z0={ysCt|a;Q{AW#QGrF&Y@EZgiBw>@6)sjdhrVxtnmHZ>{btds<9d73x$M{X!#Kos z1=aUKvO+R_)Ypn<*&dD*L3ZltU}ht!zk-$mR@hAzC%@CCsB$Vik8*YZqd{|)Hb5Sc zJ_3>-m3DOx7Stn{F*C_%OF2F@>VW_?s%Xi6)OLL)b>wMX_|6qQcyBc}6%TiD#&phx zzrqR;(%H*tVH(1cH(2yF>m(e^iS73!j|%H099VdU5+KimV!ri9fi%n3nBA>F!&j;M ze=^WG%O6AfY(*Wr>Nv~TfU85#zUO+2#<|N-#EGKf4=Oj!e6qPpzZW77(zzoG^%zKn zdhCb6T09M>35AoCOhbSggsc{cnaPT18|FUsc&weD$O{=Eeg_&LVNN^3#}u7>ohwO+ zo4-=K>DD(c%uFU|4b0L}Rz!_9k?Yu$t3SbXK};nuz7Wxl75mmJ$Nce{-DG#|GXr*8 zItEhn>N6xQ2pCtnWr?{0HvbB+`TEVVmwc{WJa!-*Hc8YTOVn)jMKHd~=4tb-_FfGa z!JBhczbZty2>1SdTe0-rC5st4;1j1UM1P1ZPu$~g`Q)R$-d{~>KHbj$TM1f5@WNcqYZPx&vSf(S>-!Ccz8Jd>H`T0(TKv{U=Oye$S%i&;Y% zlp~9YG(+K&3g_8{3M+lZ!_*zymO$^`fao6s0ny0v9(70Dt;=(Np)E2*r~428-<=26 z-RPTSc%pVUm~;TsJnQ?32`L^TOT*!)CV5@ZiJC~C-4k9#BZKzq{@pTFgm9tlttRAy zx9(HfHFv`w-)59yUpX~q+tzArSmd>t;eI+B$d@u9I%9wM?LciOe*kv;x}ptyAyw@0 zQT-@B*G?WgNhl*i#Z3n@!Z7Dn9M#Bp2fQg;{1U zs+HP)+WHW)$e8`;$s2xv32xroe7pec$`1O+7>TP6mVJXp&@sjG7sOZ)zM(nA;g0GX zEOQUZ&+)g4v)^S;PZFNWCZ)Oy@s)DWovq!EF-@|Cm-uu8$_dAJ z8IQOUONScMAPZz~=mkPOBED-{Dt>W1+3+lnWpkvKlw{r1DsMBs9VaVfZFGJx_glEg zZkqY++V~#@F^IkeM7^1B_8|4Uu3q<6$%>o%cR%P`bw1Y%dZ%Mfaa?rFPNU2AQnyPh zoSa)I7r*O!FIw>(#oEZwnpPn-W+RRZ*C8CB>bp`&#VCc9EYP{xy+tF|%&9h4_P}39 z%2sU7ay=BC0k^egWbm%Ca?y`*z&hypJxxeXr|H0Gb9czjVV6uq&@V!nM+GN zVGBY{`;xY88~xzX{SgX9eU;rn7D|fV>Zp8#-f6{e1FTh7-`8p@#Y3t1ioU5SpU#EC z*X|>VP*9R!a!ZS~S%+n~P|5eA@H1^*Ec(eNRdU@HR7|9=`loN-W6_hEt9;)8&*k{3 zc^TOL;s}#TV#RBNq@dql*>kJ&g2b#8ix8|nf+Q-=-g{~p$Ws%ScA857{#fr*Og!=g zhLKR*SRNt{&YQ23t_o1E&v|_oFuOP(jv}9?WEvDdf`A!>s8gKi`G}|GAd;b5dwbYNT4CL<`euw{)mXnDx6yr5 z7sV>Zs5^N3mNNz;;v00xu>$9a*xOikvMe3hA!)fZ$TnqZljI4>qW@u&2Ovo~7Wky40w32@f{5Ffcg53gZM(iG|h zz?T;QUy@U9?)*aHgm2&MCTVL^+ zfSjRdOzJe@mi>c^daM61IzOC)85KWTUr?fmp()?>zlAoC-9*xY>?Zq}1C2&qMDIO1 zI!t|ZbafDIDl%;N7Ec6N$crDk$`7%AnhtBAnK>$M>s-%AUj&q82_r=Qa{`H-3wDalNnb_GLB*&_uut>ass(Gy#GsDMc=2ew! ztCxH4u&f6DW;l4~!QCrR>@(6)DLV60O}!{Uz+#Y>U24rCrMDVYT#bO1a!1zLgb9^@ zt8ogK`lDfgg}%kRt8_Urrc;9cn*7d{UeU7D@34{>YB6lH-*4|vA!G(E_E)SK9)?kO z8E#H(n<#=VpeVzB=&CWx&NNYRP0<4;l`Ejq1(Mg-s=rS?diexHj7~s?vb3!*kyo05UR?1}izq&EAS5oLd&i3UQfb=H{-o2#cE}(}_8jbJ zDYy4IMwo#@D%^T~MYZkS9-;WRX9UorE?&BwMU?4gmUn>m4LY8xTLc7jyb%B${T^-% zIR7^%c4+lO)c&Ol7XsJyLz23zsf!R9ErKMBXK&0mGqM4IOTnreoAhhhC%}5y9b|z& ztjy!hdc4z26j@u^|3Apc-#l_>R6RY&G$|S?KUVUC-HcIrYZQDU8>soNc#IDsGp*(8 zX|J=K@E*U11N;MDy>)W4*J42SD!24gk7Q};H&}_EXhbZSyX8l){Y!g*HUZY7+L-EY z2U#PK5#C6E%o*ywV!AMp*5PnAx5fBgphGiqBLTUOcfJf>bK4~W9??~$R?Tw|346c3 z{vD!-g!qhHI-zG$pa?aroGc_7j5SUth#fm*`t;ujn{=A_KY6`)Imi|i`ENb|fu=DX z_pL+WBtg=$es$0>t(F4X|GkqcVDb6mV$K$HtKvPPuasFyf|9fCqQ&~YYS>jE=Yz$l z>_Ilm7i6<=xPN7{j(qfCf*k)7&62QFuV*La=e*$#RE2ylxOpsaE*FTlc%W%~t z(fqm(C_$nvK}SFP=vliA(vB9+WO|ODV4!YyC6Ql08`Ua>F#~G_7P4Q!NjV@UJD~sB zj6qtg%rjXB5vPx3_gJ$Ay!xKs=kN%$OxdLd7NG1i3-p>d%yLjM{)Miq&0f5Ehj&rB z<9OxjA&4-#UJ@^BT48~~9;Z?(1vM=3kZ}c5C7sb**JtXuEv<#PrHnkDRFNIZm-Zb`n$9%18;Pm|y zJ+DbeUk#fH&K6iJuoZ~Y`T*e)l($=%kuU;w=*u|I3ICj~l~OvS=mpU)eTf1O83ucH zLfX~2<4{;dFn`C!g#Ttgfd%g&^SnE###wVQ+ko#MhhjOyOrd3PcIz`}(G$y;Opp4M zSTjL7zcEMI7q~6a4rTHDg5OXUM*mXvW<7VKd`s+LOxS3avhxsKR}1y=n+5J3um_!7 zMiIZ~KYnRB+RS-B{wo6pOw`?tNEZB|y}3fE+gi<`H%_EVF?4g;+*P<`RbczYO8C2W z!a^}rQ3?W!skX8Q&&yQ(v8mE){?gSR(r4*QMMx@t7iHU|_s5ODgHr@Zg~YpBEJk{EZ;QPYcAew5Z3vTA*?vycJnJ9 zJkX^meI0_huVizZI4s~`JaY3M3qP?~7fr;#Oho|Y25RC-Jfd&!PrT)6>OOnjFRQnl z@jdOXT3bv?S+j|n zmzUKxO0wmj*uM7iWeQ{zyCC6rgTKr`w?G#z0_4jG@FS%^I^m+Yu!j#jbjZ#uia8Hv zrBD1sl08A53P3OpW{N-U0vhJ`9&aMe;QX{-iVMgQXR%=F9W^aqSc3Tkd75D9?UR$~CEp=>RAz}qG^GSiyaOH9+ zYLRA6<(YiLvHVwEQwAp%t5v`Ev8K>aM-qeN_m^)!(7!Y)k*Kw`2ag}Q0s3dT@}B+u zkxK8Qv&&k=n_!tqpp5iC1Z?aI)LHZm9H}jk*3OljDlqqX?K5DnH<(`5L?Knsbv!j3 z`P}_ZXT?@&VA|9CAb@*X*uu3u`qK^55q{%|ZYfS`55143%XQP|G@a9ovCtL`@T3kb z2#o6n|Iu`&R%qWBx?$p|*@0V-5vO;(@c~#4GFLC7z-*R^G#dM}sGfUX9?kSEx=584 z5tK1*@|_^b?JKrL@i)SX*$k`}VcnvUD*1(=!t*jE_5*gPHQh}9HvlW6C%SV4#GKYq zb~UuYmInx0c`H&3rVAh<{x%sI&J|B71*;X)Q3ZkndGJ`zGmwS6H1_BD=7I=m%J(&o z_vGcC&3R5Re1Cg*Ym45ke}b-XDrVNEOZQc3@K>8_OVPG8p}1c}j_-y!NSVU2ozNNE zmJ`{LYU{yP$V$NO+ccm%wn!d|>Tj3FZU=HBvjk*2&n)b!zDT6WHjZ<|jwdXR#`WpW zx!`u~4rKW#a;6=(pen2j>;aT=4s{|QPx7krMQZ+9=GZ$&>J^w)yb0{RfowEb0V5~d zwzYmk-TW={djs8>DZc{R6i-6bQ4UHKV2Jz&o9dk1=Qr8*dlB)$7+gmaxYgcfN1yTu zDERIT@tI331K`2y-1ABDD;P?kVF8p9CK?gEHX-&Ge%I=&$|{%LwWab&GdcOBej=Tx zdyyU9{`2gN!tOKR(FW~H2utA&4o-<3eX(~rUkwgBAgtG4~=B5$~x8Rl+63jie38c|#IYHMS*UnjVe3vJJz8F%Pq z5KU{8yHJ@H-YBzE zbbLTl|LLmdoY>ewpe9sdEa!ukwwZ%TJgkC92Q@$qRFj~-K~%H%UA>0Mf$j`~9w^TE z5E&lgmBixVMmW+0U@)%Yh~RnA#J69#jG%^*#?Y?M7KQw!c&xTkIkb3~Ce6%*zx0Fm z&s6OWGF_7%7s}#7gign+=t}kp!sOM|lN2tPEns#`tFcV@@_jeO^D<*8$bSDv4wula z{X~%~FqsyY+IaOvfH6**1!HMyj4LK2Xy4CBa_re|;Y&9InO^>sxN1IW4C>@}_QMq* z-8%}NMWPKxzG5)(!9+6i=6Q2d5a|W(Ft~Jz6E*+VlFh?=9~rQ!CgkU`cIK#QYZCB2 z0`P(RUE0`-_)FouCew4gCQFezLZzu62zop$jo4xWy^f}3YMdJ>>;Fg*JBoAv$C3X# zK@84Q#>sgqonOoR*TI7~ox8m;X1{l1%=TF&N^p)=Gt@3Shd%Z(av5&AWrbXY|tvTxYj*L@<;}! zyyDl#g)*Woau3AAqElJqiFr7oA0_IGV85<}{_`^ctDu(#MNjbiDb|;A%2C^4zIjFm zvv(8+Ku2Okoov*mlQatHl{j)MJ;4_QXc?+nbA@p}uy^;7csY7J-A$JHEoMj@OKtj% z$)f1O2KxfgmXNgZLo{U4N<_tQ{cYlr#+|_whQmtX*~oQeu00Z21M8u)U~qI8g00FJO0FM{Y}lM}Qx*wP_y=F$X}j|4I>w6Eag7u8k5i1Mt0ab=6g&i6pRYJVAOp(AZp9$pTpa z3{`f{h6p&@p2{!Bvez20R=b;dw%}QM&@vSu1fz)h2H|9?xz~7o4x%sNLPalHB&eeaF2u}_bx_i zTcoU=$nm~G)G5`ZD7KsqfejUfeNKY>SpNrTpD`^G@eHg__+)7Rh~EJ5tHm94p_&1_0C74%Ip)W1^2Lk-aX zh5R=g0@RWjrWfoA#YRJTfpVeg62*^4@}{r}X})q#{}%%sv~dly|J(BS%ua3Z5^?3J z;~WbS00}vby6Gd?lBp;MW>p6p3ov&|_4z(}#`c5EsPwETMSrDHND?%|hTA=cPa)>{ zqY@u3&cB87kAyU>KENJ7Aug85IZy8U^sqC16dFCBJ(NyI2|`eR2032lsI)4fHZLiO z`0g_R=#R zOtOp~K(5n)`Q@Sl@Cvd_s=r|?J#}-+5e~9bBrvqj2So2H*PSj_Y&?=X1(Gf43mTDM z(DBNRTD#dLdg3sMOpz#742Q1R%b1iEwP@j)+++@#u!^bg*O$OWA|KWhK}@}$9D zuk3RDvNf{cUB^3KK59j1UEr6viJeuV>T?#4K+_?Em_D{W58MXyUNa_#Kt_6bvAjB+ zI<7p)kp5-wKJspXe9>MEwDfgIe{NV@jzX`Fv7wcw3@iGh2Ey0j8TDgwZl76t)L(x* zTw^uFL1Soa(UD=bs(7Q(({(FGKVoRs^t#^oO&YQ;m7NI(w?fR528#$bUGh*OV(PU9 z3!V10R0rFVbUS_17$Ns&t8UG?8V42+^iOlOe4&GyR@RP;3Xfb;Nct$>roO4b2!*<9 z7NV(B<*H$HJ$nrtNaoi$2VIIBk;z1_4%HI0 zIdJWz<=qX;^Wh<-FW^ZJi%4a6BTk%vaW1Q;O;pr}Ot`-)L@S``Rm1S#mKiBE=<(YQ z4^!NQX3R1j3RzgM@|16gOVRfFkD)&iw*-#8D8g)2c~&RgJf4yIO-NNi+%js=l6n1S zGIzJ{3Rv10up^vK%eXnPBf3<_u6+f8Uu7tf{yDmKn$0FrTjYfdhe^OBnh@FHJivk8 z;V%wO|Ki}0_rfKQhi%QZFx_tqKWQ8?;{PM5zWcbLz;=-h>>g9+mNyJ+h#v9TP2!?P*Tr8ZIX^GF{jEoy3eD0x9?wNM!ZBD21Ihu6#T`IB$T~DPy zC+|BJ7TCq>S_vcd>8k3Gx1ZN_uokFM>s4w_N6BIKknO!j71r31a<~iZNu}rYXNPyY z1$S2)a-d>_PLP|IMI_%Gd8%<*45Ih3W^|_0@c^6Si%H7t%%fvq-;eR_HvlJ)G=!bn z(DJJCOO@)#ZA#a8O7CfMfvF0h>1a9gmbfIfa=b9r9rzvE@t7km@9YUt-L-b0 zoAhwpRZ&{qwMO7~7B-2kxaEm&A8t2qOfF8l*C|DB_}eqz5sz0ov@hF#RAT2#&dEFb zu43fj>VET;<9Ms_CO3AF^23GSrTJ0bY|_!jV0j6mp;GArbhyLn%dgny3O1)u@{2q^IC)heF_; zefBYVFBVI5GZO8TUtyJ?VH`S5oTebkah6?-`Z?X07Osm`ZE->sF(y%Sda_E9V1bzgwdm%|JeoRmVF1yl|&Uwbx`P6mE|dMRv`x~oxU;-d5IGM z9?Zt!V#UOh^px>5bOYO`eZ_^ZCCZ6-ubQ6`yI}&$P18Rz8NQIlf@i7zC_>x95^)wC z2Dzf(P8I7IIN?P&jtpj!NaGaD??5`r^b)7KRD*qELC(RI!y>ep@5;=#6pm;c;F3qM zvRXU->VVOZ9J0u0CmAck<1^TmGwqqFJisGI2;V!EMyRJlKr&`Go@Rz`*2SHbFDQdg zHJe6gSN8)X3693Hb3BZC5QjpZnN4DLiKO^D}?7PaIBp2i_WnvDD4>sa@f6Qqj@)j8Mqe*$LSjy&5rur2%3WjI^J6&XfnF=Kd!y+@3JuhD zOf`6!FH|+(b5w|VqII2&U{jNAvpt~C=_uos$#aDg^?lXe<@%L~|IkbPmO{C0`Lo}H{)v5+(K*+UNRL<8@pwXDA=if&9G%q6ki zDG!mCXmLA2fsnyjRhQvl3{!8#&tOGxN#)k-+_~`Cc;E`+Jgw7yYSAsNGI1I$z0b+D z;?jz4sMG}^r}CUE;YnmYv{x*b^r4$CWRuSoesvH&W?^$4lmSkKpKj@$dqhYw-KuZq zTT51<)RR3APKCf)5`>Pg3w>nZYzMf4NcOwh*PJS_J}pEfyto}7X?2}s-8_pZ5!jt? zoZD)SeeM&x=X)Qqa$evjFh#8w4VA#$oNfuCeacv>GBzMrk)q>Oc1hnd#rXZ9JU8$T z&&NHi5#N4p-_8o_likOHyBNrgN*qq*;v{nv1D{l@MGjc4crk{VpJ+myfFPgxh2Pm^PA-ZJs&A8qNu-5fu;x*4q4lJE@Ujo% z0ungLV5-zPt57lco#gXXIUOB{9Rt!O5xE`P+1KlFr-&Axy~jfnR5*#>$_XGG&)}tN z{bXKF4JQ60Tz zo|}ZejL{6P33-B3B@GWu-y~O*Ycj{A;D2C!M649NyZ1Rh;@%10ppmQGyWtfsbv2GK z1v0Cto_XWy$+FitF4Q6l`BEb~aEC9>Rr5qu;7qiby(Uqa*YV2j4D>rmd+AG3ciHxK zGL&no=CETlF_dwzEEL|f=^PK8Mz_Op)<^DmwB|Ut%Z`Jb9)y%=rgD$!IJPb1PO%2> z&dMz0$hE1t9?@OeqHzl>?s>Fn5170liNDd&eP8vAq`pD}+3^vfq2c>kASfzzcap~U z3J@N)v5d-&_L2fGrt<4NvVDH)gLp!0HZUJc)Q)+0A_tdO^pM12vRE74yCk(&C+fg; z5+MkQyWA=QM#~^YCW-IqB#e!3i{ur$GDX^oKNLezP=3yWh9Pw+tNZm*3A0D129R;jxAe#T8#jHxA$|Y%pzFpnsK9Li*w-a*0wh zhM5hr$J(ltBU`CJu@KkGYU*oc$@e-Bn(aR$*T%x-(u8$i$R-NE*B{f)LaTyvuri=S zJL}ipwPf!zUlQ+oF1}2jEmXM|>spriA>YmJ)6+hMAu%F(R!V{VtMIAIhe*Qya-`Jp+i)%%k8MPikHhx+>#kpV~&D|XP<1GO(rSKlR_ zp7+Iqk3o};eT8KUN;itur!6KUjF7LMyX|US}2WkF~gk4FA=Y9RPbrRD1VZ% z{y2IA1-RV9+bnH%=v^9GN#y>yKVi2UgKYfZkw47H6iesyBM zrZN)Nv2qms`ytDSXql!8Q4n$y@nW1`c%g^*aw$gUc2`s7M;bGmor262Is@#PZj&r>9 z6RY=qCX}I9?qwn3I4Mpd#>^oBL2r2N?T?4p#$={aII7XdDos-vniv~}WlA&bKGVlp za;Gz}RN!E_9S=Dqi*zv4uW^7=TprwwLnb#+^oYu377(bGR#Btf+8f~6AzcNiYbY4L z@a)YBdqGygJ6c{Cj}gxX*E=3Qzw8(llJ9=Sr{Hgs*9TS+jDt|$pAW4yD?GqAnf|g{ zORN2OKpOH?FGIHmc{RSey)qcXzX6#sSVds^6(a(YC-?2r26ZS>jf2ZAG$&4R{kQ?S zMWmW7^AvO3`KT)w!bpi9PngO{>7{B*<{E~ey zJ;u|zokjus#)X0v(|w_{exbCUm!Yg79Gv_d8X=pJ`5=4_nMx0o1+!tNSoocE+;T<& z+A^e(+*RM zb4T-bW|6E%Kc_Nd#)-hJ_;qOG=;f(cUxy;a(&^^rL-;(_ZJ+GqjlmnK2I;Nz32(3k zRfdHdq5g-+_*wey`GBN4(gD-NyU66QWV7$|)D*(R*vdh5qyD!dQ;vW#{dX@Hl?Oc$ z3@BnZ&`zQTegX7y&Ar^rT^x+<6s#R=U4QxXv&j^#@zG&{^Fdo)ooBTlA`w-XyZ+-a z=6KDho|&lkb-D~(FP;ZdFK(^uZO|gd7|XZ4j8l&7IKG(6m7pL-bkt&|_PQTB`2c>z zRTbMH!l}{!l;WWZ$q~G2r}KSftY$h2vw4(}#wX(fUQ)baZ%~hLL+cqIcU_SO-LlD7 zM?2{S9e{}w!*S&6X7{%q^WIa{msgdaO0iCO!1~;Xw0QXqzG`J#+U{8tGpXbB(I(&I z%iD+AbDK5VEjb9X_4e^G$8PI(PD(Z=EjFfNSi#6s^uUn_a_drP_DLlMxcx>SRPgo} zEeu9OgY&~R5Ywcs=lNJ{+~{6CEtc6U_Q<%bUn=jEn~2aB<0uw1+A6QT5?L;3Xd~Ps zMv%ay~4RrHp*fUAGp|tE4m+C zguipf+DJ3ujO)tYM>KCM>a>D~eA^rRN$9W&^LjZr&~*h1({KCR8t<+PSraQV`}n$p zsO3#ZNsAsb`(p60=L$DPQ?|13hQBcTi^K6aMc*U#^)%zFQ`Xx*I>k3EOz!)D{y6|% zkL^!-Xy{^Y_Xllc|B)RQuPBeeg&lDaaz{1NCbCE&J!J8+2^EbUY(I{hIRA0NFQI4Q z))zsi`k@!`g9z{3w8r`Da^ojaPksB1wWr??JgutB65RD1f9Y$)gjO1M6f2m`)LGKj zX5C;W&8}=cLscW`{GvPCRf~pRD2V|2Em*HPT3jb#n1lgclA=et>Gh6CbO-T-0he}_ zV2=S?@ZMNK`km#KGb~Y96&MjuzLOpwwq~%egc3F)a+h48duT2u;T7~87 zmc48-uOG!A`^?B$A1mVNfuQYqpdwn8uDFju-p>8<)zX#*S%iM}>8{O{!8BIF{1{W^ zlyjFUD|6x0!GUH;|N9Z+%t>44jL%3|#d}yFD5)wXRVWvc6S0mc^zn%p<0P=%(1*t? zn1}p0qz~;nzr6}JdhdMC&itm}rM)5-g)BIDHqbvq zv*UajD?xzHw*f)xN5}rxOIOO#!Oh&k%@E@4WbUf}!w6L5e+8)5;H3EuP$>W=h4u?| zGhjx4xtgDz<#$5cr=>xZ0OzZFAP~tfgv&r#fZ+KTt*4`lt%;+f?GN66#7W;Dh^YWr zsiA;CSij(i0XYE9zY%^fL}K2j06w4qFM(m{2fq*#0XcsYyIPr>ySZ}wOnCn!gEycl zkvG8eR{#b76(t1ck0{)KLYWEP#a9DR;DM3yUr|i)|A@l#CzNL(lMSl`5XdJF9;oJj zUps_9QCzKzUChnY-CV34EPoEb|1J{=>W}8LjX)X1kbWyic-bGz@v|-SJIe24{{NID z(FpE`WdE4~_?`6k!KQymAr?PLe;<1K9pKk)!9V06(2gA(=)XD%zmxyokoyO~%kd}w z|3kCxcf8*N@;`VWmp|bB@51x%(7%SPe*uGpT>tf?e+MuXc|<@%f0zjt=mszu77xJw GK>rIJ$eelr From 07eb063e05e6f36cfc9bd8eff070f7d0bbbe0bbd Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Thu, 20 Dec 2018 15:34:48 -0500 Subject: [PATCH 18/52] minor tweaks --- .../awards/tests/test_awards_idv_v2.py | 19 ++++--------------- usaspending_api/awards/v2/data_layer/orm.py | 5 +---- .../awards/v2/data_layer/orm_utils.py | 12 +++++++++++- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index f9a0b64ba0..2bf3ecd86c 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -199,23 +199,12 @@ def test_null_awards(): @pytest.mark.django_db -def test_award_last_updated_endpoint(client): - """Test the awards endpoint.""" - - test_date = datetime.datetime.now() - test_date_reformatted = test_date.strftime("%m/%d/%Y") - - mommy.make("awards.Award", update_date=test_date) - mommy.make("awards.Award", update_date="") - - resp = client.get("/api/v2/awards/last_updated/") +def test_award_endpoint_different_ids(client, awards_and_transactions): + resp = client.get("/api/v2/awards/CONT_AW_9700_9700_03VD_SPM30012D3486/", content_type="application/json") assert resp.status_code == status.HTTP_200_OK - assert resp.data["last_updated"] == test_date_reformatted - + assert json.loads(resp.content.decode("utf-8")) == expected_response_idv -@pytest.mark.django_db -def test_award_endpoint_generated_id(client, awards_and_transactions): - resp = client.get("/api/v2/awards/CONT_AW_9700_9700_03VD_SPM30012D3486/", content_type="application/json") + resp = client.get("/api/v2/awards/2/", content_type="application/json") assert resp.status_code == status.HTTP_200_OK assert json.loads(resp.content.decode("utf-8")) == expected_response_idv diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 6f18f81f7f..eee1d50558 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -23,7 +23,6 @@ def construct_assistance_response(requested_award_dict): """ response = OrderedDict() - award = fetch_award_details(requested_award_dict, FABS_AWARD_FIELDS) if not award: return None @@ -60,7 +59,6 @@ def construct_contract_response(requested_award_dict): """ response = OrderedDict() - award = fetch_award_details(requested_award_dict, FPDS_AWARD_FIELDS) if not award: return None @@ -103,7 +101,6 @@ def construct_idv_response(requested_award_dict): mapper.update(idv_specific_award_fields) response = OrderedDict() - award = fetch_award_details(requested_award_dict, FPDS_AWARD_FIELDS) if not award: return None @@ -279,7 +276,7 @@ def fetch_business_categories_by_legal_entity_id(legal_entity_id): if le: return le["business_categories"] - return None + return [] def fetch_officers_by_legal_entity_id(legal_entity_id): diff --git a/usaspending_api/awards/v2/data_layer/orm_utils.py b/usaspending_api/awards/v2/data_layer/orm_utils.py index d0bfb44cf0..04c2aeb401 100644 --- a/usaspending_api/awards/v2/data_layer/orm_utils.py +++ b/usaspending_api/awards/v2/data_layer/orm_utils.py @@ -4,6 +4,16 @@ def delete_keys_from_dict(dictionary): + """ + Recursive function to remove all keys from a dictionary/OrderedDict which + start with an underscore: "_" + + parameters: + - dictionary: dictionary/OrderedDict + + return: + OrderedDict + """ modified_dict = OrderedDict() for key, value in dictionary.items(): if not key.startswith("_"): @@ -30,7 +40,7 @@ def split_mapper_into_qs(mapper): return: - values_list: list - -annotate_dict: dictionary/OrderedDict + -annotate_dict: OrderedDict """ values_list = [k for k, v in mapper.items() if k == v] annotate_dict = OrderedDict([(v, F(k)) for k, v in mapper.items() if k != v]) From 93a10f161c7b62e3da01909b053a2ab31f7d1588 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Fri, 21 Dec 2018 13:27:33 -0500 Subject: [PATCH 19/52] fix flake8 --- usaspending_api/awards/tests/test_awards_idv_v2.py | 1 - usaspending_api/awards/v2/data_layer/orm_mappers.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index 2bf3ecd86c..ca17242580 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -1,4 +1,3 @@ -import datetime import json import pytest diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index 0ef31290c3..8aaada721b 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -164,7 +164,7 @@ ("place_of_performance_state", "_pop_state_code"), ("place_of_perform_city_name", "_pop_city_name"), ("place_of_perform_county_na", "_pop_county_name"), - ("place_of_performance_zip4a", "_pop_zip4"), + ("place_of_perform_zip_last4", "_pop_zip4"), ("place_of_performance_congr", "_pop_congressional_code"), ("place_of_performance_zip5", "_pop_zip5"), ] From aa5b882f8115b927cfddb8ddb29563efdf12e386 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Fri, 21 Dec 2018 13:59:26 -0500 Subject: [PATCH 20/52] Fixed tests to use the different zip4 columns --- usaspending_api/awards/tests/test_awards_idv_v2.py | 2 +- usaspending_api/awards/tests/test_awards_v2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index ca17242580..25cceb9430 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -118,7 +118,7 @@ def awards_and_transactions(db): "place_of_perform_county_na": "Tripoli", "place_of_performance_congr": "-0-", "place_of_performance_state": "TX", - "place_of_performance_zip4a": "2135", + "place_of_perform_zip_last4": "2135", "place_of_performance_zip5": "40221", "price_evaluation_adjustmen": None, "product_or_service_code": "4730", diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 501f77788e..23dbe2b843 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -132,7 +132,7 @@ def awards_and_transactions(db): "place_of_perform_county_na": "BUNCOMBE", "place_of_performance_congr": "90", "place_of_performance_state": "NC", - "place_of_performance_zip4a": "5312", + "place_of_perform_zip_last4": "5312", "place_of_performance_zip5": "12204", "price_evaluation_adjustmen": None, "product_or_service_co_desc": None, From eedac0c1565fe6f331613ea49356f9aacbb10a89 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Fri, 21 Dec 2018 15:40:00 -0500 Subject: [PATCH 21/52] migration should not have been created --- .../migrations/0026_auto_20181217_1702.py | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 usaspending_api/references/migrations/0026_auto_20181217_1702.py diff --git a/usaspending_api/references/migrations/0026_auto_20181217_1702.py b/usaspending_api/references/migrations/0026_auto_20181217_1702.py deleted file mode 100644 index 701deca9e0..0000000000 --- a/usaspending_api/references/migrations/0026_auto_20181217_1702.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.15 on 2018-12-17 17:02 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('references', '0025_auto_20181126_1528'), - ] - - operations = [ - migrations.AlterField( - model_name='legalentityofficers', - name='officer_1_amount', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), - ), - migrations.AlterField( - model_name='legalentityofficers', - name='officer_2_amount', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), - ), - migrations.AlterField( - model_name='legalentityofficers', - name='officer_3_amount', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), - ), - migrations.AlterField( - model_name='legalentityofficers', - name='officer_4_amount', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), - ), - migrations.AlterField( - model_name='legalentityofficers', - name='officer_5_amount', - field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True), - ), - ] From bdd105e5be50a3765d2b875e1164ecc4823a7422 Mon Sep 17 00:00:00 2001 From: Davis Ford Date: Thu, 27 Dec 2018 10:31:09 -0500 Subject: [PATCH 22/52] Fix typos and grammatical errors --- README.md | 5 +++-- data_reformatting.md | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 36bc712c45..1759fe0314 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,13 @@ Your prompt should then look as below to show you are _in_ the virtual environme [`pip`](https://pip.pypa.io/en/stable/installing/) `install` application dependencies -:bulb: _(try a different WiFi if you're current one blocks dependency downloads)_ +:bulb: _(try a different WiFi if your current one blocks dependency downloads)_ (usaspending-api) $ pip install -r requirements/requirements.txt Set environment variables (fill in the connection string placeholders, e.g. `USER`, `PASSWORD`, `HOST`, `PORT`) -*note: default port for PostgreSQL is `5432` + +_Note: the default port for PostgreSQL is `5432`_ ```shell diff --git a/data_reformatting.md b/data_reformatting.md index da45d81225..ee7f9585e9 100644 --- a/data_reformatting.md +++ b/data_reformatting.md @@ -18,7 +18,7 @@ Making sure our location-related fields are as robust and accurate as possible p We attempt to match incoming records that have partial location data to already-stored unique combinations of _state code_, _state name_, _city code_, _county code_, and _county name_. -For example, if a record has a state code but no state name, we'll pull in state name from our master list of geographical data and add it to the record during the load. +For example, if a record has a state code but no state name, we'll pull in the state name from our master list of geographical data and add it to the record during the load. ### Country Codes and Names @@ -50,7 +50,7 @@ The following fields are canonicalized this way: **Data Source:** USAspending history **Code:** `etl/helpers.py` -Codes and descriptions in legacy Usaspending data are often stored in the same field. For example, a funding agency column looks like `7300: SMALL BUSINESS ADMINISTRATION`. +Codes and descriptions in legacy USAspending data are often stored in the same field. For example, a funding agency column looks like `7300: SMALL BUSINESS ADMINISTRATION`. In these cases, we extract the code to use in our data load. We then use that code to look up the description against a canonical data source. Using a single, central source for code descriptions ensures data consistency. From 67c45c8d7616ec14acbb4c75ae3ccde3131f1338 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Thu, 31 Jan 2019 16:13:56 -0500 Subject: [PATCH 23/52] minor django version update --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 771ab2c57f..261bf15a04 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -12,7 +12,7 @@ git+https://github.com/fedspendingtransparency/django-mock-queries#egg=django-mo django-queryset-csv==1.0.1 django-simple-history==1.8.2 django-spaghetti-and-meatballs==0.2.2 -Django==1.11.15 +Django==1.11.18 djangorestframework==3.4.6 drf-extensions==0.3.1 drf-tracking==1.4.0 From 359fda4840887427c9f038588565605691c1b0aa Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Fri, 1 Feb 2019 14:20:38 -0500 Subject: [PATCH 24/52] uppercased a couple of additional columns --- usaspending_api/broker/management/sql/restock_subawards.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usaspending_api/broker/management/sql/restock_subawards.sql b/usaspending_api/broker/management/sql/restock_subawards.sql index ebdada55a2..882ffc6449 100644 --- a/usaspending_api/broker/management/sql/restock_subawards.sql +++ b/usaspending_api/broker/management/sql/restock_subawards.sql @@ -167,8 +167,8 @@ CREATE TABLE public.temporary_restock_subaward AS ( ( SELECT fp.id AS broker_award_id, - fp.internal_id, - fp.contract_number AS piid, + UPPER(fp.internal_id) internal_id, + UPPER(fp.contract_number) AS piid, UPPER(''CONT_AW_'' || COALESCE(fp.contract_agency_code,''-NONE-'') || ''_'' || COALESCE(fp.contract_idv_agency_code,''-NONE-'') || ''_'' || @@ -233,7 +233,7 @@ CREATE TABLE public.temporary_restock_subaward AS ( ( SELECT fg.id AS broker_award_id, - fg.internal_id, + UPPER(fg.internal_id) internal_id, NULL AS piid, NULL AS expected_generated_unique_award_id, UPPER(fg.fain) AS fain, From 5568ba66dd71ef6c9899930161c5b191180c2cb0 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Fri, 1 Feb 2019 15:02:38 -0500 Subject: [PATCH 25/52] add new idv endpoint --- .../api_documentation/awards/idvs/awards.md | 68 ++++ usaspending_api/awards/award_id_helper.py | 81 +++++ .../awards/tests/test_award_id_helper.py | 44 +++ .../tests/test_awards_idvs_amounts_v2.py | 55 +++ .../tests/test_awards_idvs_awards_v2.py | 318 ++++++++++++++++++ .../awards/tests/test_awards_v2.py | 50 --- .../awards/v2/filters/view_selector.py | 8 +- usaspending_api/awards/v2/urls_awards.py | 5 +- usaspending_api/awards/v2/urls_idv_awards.py | 7 + usaspending_api/awards/v2/views/awards.py | 61 +--- .../awards/v2/views/idvs/__init__.py | 2 + .../awards/v2/views/idvs/amounts.py | 66 ++++ .../awards/v2/views/idvs/awards.py | 151 +++++++++ .../awards/v2/views/transactions.py | 8 +- .../validator/tests/unit/test_tinyshield.py | 65 +++- usaspending_api/core/validator/tinyshield.py | 106 ++++-- usaspending_api/urls.py | 1 + 17 files changed, 952 insertions(+), 144 deletions(-) create mode 100644 usaspending_api/api_docs/api_documentation/awards/idvs/awards.md create mode 100644 usaspending_api/awards/award_id_helper.py create mode 100644 usaspending_api/awards/tests/test_award_id_helper.py create mode 100644 usaspending_api/awards/tests/test_awards_idvs_amounts_v2.py create mode 100644 usaspending_api/awards/tests/test_awards_idvs_awards_v2.py create mode 100644 usaspending_api/awards/v2/urls_idv_awards.py create mode 100644 usaspending_api/awards/v2/views/idvs/__init__.py create mode 100644 usaspending_api/awards/v2/views/idvs/amounts.py create mode 100644 usaspending_api/awards/v2/views/idvs/awards.py diff --git a/usaspending_api/api_docs/api_documentation/awards/idvs/awards.md b/usaspending_api/api_docs/api_documentation/awards/idvs/awards.md new file mode 100644 index 0000000000..840f173e74 --- /dev/null +++ b/usaspending_api/api_docs/api_documentation/awards/idvs/awards.md @@ -0,0 +1,68 @@ +## IDV Related Awards +**Route:** `/api/v2/awards/idvs/awards/` + +**Method:** `POST` + +Returns IDVs or contracts related to the requested Indefinite Delivery Vehicle award (IDV). + +## Request Parameters + +- award_id: (required) ID of award to retrieve. This can either be `generated_unique_award_id` or `id` from awards table. +- idv: (optional, default: True) True to return related IDVs or False to return related contracts. +- limit: (optional, default: 10) The number of records to return. +- page: (optional, default: 1) The page number to return. +- sort: (optional, default: `period_of_performance_start_date`) The field on which to sort results. Can be one of: `award_type`, `description`, `funding_agency`, `last_date_to_order`, `obligated_amount`, `period_of_performance_current_end_date`, `period_of_performance_start_date`, or `piid`. +- order: (optional, default `desc`) The sort order. Can be `desc` or `asc`. + +### Response (JSON) + +``` +{ + "results": [ + { + "award_id": 8330000, + "award_type": "DO", + "description": "4524345064!OTHER GROCERY AND R", + "funding_agency": "DEPARTMENT OF DEFENSE (DOD)", + "generated_unique_award_id": "CONT_AW_9700_9700_71T0_SPM30008D3155", + "last_date_to_order": null, + "obligated_amount": 4080.71, + "period_of_performance_current_end_date": "2013-05-06", + "period_of_performance_start_date": "2013-04-28", + "piid": "71T0" + } + ], + "page_metadata": { + "hasPrevious": false, + "next": 2, + "hasNext": true, + "previous": null, + "page": 1 + } +} +``` + +### Response Fields + +- `award_id`: Internal primary key of Award. +- `award_type`: Type of the award. See https://fedspendingtransparency.github.io/whitepapers/types/ for a better description. +- `description`: Description of the award as provided by the funding agency. +- `funding_agency`: Name of the agency that paid/is paying for the award. +- `generated_unique_award_id`: Natural key of Award. +- `last_date_to_order`: The date after which no more orders may be placed against the award. +- `obligated_amount`: The amount of money agreed upon for this award. +- `period_of_performance_current_end_date`: The date after which no additional costs may be incurred. May be extended. +- `period_of_performance_start_date`: The date before which no costs may be incurred. +- `piid`: Procurement instrument identifier for the award. +- `hasPrevious`: True if there's a previous page. False if not. +- `hasNext`: True if there's a next page. False if not. +- `previous`: The previous page number. +- `next`: The next page number. +- `page`: The current page number. + + +### Errors +Possible HTTP Status Codes: + +* 200: On success. +* 400 or 422 for various types of invalid POST data. diff --git a/usaspending_api/awards/award_id_helper.py b/usaspending_api/awards/award_id_helper.py new file mode 100644 index 0000000000..be89b76068 --- /dev/null +++ b/usaspending_api/awards/award_id_helper.py @@ -0,0 +1,81 @@ +from enum import Enum + + +# Award id is currently defined as bigint in the database. These are minimum +# and maximum values for bigint. While using sys.maxsize may look appealing, +# it is based on the word size of the machine which will cause problems on +# anything other than a 64 bit machine thus making it incorrect for this purpose. +MIN_INT = -2**63 # -9223372036854775808 +MAX_INT = 2**63 - 1 # +9223372036854775807 + + +class AwardIdType(Enum): + """ + Multi-state return value for detect_award_id_type. + """ + internal = 1 # award_id is probably an internal (integer) award id + generated = 2 # award_id is probably a generated_unique_award_id + unknown = 3 # type is unknown + + +def detect_award_id_type(award_id): + """ + THE PROBLEM: We have several places in code where we need to determine + whether a provided value is an internal award id (integer), a generated + award id (string), or neither. + + internal award id: 123456 + also an internal award id: "123456" + generated unique award id: "CONT_AW_4732_-NONE-_47QRAA18D0081_-NONE-" + + not an award id: 123.456 + also not an award id: {"award_id": "123456"} + + At present, this is being performed a few different ways, each of which has + its own set of problems. Simply attempting to cast using int will truncate + fractional bits for floats which is bad. Checking isdigit or isdecimal + will allow values to slip through that the database will not understand, + for example U+0660 (Arabic-Indic Digit Zero). + + THE SOLUTION: Here we will provide what we hope to be THE single solution + for this problem in a nice, little, encapsulated function using a + combination of type interrogation, hard casting, and exception handling. + + Input: + award_id - Can be anything, but hopefully it will be something capable + of being a valid award id. Only strings and integers have any hope of + being award ids at this time. + + Returns: + recast_award_id, AwardIdType - If award_id is an internal award id, an + integer will be returned along with AwardIdType.internal. If award_id + is a generated award id, a string will be returned along with + AwardIdType.generated. Otherwise, the original award_id and + AwardIdType.unknown will be returned. + + Examples: + detect_award_id_type(123456) => 123456, AwardIdType.internal + detect_award_id_type("123456") => 123456, AwardIdType.internal + detect_award_id_type("CONT_AW_-NONE-") => "CONT_AW_-NONE-", AwardIdType.generated + detect_award_id_type(123.456) => 123.456, AwardIdType.unknown + + IMPORTANT THING 1: I highly recommend you use the return value in your + queries instead of the original award_id provided to this function since + it is being cast to an appropriate data type for you. This eliminates odd + situations where the award_id is successfully cast to an integer, yet the + original value is something that the database does not understand (for + example, there are certain unicode characters that are numbers but the + database will not see them as such). + + IMPORTANT THING A: We will NOT be querying the database here. We are + strictly attempting to discern the type of the id based on its value. + """ + if type(award_id) in (int, str): + try: + _award_id = int(award_id) + if MIN_INT <= _award_id <= MAX_INT: + return _award_id, AwardIdType.internal + except ValueError: + pass + return str(award_id).upper(), AwardIdType.generated + return award_id, AwardIdType.unknown # ¯\_(ツ)_/¯ diff --git a/usaspending_api/awards/tests/test_award_id_helper.py b/usaspending_api/awards/tests/test_award_id_helper.py new file mode 100644 index 0000000000..8e9f0cb198 --- /dev/null +++ b/usaspending_api/awards/tests/test_award_id_helper.py @@ -0,0 +1,44 @@ +from usaspending_api.awards.award_id_helper import AwardIdType, detect_award_id_type, MAX_INT + + +def test_acceptable_internal_award_ids(): + + assert detect_award_id_type(12345) == (12345, AwardIdType.internal) + assert detect_award_id_type(-12345) == (-12345, AwardIdType.internal) + assert detect_award_id_type(+12345) == (12345, AwardIdType.internal) + assert detect_award_id_type('12345') == (12345, AwardIdType.internal) + assert detect_award_id_type('-12345') == (-12345, AwardIdType.internal) + assert detect_award_id_type('+12345') == (12345, AwardIdType.internal) + + assert detect_award_id_type(0) == (0, AwardIdType.internal) + assert detect_award_id_type(-0) == (0, AwardIdType.internal) + assert detect_award_id_type(+0) == (0, AwardIdType.internal) + assert detect_award_id_type('0') == (0, AwardIdType.internal) + assert detect_award_id_type('-0') == (0, AwardIdType.internal) + assert detect_award_id_type('+0') == (0, AwardIdType.internal) + + # A unicode Arabic-Indic zero. + assert detect_award_id_type('\u0660') == (0, AwardIdType.internal) + + +def test_acceptable_generated_award_ids(): + + assert detect_award_id_type('1a') == ('1A', AwardIdType.generated) + assert detect_award_id_type('1.1') == ('1.1', AwardIdType.generated) + assert detect_award_id_type('CONT_AW_4732_-NONE-_47QRAA18D0081_-NONE-') == \ + ('CONT_AW_4732_-NONE-_47QRAA18D0081_-NONE-', AwardIdType.generated) + assert detect_award_id_type('') == ('', AwardIdType.generated) # Unfortunately + assert detect_award_id_type(' ') == (' ', AwardIdType.generated) # Also unfortunately + + # An integer that's too big for the database will be cast to a generated + # type. Is this right? Who's to say. + assert detect_award_id_type(MAX_INT + 1) == (str(MAX_INT + 1), AwardIdType.generated) + + +def test_invalid_award_ids(): + + assert detect_award_id_type(None) == (None, AwardIdType.unknown) + assert detect_award_id_type(1.0) == (1.0, AwardIdType.unknown) + assert detect_award_id_type([1, 2, 3]) == ([1, 2, 3], AwardIdType.unknown) + assert detect_award_id_type(('1', '2', '3')) == (('1', '2', '3'), AwardIdType.unknown) + assert detect_award_id_type({'a': 1, 'b': 2}) == ({'a': 1, 'b': 2}, AwardIdType.unknown) diff --git a/usaspending_api/awards/tests/test_awards_idvs_amounts_v2.py b/usaspending_api/awards/tests/test_awards_idvs_amounts_v2.py new file mode 100644 index 0000000000..622a5c9ded --- /dev/null +++ b/usaspending_api/awards/tests/test_awards_idvs_amounts_v2.py @@ -0,0 +1,55 @@ +import json + +from django.test import TestCase +from model_mommy import mommy +from rest_framework import status + + +ENDPOINT = '/api/v2/awards/idvs/amounts/' + +EXPECTED_GOOD_OUTPUT = { + 'award_id': 1, + 'generated_unique_award_id': 'CONT_AW_2', + 'idv_count': 3, + 'contract_count': 4, + 'rollup_total_obligation': 10.04, + 'rollup_base_and_all_options_value': 11.05, + 'rollup_base_exercised_options_val': 12.06, +} + + +class IDVAwardsTestCase(TestCase): + + @classmethod + def setUpTestData(cls): + mommy.make('awards.Award', pk=1) + mommy.make( + 'awards.ParentAward', + award_id=1, + generated_unique_award_id='CONT_AW_2', + direct_idv_count=3, + direct_contract_count=4, + direct_total_obligation='5.01', + direct_base_and_all_options_value='6.02', + direct_base_exercised_options_val='7.03', + rollup_idv_count=8, + rollup_contract_count=9, + rollup_total_obligation='10.04', + rollup_base_and_all_options_value='11.05', + rollup_base_exercised_options_val='12.06', + ) + + def _test_get(self, _id, expected_response=None, expected_status_code=status.HTTP_200_OK): + endpoint = ENDPOINT + str(_id) + '/' + response = self.client.get(endpoint) + assert response.status_code == expected_status_code + if expected_response is not None: + assert json.loads(response.content.decode('utf-8')) == expected_response + + def test_awards_idvs_amounts_v2(self): + + self._test_get(1, EXPECTED_GOOD_OUTPUT) + self._test_get('CONT_AW_2', EXPECTED_GOOD_OUTPUT) + self._test_get(3, {'detail': 'No IDV award found with this id'}, status.HTTP_404_NOT_FOUND) + self._test_get('BOGUS_ID', {'detail': 'No IDV award found with this id'}, status.HTTP_404_NOT_FOUND) + self._test_get('INVALID_ID_&&&', expected_status_code=status.HTTP_404_NOT_FOUND) diff --git a/usaspending_api/awards/tests/test_awards_idvs_awards_v2.py b/usaspending_api/awards/tests/test_awards_idvs_awards_v2.py new file mode 100644 index 0000000000..74badb2cd9 --- /dev/null +++ b/usaspending_api/awards/tests/test_awards_idvs_awards_v2.py @@ -0,0 +1,318 @@ +import json + +from django.test import TestCase +from model_mommy import mommy +from rest_framework import status +from usaspending_api.awards.v2.views.idvs.awards import SORTABLE_COLUMNS + + +ENDPOINT = '/api/v2/awards/idvs/awards/' + +AWARD_COUNT = 15 +IDVS = (1, 2, 3, 4, 5, 7, 8) +PARENTS = {3: 1, 4: 1, 5: 1, 6: 1, 7: 2, 8: 2, 9: 2, 10: 2, 11: 7, 12: 7, 13: 8, 14: 8, 15: 9} + + +class IDVAwardsTestCase(TestCase): + + @classmethod + def setUpTestData(cls): + """ + You'll have to use your imagination a bit with my budget tree drawings. + These are the two hierarchies being built by this function. "I" means + IDV. "C" means contract. The number is the award id. So in this + drawing, I1 is the parent of I3, I4, I5, and C6. I2 is the grandparent + of C11, C12, C13, C14, and C15. Please note that the C9 -> C15 + relationship is actually invalid in the IDV world. I've added it for + testing purposes, however. Anyhow, hope this helps. There's a reason + I'm in software and not showing my wares at an art gallery somewhere. + + I1 I2 + I3 I4 I5 C6 I7 I8 C9 C10 + C11 C12 C13 C14 C15 + """ + + # We'll need some "latest transactions". + for transaction_id in range(1, AWARD_COUNT + 1): + mommy.make( + 'awards.TransactionNormalized', + id=transaction_id + ) + mommy.make( + 'awards.TransactionFPDS', + transaction_id=transaction_id, + funding_agency_name='funding_agency_name_%s' % transaction_id, + ordering_period_end_date='2018-01-%02d' % transaction_id + ) + + # We'll need some awards. + for award_id in range(1, AWARD_COUNT + 1): + parent_n = PARENTS.get(award_id) + mommy.make( + 'awards.Award', + id=award_id, + generated_unique_award_id='GENERATED_UNIQUE_AWARD_ID_%s' % award_id, + type=('IDV_%s' if award_id in IDVS else 'CONTRACT_%s') % award_id, + total_obligation=award_id, + piid='piid_%s' % award_id, + fpds_agency_id='fpds_agency_id_%s' % award_id, + parent_award_piid='piid_%s' % parent_n if parent_n else None, + fpds_parent_agency_id='fpds_agency_id_%s' % parent_n if parent_n else None, + latest_transaction_id=award_id, + type_description='type_description_%s' % award_id, + description='description_%s' % award_id, + period_of_performance_current_end_date='2018-03-%02d' % award_id, + period_of_performance_start_date='2018-02-%02d' % award_id, + ) + + # We'll need some parent_awards. + for award_id in IDVS: + mommy.make( + 'awards.ParentAward', + award_id=award_id, + generated_unique_award_id='GENERATED_UNIQUE_AWARD_ID_%s' % award_id, + rollup_total_obligation=award_id * 1000, + parent_award_id=PARENTS.get(award_id) + ) + + @staticmethod + def _generate_expected_response(previous, next, page, has_previous, has_next, *award_ids): + """ + Rather than manually generate an insane number of potential responses + to test the various parameter combinations, we're going to procedurally + generate them. award_ids is the list of ids we expect back from the + request in the order we expect them. Unfortunately, for this to work, + test data had to be generated in a specific way. If you change how + test data is generated you will probably also have to change this. For + example, IDVs have obligated amounts in the thousands whereas contracts + have obligated amounts in the single digits and teens. + """ + results = [] + for award_id in award_ids: + results.append({ + 'award_id': award_id, + 'award_type': 'type_description_%s' % award_id, + 'description': 'description_%s' % award_id, + 'funding_agency': 'funding_agency_name_%s' % award_id, + 'generated_unique_award_id': 'GENERATED_UNIQUE_AWARD_ID_%s' % award_id, + 'last_date_to_order': '2018-01-%02d' % award_id, + 'obligated_amount': float(award_id * (1000 if award_id in IDVS else 1)), + 'period_of_performance_current_end_date': '2018-03-%02d' % award_id, + 'period_of_performance_start_date': '2018-02-%02d' % award_id, + 'piid': 'piid_%s' % award_id + }) + + page_metadata = { + 'previous': previous, + 'next': next, + 'page': page, + 'hasPrevious': has_previous, + 'hasNext': has_next + } + + return {'results': results, 'page_metadata': page_metadata} + + def _test_post(self, request, expected_response_parameters_tuple=None, expected_status_code=status.HTTP_200_OK): + """ + Perform the actual request and interrogates the results. + + request is the Python dictionary that will be posted to the endpoint. + expected_response_parameters are the values that you would normally + pass into _generate_expected_response but we're going to do that + for you so just pass the parameters as a tuple or list. + expected_status_code is the HTTP status we expect to be returned from + the call to the endpoint. + + Returns... nothing useful. + """ + response = self.client.post(ENDPOINT, request) + assert response.status_code == expected_status_code + if expected_response_parameters_tuple is not None: + expected_response = self._generate_expected_response(*expected_response_parameters_tuple) + assert json.loads(response.content.decode('utf-8')) == expected_response + + def test_defaults(self): + + self._test_post( + {'award_id': 1}, + (None, None, 1, False, False, 5, 4, 3) + ) + + self._test_post( + {'award_id': 'GENERATED_UNIQUE_AWARD_ID_1'}, + (None, None, 1, False, False, 5, 4, 3) + ) + + def test_with_nonexistent_id(self): + + self._test_post( + {'award_id': 0}, + (None, None, 1, False, False) + ) + + self._test_post( + {'award_id': 'GENERATED_UNIQUE_AWARD_ID_0'}, + (None, None, 1, False, False) + ) + + def test_with_bogus_id(self): + + self._test_post( + {'award_id': None}, + (None, None, 1, False, False) + ) + + def test_idv_flag(self): + + self._test_post( + {'award_id': 1, 'idv': True}, + (None, None, 1, False, False, 5, 4, 3) + ) + + self._test_post( + {'award_id': 1, 'idv': False}, + (None, None, 1, False, False, 6) + ) + + self._test_post( + {'award_id': 1, 'idv': 'BOGUS IDV'}, + expected_status_code=status.HTTP_400_BAD_REQUEST + ) + + def test_limit_values(self): + + self._test_post( + {'award_id': 1, 'limit': 1}, + (None, 2, 1, False, True, 5) + ) + + self._test_post( + {'award_id': 1, 'limit': 5}, + (None, None, 1, False, False, 5, 4, 3) + ) + + self._test_post( + {'award_id': 1, 'limit': 0}, + expected_status_code=status.HTTP_422_UNPROCESSABLE_ENTITY + ) + + self._test_post( + {'award_id': 1, 'limit': 2000000000}, + expected_status_code=status.HTTP_422_UNPROCESSABLE_ENTITY + ) + + self._test_post( + {'award_id': 1, 'limit': {'BOGUS': 'LIMIT'}}, + expected_status_code=status.HTTP_400_BAD_REQUEST + ) + + def test_page_values(self): + + self._test_post( + {'award_id': 1, 'limit': 1, 'page': 2}, + (1, 3, 2, True, True, 4) + ) + + self._test_post( + {'award_id': 1, 'limit': 1, 'page': 3}, + (2, None, 3, True, False, 3) + ) + + self._test_post( + {'award_id': 1, 'limit': 1, 'page': 4}, + (3, None, 4, True, False) + ) + + self._test_post( + {'award_id': 1, 'limit': 1, 'page': 0}, + expected_status_code=status.HTTP_422_UNPROCESSABLE_ENTITY + ) + + self._test_post( + {'award_id': 1, 'limit': 1, 'page': 'BOGUS PAGE'}, + expected_status_code=status.HTTP_400_BAD_REQUEST + ) + + def test_sort_columns(self): + + for sortable_column in SORTABLE_COLUMNS: + + self._test_post( + {'award_id': 1, 'order': 'desc', 'sort': sortable_column}, + (None, None, 1, False, False, 5, 4, 3) + ) + + self._test_post( + {'award_id': 1, 'order': 'asc', 'sort': sortable_column}, + (None, None, 1, False, False, 3, 4, 5) + ) + + self._test_post( + {'award_id': 1, 'sort': 'BOGUS FIELD'}, + expected_status_code=status.HTTP_400_BAD_REQUEST + ) + + def test_sort_order_values(self): + + self._test_post( + {'award_id': 1, 'order': 'desc'}, + (None, None, 1, False, False, 5, 4, 3) + ) + + self._test_post( + {'award_id': 1, 'order': 'asc'}, + (None, None, 1, False, False, 3, 4, 5) + ) + + self._test_post( + {'award_id': 1, 'order': 'BOGUS ORDER'}, + expected_status_code=status.HTTP_400_BAD_REQUEST + ) + + def test_complete_queries(self): + + self._test_post( + {'award_id': 1, 'idv': True, 'limit': 3, 'page': 1, 'sort': 'description', 'order': 'asc'}, + (None, None, 1, False, False, 3, 4, 5) + ) + + self._test_post( + {'award_id': 1, 'idv': False, 'limit': 3, 'page': 1, 'sort': 'description', 'order': 'asc'}, + (None, None, 1, False, False, 6) + ) + + def test_no_grandchildren_returned(self): + + self._test_post( + {'award_id': 2, 'idv': True}, + (None, None, 1, False, False, 8, 7) + ) + + self._test_post( + {'award_id': 2, 'idv': False}, + (None, None, 1, False, False, 10, 9) + ) + + def test_no_parents_returned(self): + + self._test_post( + {'award_id': 7, 'idv': True}, + (None, None, 1, False, False) + ) + + self._test_post( + {'award_id': 7, 'idv': False}, + (None, None, 1, False, False, 12, 11) + ) + + def test_nothing_returned_for_bogus_contract_relationship(self): + + self._test_post( + {'award_id': 9, 'idv': True}, + (None, None, 1, False, False) + ) + + self._test_post( + {'award_id': 9, 'idv': False}, + (None, None, 1, False, False) + ) diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 37271167c1..5125b13b1c 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -190,56 +190,6 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): assert json.loads(resp.content.decode("utf-8")) == expected_response_cont -@pytest.mark.django_db -def test_idv_award_amount_endpoint(client): - - mommy.make('awards.Award', pk=1) - mommy.make( - 'awards.ParentAward', - award_id=1, - generated_unique_award_id='CONT_AW_2', - direct_idv_count=3, - direct_contract_count=4, - direct_total_obligation='5.01', - direct_base_and_all_options_value='6.02', - direct_base_exercised_options_val='7.03', - rollup_idv_count=8, - rollup_contract_count=9, - rollup_total_obligation='10.04', - rollup_base_and_all_options_value='11.05', - rollup_base_exercised_options_val='12.06', - ) - - output_idv_amounts = { - 'award_id': 1, - 'generated_unique_award_id': 'CONT_AW_2', - 'idv_count': 3, - 'contract_count': 4, - 'rollup_total_obligation': 10.04, - 'rollup_base_and_all_options_value': 11.05, - 'rollup_base_exercised_options_val': 12.06, - } - - resp = client.get('/api/v2/awards/idvs/amounts/1/') - assert resp.status_code == status.HTTP_200_OK - assert json.loads(resp.content.decode('utf-8')) == output_idv_amounts - - resp = client.get('/api/v2/awards/idvs/amounts/CONT_AW_2/') - assert resp.status_code == status.HTTP_200_OK - assert json.loads(resp.content.decode('utf-8')) == output_idv_amounts - - resp = client.get('/api/v2/awards/idvs/amounts/3/') - assert resp.status_code == status.HTTP_404_NOT_FOUND - assert json.loads(resp.content.decode('utf-8')) == {'message': 'No IDV award found with this id'} - - resp = client.get('/api/v2/awards/idvs/amounts/BOGUS_ID/') - assert resp.status_code == status.HTTP_404_NOT_FOUND - assert json.loads(resp.content.decode('utf-8')) == {'message': 'No IDV award found with this id'} - - resp = client.get('/api/v2/awards/idvs/amounts/INVALID_ID_&&&/') - assert resp.status_code == status.HTTP_404_NOT_FOUND - - expected_response_asst = { "id": 1, "type": "11", diff --git a/usaspending_api/awards/v2/filters/view_selector.py b/usaspending_api/awards/v2/filters/view_selector.py index e7db0063f4..6c4840f5df 100644 --- a/usaspending_api/awards/v2/filters/view_selector.py +++ b/usaspending_api/awards/v2/filters/view_selector.py @@ -277,7 +277,6 @@ def spending_by_geography(filters): 'SummaryTransactionView', 'UniversalTransactionView' ] - model = None for view in view_chain: if can_use_view(filters, view): queryset = get_view_queryset(filters, view) @@ -291,7 +290,6 @@ def spending_by_geography(filters): def spending_by_award_count(filters): view_chain = ['SummaryAwardView', 'UniversalAwardView'] - model = None for view in view_chain: if can_use_view(filters, view): queryset = get_view_queryset(filters, view) @@ -311,7 +309,6 @@ def download_transaction_count(filters): 'SummaryTransactionView', 'UniversalTransactionView' ] - model = None for view in view_chain: if can_use_view(filters, view): queryset = get_view_queryset(filters, view) @@ -331,7 +328,6 @@ def transaction_spending_summary(filters): 'SummaryTransactionView', 'UniversalTransactionView' ] - model = None for view in view_chain: if can_use_view(filters, view): queryset = get_view_queryset(filters, view) @@ -347,8 +343,8 @@ def recipient_totals(filters): view_chain = [ 'SummaryTransactionMonthView', 'SummaryTransactionView', - 'UniversalTransactionView'] - model = None + 'UniversalTransactionView' + ] for view in view_chain: if can_use_view(filters, view): queryset = get_view_queryset(filters, view) diff --git a/usaspending_api/awards/v2/urls_awards.py b/usaspending_api/awards/v2/urls_awards.py index ac9188d702..788e5606c0 100644 --- a/usaspending_api/awards/v2/urls_awards.py +++ b/usaspending_api/awards/v2/urls_awards.py @@ -1,8 +1,7 @@ from django.conf.urls import url -from usaspending_api.awards.v2.views.awards import AwardLastUpdatedViewSet, AwardRetrieveViewSet, IDVAmountsViewSet +from usaspending_api.awards.v2.views.awards import AwardLastUpdatedViewSet, AwardRetrieveViewSet urlpatterns = [ url(r'^last_updated', AwardLastUpdatedViewSet.as_view()), - url(r'^idvs/amounts/(?P[A-Za-z0-9_. -]+)/$', IDVAmountsViewSet.as_view()), - url(r'(?P[A-Za-z0-9_. -]+)/$', AwardRetrieveViewSet.as_view()), + url(r'^(?P[A-Za-z0-9_. -]+)/$', AwardRetrieveViewSet.as_view()), ] diff --git a/usaspending_api/awards/v2/urls_idv_awards.py b/usaspending_api/awards/v2/urls_idv_awards.py new file mode 100644 index 0000000000..ae6e3bdc88 --- /dev/null +++ b/usaspending_api/awards/v2/urls_idv_awards.py @@ -0,0 +1,7 @@ +from django.conf.urls import url +from usaspending_api.awards.v2.views.idvs import IDVAmountsViewSet, IDVAwardsViewSet + +urlpatterns = [ + url(r'^amounts/(?P[A-Za-z0-9_. -]+)/$', IDVAmountsViewSet.as_view()), + url(r'^awards/$', IDVAwardsViewSet.as_view()), +] diff --git a/usaspending_api/awards/v2/views/awards.py b/usaspending_api/awards/v2/views/awards.py index 7f97ef8188..5da42ae821 100644 --- a/usaspending_api/awards/v2/views/awards.py +++ b/usaspending_api/awards/v2/views/awards.py @@ -1,14 +1,11 @@ import logging -from collections import OrderedDict - from django.db.models import Max -from rest_framework import status from rest_framework.request import Request from rest_framework.response import Response -from usaspending_api.awards.models import Award, ParentAward +from usaspending_api.awards.models import Award from usaspending_api.awards.serializers_v2.serializers import AwardContractSerializerV2, AwardMiscSerializerV2,\ AwardIDVSerializerV2 from usaspending_api.common.cache_decorator import cache_response @@ -91,59 +88,3 @@ def get(self, request: Request, requested_award: str) -> Response: request_data = self._parse_and_validate_request(requested_award) response = self._business_logic(request_data) return Response(response) - - -class IDVAmountsViewSet(APIDocumentationView): - """Return IDV values from parent_award table. - endpoint_doc: /awards/idvs/amounts.md - """ - - @staticmethod - def _parse_and_validate_request(requested_award: str) -> dict: - try: - request_dict = {'award_id': int(requested_award)} - models = [{ - 'key': 'award_id', - 'name': 'award_id', - 'type': 'integer', - 'optional': False, - }] - except ValueError: - request_dict = {'generated_unique_award_id': requested_award.upper()} - models = [{ - 'key': 'generated_unique_award_id', - 'name': 'generated_unique_award_id', - 'type': 'text', - 'text_type': 'raw', - 'optional': False, - }] - return TinyShield(models).block(request_dict) - - @staticmethod - def _business_logic(request_data: dict) -> dict: - try: - parent_award = ParentAward.objects.get(**request_data) - return { - 'data': OrderedDict(( - ('award_id', parent_award.award_id), - ('generated_unique_award_id', parent_award.generated_unique_award_id), - ('idv_count', parent_award.direct_idv_count), - ('contract_count', parent_award.direct_contract_count), - ('rollup_total_obligation', parent_award.rollup_total_obligation), - ('rollup_base_and_all_options_value', parent_award.rollup_base_and_all_options_value), - ('rollup_base_exercised_options_val', parent_award.rollup_base_exercised_options_val), - )), - 'status': status.HTTP_200_OK - } - except ParentAward.DoesNotExist: - logger.info("No IDV Award found where '%s' is '%s'" % next(iter(request_data.items()))) - return { - 'data': OrderedDict({'message': 'No IDV award found with this id'}), - 'status': status.HTTP_404_NOT_FOUND - } - - @cache_response() - def get(self, request: Request, requested_award: str) -> Response: - """Return IDV counts and sums""" - request_data = self._parse_and_validate_request(requested_award) - return Response(**self._business_logic(request_data)) diff --git a/usaspending_api/awards/v2/views/idvs/__init__.py b/usaspending_api/awards/v2/views/idvs/__init__.py new file mode 100644 index 0000000000..19feeae88e --- /dev/null +++ b/usaspending_api/awards/v2/views/idvs/__init__.py @@ -0,0 +1,2 @@ +from .amounts import IDVAmountsViewSet +from .awards import IDVAwardsViewSet diff --git a/usaspending_api/awards/v2/views/idvs/amounts.py b/usaspending_api/awards/v2/views/idvs/amounts.py new file mode 100644 index 0000000000..baf488ee35 --- /dev/null +++ b/usaspending_api/awards/v2/views/idvs/amounts.py @@ -0,0 +1,66 @@ +import logging + +from collections import OrderedDict + +from rest_framework.exceptions import NotFound +from rest_framework.request import Request +from rest_framework.response import Response + +from usaspending_api.awards.award_id_helper import detect_award_id_type, AwardIdType +from usaspending_api.awards.models import ParentAward +from usaspending_api.common.cache_decorator import cache_response +from usaspending_api.common.views import APIDocumentationView +from usaspending_api.core.validator.tinyshield import TinyShield + + +logger = logging.getLogger('console') + + +class IDVAmountsViewSet(APIDocumentationView): + """Returns counts and dollar figures for a specific IDV. + endpoint_doc: /awards/idvs/amounts.md + """ + + @staticmethod + def _parse_and_validate_request(requested_award: str) -> dict: + award_id, award_id_type = detect_award_id_type(requested_award) + if award_id_type is AwardIdType.internal: + request_dict = {'award_id': award_id} + models = [{ + 'key': 'award_id', + 'name': 'award_id', + 'type': 'integer', + 'optional': False, + }] + else: + request_dict = {'generated_unique_award_id': award_id} + models = [{ + 'key': 'generated_unique_award_id', + 'name': 'generated_unique_award_id', + 'type': 'text', + 'text_type': 'search', + 'optional': False, + }] + return TinyShield(models).block(request_dict) + + @staticmethod + def _business_logic(request_data: dict) -> OrderedDict: + try: + parent_award = ParentAward.objects.get(**request_data) + return OrderedDict(( + ('award_id', parent_award.award_id), + ('generated_unique_award_id', parent_award.generated_unique_award_id), + ('idv_count', parent_award.direct_idv_count), + ('contract_count', parent_award.direct_contract_count), + ('rollup_total_obligation', parent_award.rollup_total_obligation), + ('rollup_base_and_all_options_value', parent_award.rollup_base_and_all_options_value), + ('rollup_base_exercised_options_val', parent_award.rollup_base_exercised_options_val), + )) + except ParentAward.DoesNotExist: + logger.info("No IDV Award found where '%s' is '%s'" % next(iter(request_data.items()))) + raise NotFound("No IDV award found with this id") + + @cache_response() + def get(self, request: Request, requested_award: str) -> Response: + request_data = self._parse_and_validate_request(requested_award) + return Response(self._business_logic(request_data)) diff --git a/usaspending_api/awards/v2/views/idvs/awards.py b/usaspending_api/awards/v2/views/idvs/awards.py new file mode 100644 index 0000000000..1d15b32ac9 --- /dev/null +++ b/usaspending_api/awards/v2/views/idvs/awards.py @@ -0,0 +1,151 @@ +from collections import OrderedDict +from copy import deepcopy + +from django.db import connection +from psycopg2.sql import Identifier, Literal, SQL +from rest_framework.request import Request +from rest_framework.response import Response + +from usaspending_api.awards.award_id_helper import detect_award_id_type, AwardIdType +from usaspending_api.common.cache_decorator import cache_response +from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata +from usaspending_api.common.views import APIDocumentationView +from usaspending_api.core.validator.pagination import PAGINATION +from usaspending_api.core.validator.tinyshield import TinyShield +from usaspending_api.etl.broker_etl_helpers import dictfetchall + + +# Columns upon which the client is allowed to sort. +SORTABLE_COLUMNS = ( + 'award_type', + 'description', + 'funding_agency', + 'last_date_to_order', + 'obligated_amount', + 'period_of_performance_current_end_date', + 'period_of_performance_start_date', + 'piid', +) + +DEFAULT_SORT_COLUMN = 'period_of_performance_start_date' + +GET_IDVS_SQL = SQL(""" + select + ac.id award_id, + ac.type_description award_type, + ac.description, + tf.funding_agency_name funding_agency, + ac.generated_unique_award_id, + tf.ordering_period_end_date last_date_to_order, + pac.rollup_total_obligation obligated_amount, + ac.period_of_performance_current_end_date, + ac.period_of_performance_start_date, + ac.piid + from + parent_award pap + inner join parent_award pac on pac.parent_award_id = pap.award_id + inner join awards ac on ac.id = pac.award_id + inner join transaction_fpds tf on tf.transaction_id = ac.latest_transaction_id + where + pap.{award_id_column} = %s + order by + {sort_column} {sort_direction}, ac.id {sort_direction} + limit {limit} offset {offset} +""") + +GET_CONTRACTS_SQL = SQL(""" + select + ac.id award_id, + ac.type_description award_type, + ac.description, + tf.funding_agency_name funding_agency, + ac.generated_unique_award_id, + tf.ordering_period_end_date last_date_to_order, + ac.total_obligation obligated_amount, + ac.period_of_performance_current_end_date, + ac.period_of_performance_start_date, + ac.piid + from + parent_award pap + inner join awards ap on ap.id = pap.award_id + inner join awards ac on ac.fpds_parent_agency_id = ap.fpds_agency_id and ac.parent_award_piid = ap.piid and + ac.type not like 'IDV\_%%' + inner join transaction_fpds tf on tf.transaction_id = ac.latest_transaction_id + where + pap.{award_id_column} = %s + order by + {sort_column} {sort_direction}, ac.id {sort_direction} + limit {limit} offset {offset} +""") + + +def _prepare_tiny_shield_rules(): + """ + Our TinyShield rules never change. Encapsulate them here and store them + once in TINY_SHIELD_RULES. + """ + + # This endpoint supports paging. + models = deepcopy(PAGINATION) + + # Add the list of sortable columns for validation. + sort_rule = TinyShield.get_model_by_name(models, 'sort') + sort_rule['type'] = 'enum' + sort_rule['enum_values'] = SORTABLE_COLUMNS + sort_rule['default'] = DEFAULT_SORT_COLUMN + + # Add additional models for the award id and the idv filter. + models.extend([ + # Award id can be either an integer or a string depending upon whether + # it's an internal id or a unique generated id. + {'key': 'award_id', 'name': 'award_id', 'type': 'any', 'optional': False, 'models': [ + {'type': 'integer'}, + {'type': 'text', 'text_type': 'search'} + ]}, + {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True} + ]) + + return models + + +TINY_SHIELD_RULES = _prepare_tiny_shield_rules() + + +class IDVAwardsViewSet(APIDocumentationView): + """Returns the direct children of an IDV. + endpoint_doc: /awards/idvs/awards.md + """ + + @staticmethod + def _parse_and_validate_request(request: Request) -> dict: + return TinyShield(deepcopy(TINY_SHIELD_RULES)).block(request) + + @staticmethod + def _business_logic(request_data: dict) -> list: + award_id, award_id_type = detect_award_id_type(request_data['award_id']) + award_id_column = 'award_id' if award_id_type is AwardIdType.internal else 'generated_unique_award_id' + sql = GET_IDVS_SQL if request_data['idv'] else GET_CONTRACTS_SQL + sql = sql.format( + award_id_column=Identifier(award_id_column), + award_id=Literal(award_id), + sort_column=Identifier(request_data['sort']), + sort_direction=SQL(request_data['order']), + limit=Literal(request_data['limit'] + 1), + offset=Literal((request_data['page'] - 1) * request_data['limit']), + ) + with connection.cursor() as cursor: + cursor.execute(sql, [award_id]) + return dictfetchall(cursor) + + @cache_response() + def post(self, request: Request) -> Response: + request_data = self._parse_and_validate_request(request.data) + results = self._business_logic(request_data) + page_metadata = get_simple_pagination_metadata(len(results), request_data['limit'], request_data['page']) + + response = OrderedDict(( + ('results', results[:request_data['limit']]), + ('page_metadata', page_metadata) + )) + + return Response(response) diff --git a/usaspending_api/awards/v2/views/transactions.py b/usaspending_api/awards/v2/views/transactions.py index 293b8866a6..6a8be10472 100644 --- a/usaspending_api/awards/v2/views/transactions.py +++ b/usaspending_api/awards/v2/views/transactions.py @@ -4,6 +4,7 @@ from rest_framework.request import Request from rest_framework.response import Response +from usaspending_api.awards.award_id_helper import AwardIdType, detect_award_id_type from usaspending_api.awards.models import TransactionNormalized from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata @@ -51,12 +52,13 @@ def _business_logic(self, request_data: dict) -> list: lower_limit = (request_data["page"] - 1) * request_data["limit"] upper_limit = request_data["page"] * request_data["limit"] - if request_data["award_id"].isdigit(): + award_id, id_type = detect_award_id_type(request_data['award_id']) + if id_type is AwardIdType.internal: # Award ID - filter = {'award_id': request_data["award_id"]} + filter = {'award_id': award_id} else: # Generated Award ID - filter = {'award__generated_unique_award_id': request_data["award_id"]} + filter = {'award__generated_unique_award_id': award_id} queryset = (TransactionNormalized.objects.all() .values(*list(self.transaction_lookup.values())) diff --git a/usaspending_api/core/validator/tests/unit/test_tinyshield.py b/usaspending_api/core/validator/tests/unit/test_tinyshield.py index 2535eedc4a..99a60b6376 100644 --- a/usaspending_api/core/validator/tests/unit/test_tinyshield.py +++ b/usaspending_api/core/validator/tests/unit/test_tinyshield.py @@ -1,5 +1,7 @@ import copy +import pytest +from usaspending_api.common.exceptions import UnprocessableEntityException from usaspending_api.core.validator.award_filter import AWARD_FILTER from usaspending_api.core.validator.helpers import validate_array from usaspending_api.core.validator.helpers import validate_boolean @@ -23,7 +25,7 @@ 'optional': True, 'value': 3.14, 'min': 2, 'max': 4} TEXT_RULE = {'name': 'test', 'type': 'text', 'key': 'filters|test', 'optional': True, 'value': "hello world", "text_type": "search"} -INTEGER_RULE = {'name': 'test', 'type': 'float', 'key': 'filters|test', +INTEGER_RULE = {'name': 'test', 'type': 'integer', 'key': 'filters|test', 'optional': True, 'value': 3, 'min': 2, 'max': 4} OBJECT_RULE = {'name': 'test', 'type': 'object', 'key': 'filters|test', 'object_keys': { @@ -107,7 +109,7 @@ TS = None ''' -Beacuse these functions all raise Exceptions on failure, all we need to do to write the unit tests is call the function. +Because these functions all raise Exceptions on failure, all we need to do to write the unit tests is call the function. If an exception is raised, the test will fail ''' @@ -170,3 +172,62 @@ def test_parse_request(): def test_enforce_rules(): TS.enforce_rules() assert TS.data == FILTER_OBJ + + +# Test the "any" rule. +def test_any_rule(): + models = [{ + 'name': 'value', + 'key': 'value', + 'type': 'any', + 'models': [ + {'type': 'integer'}, + {'type': 'text', 'text_type': 'search'} + ] + }] + + # Test integer and random other key. + ts = TinyShield(models).block({'value': 1, 'another_value': 2}) + assert ts['value'] == 1 + assert ts.get('another_value') is None + + # Test integer masquerading as a string. + ts = TinyShield(models).block({'value': '1'}) + assert ts['value'] == 1 + + # Test string. + ts = TinyShield(models).block({'value': 'XYZ'}) + assert ts['value'] == 'XYZ' + + # Test list (which should blow up). + with pytest.raises(UnprocessableEntityException): + TinyShield(models).block({'value': ['XYZ']}) + + # Test with optional 'value' missing. + ts = TinyShield(models).block({'another_value': 2}) + assert ts.get('value') is None + assert ts.get('another_value') is None + + # Make 'value' required then run the same tests as above. + models[0]['optional'] = False + + # Test integer and random other key. + ts = TinyShield(models).block({'value': 1, 'another_value': 2}) + assert ts['value'] == 1 + assert ts.get('another_value') is None + + # Test integer masquerading as a string. + ts = TinyShield(models).block({'value': '1'}) + assert ts['value'] == 1 + + # Test string. + ts = TinyShield(models).block({'value': 'XYZ'}) + assert ts['value'] == 'XYZ' + + # Test list (which should blow up). + with pytest.raises(UnprocessableEntityException): + TinyShield(models).block({'value': ['XYZ']}) + + # Test with required 'value' missing. + with pytest.raises(UnprocessableEntityException): + TinyShield(models).block({'another_value': 2}) diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py index f65498901c..039a3d7a5d 100644 --- a/usaspending_api/core/validator/tinyshield.py +++ b/usaspending_api/core/validator/tinyshield.py @@ -2,8 +2,8 @@ import copy from usaspending_api.common.exceptions import UnprocessableEntityException -from usaspending_api.core.validator.helpers import SUPPORTED_TEXT_TYPES -from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR, MAX_ITEMS +from usaspending_api.core.validator.helpers import INVALID_TYPE_MSG, MAX_ITEMS +from usaspending_api.core.validator.helpers import SUPPORTED_TEXT_TYPES, TINY_SHIELD_SEPARATOR from usaspending_api.core.validator.helpers import validate_array from usaspending_api.core.validator.helpers import validate_boolean from usaspending_api.core.validator.helpers import validate_datetime @@ -16,6 +16,11 @@ logger = logging.getLogger('console') VALIDATORS = { + 'any': { + # "any" does not use a "func", rather, it calls funcs for child models + 'required_fields': ['models'], + 'defaults': {}, + }, 'array': { 'func': validate_array, 'required_fields': ['array_type'], @@ -125,6 +130,7 @@ class TinyShield: name: this field doesn't seem to actually do anything, but it does need to be unique! key: dict|subitem|all|split|by|pipes type: type of validator + any * array boolean date @@ -136,6 +142,8 @@ class TinyShield: passthrough text schema + models: + {provide sub-models - used for "any" type} text_type: search raw @@ -150,6 +158,18 @@ class TinyShield: If False, then key-value must be present. Overrides `default` } + * "any" is a special beast as it contains a collection of models, any one of which may match the value provided. + The first model to match is the one that's used. You could use "any" to, say, accept a field that could be either + an integer or a string (like an internal award id vs. a generated award id). Child models of an "any" rule are + always optional and inherit their parent's name and key and, as such, those keys are always ignored. For example: + + models = [ + {'key': 'id', 'name': 'id', 'type': 'any', 'optional': False, 'models': [ + {'type': 'integer'}, + {'type': 'text', 'text_type': 'search'} + ]}, + ] + RETURNS A dictionary of the validated data keyed on model.key from the input "model" list provided during instantiation. @@ -165,37 +185,56 @@ def block(self, request): self.enforce_rules() return self.data - def check_models(self, models): + @staticmethod + def check_model(model, in_any=False): # Confirm required fields (both baseline and type-specific) are in the model base_minimum_fields = ('name', 'key', 'type') - for model in models: - if not all(field in model.keys() for field in base_minimum_fields): - raise Exception('Model {} missing a base required field [{}]'.format(model, base_minimum_fields)) + if not all(field in model.keys() for field in base_minimum_fields): + raise Exception('Model {} missing a base required field [{}]'.format(model, base_minimum_fields)) + + if model['type'] not in VALIDATORS: + raise Exception('Invalid model type [{}] provided in description'.format(model['type'])) - if model['type'] not in VALIDATORS: - raise Exception('Invalid model type [{}] provided in description'.format(model['type'])) + type_description = VALIDATORS[model['type']] + required_fields = type_description['required_fields'] - type_description = VALIDATORS[model['type']] - required_fields = type_description['required_fields'] + for required_field in required_fields: + if required_field not in model: + raise Exception('Model {} missing a type required field: {}'.format(model, required_field)) - for required_field in required_fields: - if required_field not in model: - raise Exception('Model {} missing a type required field: {}'.format(model, required_field)) + if model.get('text_type') and model['text_type'] not in SUPPORTED_TEXT_TYPES: + msg = 'Invalid model \'{key}\': \'{text_type}\' is not a valid text_type'.format(**model) + raise Exception(msg + ' Possible types: {}'.format(SUPPORTED_TEXT_TYPES)) - if model.get('text_type') and model['text_type'] not in SUPPORTED_TEXT_TYPES: - msg = 'Invalid model \'{key}\': \'{text_type}\' is not a valid text_type'.format(**model) - raise Exception(msg + ' Possible types: {}'.format(SUPPORTED_TEXT_TYPES)) + for default_key, default_value in type_description['defaults'].items(): + model[default_key] = model.get(default_key, default_value) - for default_key, default_value in type_description['defaults'].items(): - model[default_key] = model.get(default_key, default_value) + model['optional'] = model.get('optional', True) - model['optional'] = model.get('optional', True) + if model['type'] == 'any': + if in_any is True: + raise Exception('Nested "any" rules are not supported.') + # "any" child models are always optional and inherit their parent's name and key. + for sub_model in model['models']: + sub_model['name'] = model['name'] + sub_model['key'] = model['key'] + sub_model['optional'] = True + TinyShield.check_model(sub_model, True) + + return model + + @staticmethod + def check_models(models): + + for model in models: + TinyShield.check_model(model) # Check to ensure unique names for destination dictionary keys = [x['name'] for x in models if x['type'] != 'schema'] # ignore schema as they are schema-only if len(keys) != len(set(keys)): raise Exception('Duplicate destination keys provided. Name values must be unique') + return models def parse_request(self, request): @@ -227,7 +266,7 @@ def enforce_rules(self): def apply_rule(self, rule): if rule.get('allow_nulls', False) and rule['value'] is None: return rule['value'] - elif rule['type'] not in ('array', 'object'): + elif rule['type'] not in ('array', 'object', 'any'): if rule['type'] in VALIDATORS: return VALIDATORS[rule['type']]['func'](rule) else: @@ -268,6 +307,21 @@ def apply_rule(self, rule): child_rule = self.promote_subrules(child_rule, v) object_result[k] = self.apply_rule(child_rule) return object_result + # Any is a "special" type since it is is really a collection of other rules. + elif rule['type'] == 'any': + for child_rule in rule['models']: + child_rule['value'] = rule['value'] + try: + # First successful rule wins. + return self.apply_rule(child_rule) + except Exception: + pass + # No rules succeeded. + raise UnprocessableEntityException(INVALID_TYPE_MSG.format( + key=rule['key'], + value=rule['value'], + type=', '.join(sorted([m['type'] for m in rule['models']])) + )) def promote_subrules(self, child_rule, source={}): param_type = child_rule['type'] @@ -302,3 +356,15 @@ def recurse_append(self, struct, mydict, data): else: mydict[level] = {} self.recurse_append(struct, mydict[level], data) + + @staticmethod + def get_model_by_name(models, name): + """ + Little helper function to return a TinyShield model from a list of + models given the model's name. Returns None if the model was not + found. + """ + for model in models: + if model.get('name') == name: + return model + return None diff --git a/usaspending_api/urls.py b/usaspending_api/urls.py index 8e2ec2f8b3..2fbe7a2ec0 100644 --- a/usaspending_api/urls.py +++ b/usaspending_api/urls.py @@ -32,6 +32,7 @@ url(r'^api/v1/transactions/', include('usaspending_api.awards.v1.urls_transactions')), url(r'^api/v2/autocomplete/', include('usaspending_api.references.v2.urls_autocomplete')), url(r'^api/v2/awards/', include('usaspending_api.awards.v2.urls_awards')), + url(r'^api/v2/awards/idvs/', include('usaspending_api.awards.v2.urls_idv_awards')), url(r'^api/v2/award_spending/', include('usaspending_api.awards.v2.urls_award_spending')), url(r'^api/v2/subawards/', include('usaspending_api.awards.v2.urls_subawards')), url(r'^api/v2/budget_authority/', include('usaspending_api.accounts.urls_budget_authority')), From 925793d91367523562e3aa800bb49ef7920f69fb Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Mon, 4 Feb 2019 11:18:13 -0500 Subject: [PATCH 26/52] minor bug fix --- .../awards/tests/test_awards_v2.py | 19 +++++++++++++++++++ usaspending_api/awards/v2/data_layer/orm.py | 3 +++ 2 files changed, 22 insertions(+) diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 23dbe2b843..3f27442219 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -195,8 +195,22 @@ def awards_and_transactions(db): "total_subaward_amount": 12345.00, "subaward_count": 10, } + + award_3_model = { + "pk": 3, + "type": "IDV_A", + "type_description": "AN IDV", + "category": "idv", + } mommy.make("awards.Award", **award_1_model) mommy.make("awards.Award", **award_2_model) + mommy.make("awards.Award", **award_3_model) + + parent_award_3_model = { + "award_id": award_3_model["pk"], + "parent_award_id": None + } + mommy.make("awards.ParentAward", **parent_award_3_model) @pytest.mark.django_db @@ -233,6 +247,11 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): assert resp.status_code == status.HTTP_200_OK assert json.loads(resp.content.decode("utf-8")) == expected_response_cont + # Bug fix rolled into DEV-2034. This used to throw an exception because + # parent award didn't have a parent. + resp = client.get("/api/v2/awards/3/") + assert resp.status_code == status.HTTP_200_OK + @pytest.mark.django_db def test_idv_award_amount_endpoint(client): diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index eee1d50558..f4a327badd 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -212,6 +212,9 @@ def fetch_parent_award_details(guai): .first() ) + if not parent_award: + return None + parent_object = OrderedDict( [ ("agency_id", parent_award["latest_transaction__contract_data__agency_id"]), From 7aef925f9f9df7e08c343c5ba20be75b97307075 Mon Sep 17 00:00:00 2001 From: Justin Le Date: Mon, 4 Feb 2019 16:10:52 -0500 Subject: [PATCH 27/52] Fixed quasi-JSON/Python formatting to be wholly JSON --- .../references/data_dictionary.md | 129 ++++++++++++------ 1 file changed, 87 insertions(+), 42 deletions(-) diff --git a/usaspending_api/api_docs/api_documentation/references/data_dictionary.md b/usaspending_api/api_docs/api_documentation/references/data_dictionary.md index f26ea9d458..711baeed08 100644 --- a/usaspending_api/api_docs/api_documentation/references/data_dictionary.md +++ b/usaspending_api/api_docs/api_documentation/references/data_dictionary.md @@ -12,48 +12,93 @@ This route takes no parameters and returns a JSON structure of the Schema team's ``` { - "rows": [ - [ - "1862 Land Grant College", - "https://www.sam.gov", - "1862 Land Grant College", - "all_contracts_prime_awards_1.csv,\nall_contracts_prime_transactions_1.csv", - "1862_land_grant_college", - None, - None, - None, - None, - "Contracts", - "is1862landgrantcollege", - None, - ] - ], - "headers": [ - {"raw": "element", "display": "Element"}, - {"raw": "definition", "display": "Definition"}, - {"raw": "fpds_element", "display": "FPDS Data Dictionary Element"}, - {"raw": "award_file", "display": "Award File"}, - {"raw": "award_element", "display": "Award Element"}, - {"raw": "subaward_file", "display": "Subaward File"}, - {"raw": "subaward_element", "display": "Subaward Element"}, - {"raw": "account_file", "display": "Account File"}, - {"raw": "account_element", "display": "Account Element"}, - {"raw": "legacy_award_file", "display": "Award File"}, - {"raw": "legacy_award_element", "display": "Award Element"}, - {"raw": "legacy_subaward_element", "display": "Subaward Element"}, - ], - "metadata": { - "total_rows": 1, - "total_size": "10.80KB", - "total_columns": 12, - "download_location": None, - }, - "sections": [ - {"colspan": 3, "section": "Schema Data Label & Description"}, - {"colspan": 6, "section": "USA Spending Downloads"}, - {"colspan": 3, "section": "Legacy USA Spending"}, - ], - } + "rows":[ + [ + "1862 Land Grant College", + "https://www.sam.gov", + "1862 Land Grant College", + "all_contracts_prime_awards_1.csv,\nall_contracts_prime_transactions_1.csv", + "1862_land_grant_college", + "", + "", + "", + "", + "Contracts", + "is1862landgrantcollege", + "" + ] + ], + "headers":[ + { + "raw":"element", + "display":"Element" + }, + { + "raw":"definition", + "display":"Definition" + }, + { + "raw":"fpds_element", + "display":"FPDS Data Dictionary Element" + }, + { + "raw":"award_file", + "display":"Award File" + }, + { + "raw":"award_element", + "display":"Award Element" + }, + { + "raw":"subaward_file", + "display":"Subaward File" + }, + { + "raw":"subaward_element", + "display":"Subaward Element" + }, + { + "raw":"account_file", + "display":"Account File" + }, + { + "raw":"account_element", + "display":"Account Element" + }, + { + "raw":"legacy_award_file", + "display":"Award File" + }, + { + "raw":"legacy_award_element", + "display":"Award Element" + }, + { + "raw":"legacy_subaward_element", + "display":"Subaward Element" + } + ], + "metadata":{ + "total_rows":1, + "total_size":"10.80KB", + "total_columns":12, + "download_location":"" + }, + "sections":[ + { + "colspan":3, + "section":"Schema Data Label & Description" + }, + { + "colspan":6, + "section":"USA Spending Downloads" + }, + { + "colspan":3, + "section":"Legacy USA Spending" + } + ] +} ``` From d2b246e670c0cc28f2d6b54c6590603c0d8559fd Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 10:23:19 -0500 Subject: [PATCH 28/52] real fix for parent_award bug --- usaspending_api/awards/tests/test_awards_v2.py | 18 ------------------ usaspending_api/awards/v2/data_layer/orm.py | 11 ++++------- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 3f27442219..7ab16cf07c 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -196,21 +196,8 @@ def awards_and_transactions(db): "subaward_count": 10, } - award_3_model = { - "pk": 3, - "type": "IDV_A", - "type_description": "AN IDV", - "category": "idv", - } mommy.make("awards.Award", **award_1_model) mommy.make("awards.Award", **award_2_model) - mommy.make("awards.Award", **award_3_model) - - parent_award_3_model = { - "award_id": award_3_model["pk"], - "parent_award_id": None - } - mommy.make("awards.ParentAward", **parent_award_3_model) @pytest.mark.django_db @@ -247,11 +234,6 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): assert resp.status_code == status.HTTP_200_OK assert json.loads(resp.content.decode("utf-8")) == expected_response_cont - # Bug fix rolled into DEV-2034. This used to throw an exception because - # parent award didn't have a parent. - resp = client.get("/api/v2/awards/3/") - assert resp.status_code == status.HTTP_200_OK - @pytest.mark.django_db def test_idv_award_amount_endpoint(client): diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index f4a327badd..f3044bc08e 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -193,7 +193,7 @@ def fetch_award_details(filter_q, mapper_fields): def fetch_parent_award_details(guai): parent_award_ids = ( ParentAward.objects.filter(generated_unique_award_id=guai) - .values("parent_award__award_id", "parent_award__generated_unique_award_id") + .values("award_id", "generated_unique_award_id") .first() ) @@ -201,7 +201,7 @@ def fetch_parent_award_details(guai): return None parent_award = ( - Award.objects.filter(id=parent_award_ids["parent_award__award_id"]) + Award.objects.filter(id=parent_award_ids["award_id"]) .values( "latest_transaction__contract_data__agency_id", "latest_transaction__contract_data__idv_type_description", @@ -212,14 +212,11 @@ def fetch_parent_award_details(guai): .first() ) - if not parent_award: - return None - parent_object = OrderedDict( [ ("agency_id", parent_award["latest_transaction__contract_data__agency_id"]), - ("award_id", parent_award_ids["parent_award__award_id"]), - ("generated_unique_award_id", parent_award_ids["parent_award__generated_unique_award_id"]), + ("award_id", parent_award_ids["award_id"]), + ("generated_unique_award_id", parent_award_ids["generated_unique_award_id"]), ("idv_type_description", parent_award["latest_transaction__contract_data__idv_type_description"]), ( "multiple_or_single_aw_desc", From aa9e80e7654260670284135f84e2f32803dbcddb Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 11:34:18 -0500 Subject: [PATCH 29/52] added last_modified_date and potential_end_date --- .../awards/tests/test_awards_v2.py | 6 ++++ usaspending_api/awards/v2/data_layer/orm.py | 28 ++++++++++++------- .../awards/v2/data_layer/orm_mappers.py | 3 ++ usaspending_api/common/helpers/date_helper.py | 27 ++++++++++++++++++ 4 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 usaspending_api/common/helpers/date_helper.py diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 7ab16cf07c..26b9b09ca3 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -85,6 +85,7 @@ def awards_and_transactions(db): "place_of_perform_zip_last4": "2135", "place_of_performance_zip5": "40221", "place_of_performance_forei": None, + "modified_at": "2000-01-02T00:00:00Z" } cont_data = { "awardee_or_recipient_legal": "John's Pizza", @@ -106,6 +107,7 @@ def awards_and_transactions(db): "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", "interagency_contract_desc": "NOT APPLICABLE", "labor_standards_descrip": "NO", + "last_modified": "2001-02-03", "legal_entity_address_line1": "123 main st", "legal_entity_address_line2": None, "legal_entity_address_line3": None, @@ -134,6 +136,7 @@ def awards_and_transactions(db): "place_of_performance_state": "NC", "place_of_perform_zip_last4": "5312", "place_of_performance_zip5": "12204", + "period_of_perf_potential_e": "2003-04-05", "price_evaluation_adjustmen": None, "product_or_service_co_desc": None, "product_or_service_code": "4730", @@ -344,6 +347,7 @@ def test_idv_award_amount_endpoint(client): "period_of_performance": { "period_of_performance_current_end_date": "2005-02-04", "period_of_performance_start_date": "2004-02-04", + "last_modified_date": "2000-01-02", }, "place_of_performance": { "address_line1": None, @@ -412,6 +416,8 @@ def test_idv_award_amount_endpoint(client): "period_of_performance": { "period_of_performance_start_date": "2004-02-04", "period_of_performance_current_end_date": "2005-02-04", + "last_modified_date": "2001-02-03", + "potential_end_date": "2003-04-05", }, "place_of_performance": { "address_line1": None, diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index f3044bc08e..a385e511ab 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -9,6 +9,7 @@ FABS_ASSISTANCE_FIELDS, ) from usaspending_api.awards.models import Award, TransactionFABS, TransactionFPDS, ParentAward +from usaspending_api.common.helpers.date_helper import get_date_from_datetime from usaspending_api.recipient.models import RecipientLookup from usaspending_api.references.models import Agency, LegalEntity, LegalEntityOfficers, Cfda from usaspending_api.awards.v2.data_layer.orm_utils import delete_keys_from_dict, split_mapper_into_qs @@ -41,6 +42,7 @@ def construct_assistance_response(requested_award_dict): [ ("period_of_performance_start_date", award["_start_date"]), ("period_of_performance_current_end_date", award["_end_date"]), + ("last_modified_date", get_date_from_datetime(transaction["_modified_at"])), ] ) transaction["_lei"] = award["_lei"] @@ -64,19 +66,23 @@ def construct_contract_response(requested_award_dict): return None response.update(award) + transaction = fetch_fpds_details_by_pk(award["_trx"], FPDS_CONTRACT_FIELDS) + response["executive_details"] = fetch_officers_by_legal_entity_id(award["_lei"]) - response["latest_transaction_contract_data"] = fetch_fpds_details_by_pk(award["_trx"], FPDS_CONTRACT_FIELDS) + response["latest_transaction_contract_data"] = transaction response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) response["period_of_performance"] = OrderedDict( [ ("period_of_performance_start_date", award["_start_date"]), ("period_of_performance_current_end_date", award["_end_date"]), + ("last_modified_date", transaction["_last_modified"]), + ("potential_end_date", transaction["_period_of_perf_potential_e"]), ] ) - response["latest_transaction_contract_data"]["_lei"] = award["_lei"] - response["recipient"] = create_recipient_object(response["latest_transaction_contract_data"]) - response["place_of_performance"] = create_place_of_performance_object(response["latest_transaction_contract_data"]) + transaction["_lei"] = award["_lei"] + response["recipient"] = create_recipient_object(transaction) + response["place_of_performance"] = create_place_of_performance_object(transaction) return delete_keys_from_dict(response) @@ -107,22 +113,24 @@ def construct_idv_response(requested_award_dict): response.update(award) parent_award = fetch_parent_award_details(award["generated_unique_award_id"]) + transaction = fetch_fpds_details_by_pk(award["_trx"], mapper) + response["parent_award"] = parent_award response["parent_generated_unique_award_id"] = parent_award["generated_unique_award_id"] if parent_award else None response["executive_details"] = fetch_officers_by_legal_entity_id(award["_lei"]) - response["latest_transaction_contract_data"] = fetch_fpds_details_by_pk(award["_trx"], mapper) + response["latest_transaction_contract_data"] = transaction response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) response["idv_dates"] = OrderedDict( [ ("start_date", award["_start_date"]), - ("last_modified_date", response["latest_transaction_contract_data"]["_last_modified_date"]), - ("end_date", response["latest_transaction_contract_data"]["_end_date"]), + ("last_modified_date", transaction["_last_modified_date"]), + ("end_date", transaction["_end_date"]), ] ) - response["latest_transaction_contract_data"]["_lei"] = award["_lei"] - response["recipient"] = create_recipient_object(response["latest_transaction_contract_data"]) - response["place_of_performance"] = create_place_of_performance_object(response["latest_transaction_contract_data"]) + transaction["_lei"] = award["_lei"] + response["recipient"] = create_recipient_object(transaction) + response["place_of_performance"] = create_place_of_performance_object(transaction) return delete_keys_from_dict(response) diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index 8aaada721b..fab4fe0575 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -68,6 +68,7 @@ [ ("cfda_number", "cfda_number"), ("cfda_title", "cfda_title"), + ("modified_at", "_modified_at"), # "Recipient" fields below ("awardee_or_recipient_legal", "_recipient_name"), ("awardee_or_recipient_uniqu", "_recipient_unique_id"), @@ -142,6 +143,8 @@ ("purchase_card_as_paym_desc", "purchase_card_as_paym_desc"), ("consolidated_contract_desc", "consolidated_contract_desc"), ("type_of_contract_pric_desc", "type_of_contract_pric_desc"), + ("last_modified", "_last_modified"), + ("period_of_perf_potential_e", "_period_of_perf_potential_e"), # "Recipient" fields below ("awardee_or_recipient_legal", "_recipient_name"), ("awardee_or_recipient_uniqu", "_recipient_unique_id"), diff --git a/usaspending_api/common/helpers/date_helper.py b/usaspending_api/common/helpers/date_helper.py new file mode 100644 index 0000000000..1f6a790ff0 --- /dev/null +++ b/usaspending_api/common/helpers/date_helper.py @@ -0,0 +1,27 @@ +def get_date_from_datetime(date_time, **kwargs): + """ + Pass a keyword argument called "default" if you wish to have a specific + value returned when the date cannot be extracted from date_time, otherwise + date_time will be returned. + """ + try: + return date_time.date() + except: + return kwargs.get('default', date_time) + + +def test_get_date_from_datetime(): + from datetime import date, datetime + + assert get_date_from_datetime(1) == 1 + assert get_date_from_datetime(1, default=2) == 2 + assert get_date_from_datetime(1, default=None) is None + + assert get_date_from_datetime('a') == 'a' + assert get_date_from_datetime('a', default='b') == 'b' + + assert get_date_from_datetime(datetime(2000, 1, 2)) == date(2000, 1, 2) + assert get_date_from_datetime(datetime(2000, 1, 2), default='no') == date(2000, 1, 2) + + assert get_date_from_datetime(datetime(2000, 1, 2, 3, 4, 5)) == date(2000, 1, 2) + assert get_date_from_datetime(datetime(2000, 1, 2, 3, 4, 5), default='maybe') == date(2000, 1, 2) From cc3c6555e27f5552cb84c6236cebcbacd59aad91 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 11:36:10 -0500 Subject: [PATCH 30/52] travis issue --- usaspending_api/common/helpers/date_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/common/helpers/date_helper.py b/usaspending_api/common/helpers/date_helper.py index 1f6a790ff0..24bf7fa16d 100644 --- a/usaspending_api/common/helpers/date_helper.py +++ b/usaspending_api/common/helpers/date_helper.py @@ -6,7 +6,7 @@ def get_date_from_datetime(date_time, **kwargs): """ try: return date_time.date() - except: + except Exception: return kwargs.get('default', date_time) From a3690096491798073e8651d830d4e7f6d522249e Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 14:01:49 -0500 Subject: [PATCH 31/52] flake8 --- usaspending_api/awards/v2/urls_idv_awards.py | 3 ++- usaspending_api/awards/v2/views/idvs/__init__.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/usaspending_api/awards/v2/urls_idv_awards.py b/usaspending_api/awards/v2/urls_idv_awards.py index ae6e3bdc88..efdb1a510e 100644 --- a/usaspending_api/awards/v2/urls_idv_awards.py +++ b/usaspending_api/awards/v2/urls_idv_awards.py @@ -1,5 +1,6 @@ from django.conf.urls import url -from usaspending_api.awards.v2.views.idvs import IDVAmountsViewSet, IDVAwardsViewSet +from usaspending_api.awards.v2.views.idvs.amounts import IDVAmountsViewSet +from usaspending_api.awards.v2.views.idvs.awards import IDVAwardsViewSet urlpatterns = [ url(r'^amounts/(?P[A-Za-z0-9_. -]+)/$', IDVAmountsViewSet.as_view()), diff --git a/usaspending_api/awards/v2/views/idvs/__init__.py b/usaspending_api/awards/v2/views/idvs/__init__.py index 19feeae88e..e69de29bb2 100644 --- a/usaspending_api/awards/v2/views/idvs/__init__.py +++ b/usaspending_api/awards/v2/views/idvs/__init__.py @@ -1,2 +0,0 @@ -from .amounts import IDVAmountsViewSet -from .awards import IDVAwardsViewSet From 6846932328e093fee8d50ee75f9207975429b86b Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 14:12:02 -0500 Subject: [PATCH 32/52] codeclimate --- usaspending_api/core/validator/tinyshield.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py index 039a3d7a5d..6bf743b85b 100644 --- a/usaspending_api/core/validator/tinyshield.py +++ b/usaspending_api/core/validator/tinyshield.py @@ -264,11 +264,12 @@ def enforce_rules(self): self.recurse_append(struct, self.data, self.apply_rule(item)) def apply_rule(self, rule): + _return = None if rule.get('allow_nulls', False) and rule['value'] is None: - return rule['value'] + _return = rule['value'] elif rule['type'] not in ('array', 'object', 'any'): if rule['type'] in VALIDATORS: - return VALIDATORS[rule['type']]['func'](rule) + _return = VALIDATORS[rule['type']]['func'](rule) else: raise Exception('Invalid Type {} in rule'.format(rule['type'])) # Array is a "special" type since it is a list of other types which need to be validated @@ -285,7 +286,7 @@ def apply_rule(self, rule): for v in value: child_rule['value'] = v array_result.append(self.apply_rule(child_rule)) - return array_result + _return = array_result # Object is a "special" type since it is comprised of other types which need to be validated elif rule['type'] == 'object': rule['object_min'] = rule.get('object_min', 1) @@ -306,7 +307,7 @@ def apply_rule(self, rule): child_rule['value'] = value child_rule = self.promote_subrules(child_rule, v) object_result[k] = self.apply_rule(child_rule) - return object_result + _return = object_result # Any is a "special" type since it is is really a collection of other rules. elif rule['type'] == 'any': for child_rule in rule['models']: @@ -322,6 +323,7 @@ def apply_rule(self, rule): value=rule['value'], type=', '.join(sorted([m['type'] for m in rule['models']])) )) + return _return def promote_subrules(self, child_rule, source={}): param_type = child_rule['type'] From e39d1694d12a8316061d35adc08faafe3d342deb Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 14:35:33 -0500 Subject: [PATCH 33/52] reversing this mornings 'fix' --- usaspending_api/awards/v2/data_layer/orm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index a385e511ab..f9660c26ff 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -201,7 +201,7 @@ def fetch_award_details(filter_q, mapper_fields): def fetch_parent_award_details(guai): parent_award_ids = ( ParentAward.objects.filter(generated_unique_award_id=guai) - .values("award_id", "generated_unique_award_id") + .values("parent_award__award_id", "parent_award__generated_unique_award_id") .first() ) @@ -209,7 +209,7 @@ def fetch_parent_award_details(guai): return None parent_award = ( - Award.objects.filter(id=parent_award_ids["award_id"]) + Award.objects.filter(id=parent_award_ids["parent_award__award_id"]) .values( "latest_transaction__contract_data__agency_id", "latest_transaction__contract_data__idv_type_description", @@ -223,8 +223,8 @@ def fetch_parent_award_details(guai): parent_object = OrderedDict( [ ("agency_id", parent_award["latest_transaction__contract_data__agency_id"]), - ("award_id", parent_award_ids["award_id"]), - ("generated_unique_award_id", parent_award_ids["generated_unique_award_id"]), + ("award_id", parent_award_ids["parent_award__award_id"]), + ("generated_unique_award_id", parent_award_ids["parent_award__generated_unique_award_id"]), ("idv_type_description", parent_award["latest_transaction__contract_data__idv_type_description"]), ( "multiple_or_single_aw_desc", From 59d0a363b29ab2448932fa4a9faab033cd60e948 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 14:37:17 -0500 Subject: [PATCH 34/52] reversing this mornings 'fix' part 2 --- usaspending_api/awards/v2/data_layer/orm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index f9660c26ff..4d29614f52 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -220,6 +220,9 @@ def fetch_parent_award_details(guai): .first() ) + if not parent_award: + return None + parent_object = OrderedDict( [ ("agency_id", parent_award["latest_transaction__contract_data__agency_id"]), From 410f09ec86177ef1b9a8c964f9613478bc5fd812 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Tue, 5 Feb 2019 14:51:13 -0500 Subject: [PATCH 35/52] yet another fix for missing parent awards --- usaspending_api/awards/v2/data_layer/orm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 4d29614f52..c2aab1f18c 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -1,4 +1,6 @@ import copy +import logging + from collections import OrderedDict from usaspending_api.awards.v2.data_layer.orm_mappers import ( @@ -15,6 +17,9 @@ from usaspending_api.awards.v2.data_layer.orm_utils import delete_keys_from_dict, split_mapper_into_qs +logger = logging.getLogger("console") + + def construct_assistance_response(requested_award_dict): """ Build the Python object to return FABS Award summary or meta-data via the API @@ -200,7 +205,7 @@ def fetch_award_details(filter_q, mapper_fields): def fetch_parent_award_details(guai): parent_award_ids = ( - ParentAward.objects.filter(generated_unique_award_id=guai) + ParentAward.objects.filter(generated_unique_award_id=guai, parent_award__isnull=False) .values("parent_award__award_id", "parent_award__generated_unique_award_id") .first() ) @@ -221,6 +226,7 @@ def fetch_parent_award_details(guai): ) if not parent_award: + logging.debug("Unable to find award for award id %s" % parent_award_ids["parent_award__award_id"]) return None parent_object = OrderedDict( From 4b58751c215daa9da0b38aa192c7ea13a240c4e5 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Wed, 6 Feb 2019 13:38:00 -0500 Subject: [PATCH 36/52] requested changes and a little more refactoring - SORRY! --- usaspending_api/awards/award_id_helper.py | 81 -------- .../awards/tests/test_award_id_helper.py | 44 ----- .../awards/v2/views/idvs/amounts.py | 33 ++-- .../awards/v2/views/idvs/awards.py | 42 ++--- .../awards/v2/views/transactions.py | 46 ++--- usaspending_api/core/validator/award.py | 18 ++ usaspending_api/core/validator/helpers.py | 11 +- usaspending_api/core/validator/pagination.py | 34 ++++ .../core/validator/tests/unit/test_awards.py | 176 ++++++++++++++++++ .../validator/tests/unit/test_pagination.py | 84 +++++++++ usaspending_api/core/validator/tinyshield.py | 12 -- usaspending_api/core/validator/utils.py | 11 ++ 12 files changed, 380 insertions(+), 212 deletions(-) delete mode 100644 usaspending_api/awards/award_id_helper.py delete mode 100644 usaspending_api/awards/tests/test_award_id_helper.py create mode 100644 usaspending_api/core/validator/award.py create mode 100644 usaspending_api/core/validator/tests/unit/test_awards.py create mode 100644 usaspending_api/core/validator/tests/unit/test_pagination.py diff --git a/usaspending_api/awards/award_id_helper.py b/usaspending_api/awards/award_id_helper.py deleted file mode 100644 index be89b76068..0000000000 --- a/usaspending_api/awards/award_id_helper.py +++ /dev/null @@ -1,81 +0,0 @@ -from enum import Enum - - -# Award id is currently defined as bigint in the database. These are minimum -# and maximum values for bigint. While using sys.maxsize may look appealing, -# it is based on the word size of the machine which will cause problems on -# anything other than a 64 bit machine thus making it incorrect for this purpose. -MIN_INT = -2**63 # -9223372036854775808 -MAX_INT = 2**63 - 1 # +9223372036854775807 - - -class AwardIdType(Enum): - """ - Multi-state return value for detect_award_id_type. - """ - internal = 1 # award_id is probably an internal (integer) award id - generated = 2 # award_id is probably a generated_unique_award_id - unknown = 3 # type is unknown - - -def detect_award_id_type(award_id): - """ - THE PROBLEM: We have several places in code where we need to determine - whether a provided value is an internal award id (integer), a generated - award id (string), or neither. - - internal award id: 123456 - also an internal award id: "123456" - generated unique award id: "CONT_AW_4732_-NONE-_47QRAA18D0081_-NONE-" - - not an award id: 123.456 - also not an award id: {"award_id": "123456"} - - At present, this is being performed a few different ways, each of which has - its own set of problems. Simply attempting to cast using int will truncate - fractional bits for floats which is bad. Checking isdigit or isdecimal - will allow values to slip through that the database will not understand, - for example U+0660 (Arabic-Indic Digit Zero). - - THE SOLUTION: Here we will provide what we hope to be THE single solution - for this problem in a nice, little, encapsulated function using a - combination of type interrogation, hard casting, and exception handling. - - Input: - award_id - Can be anything, but hopefully it will be something capable - of being a valid award id. Only strings and integers have any hope of - being award ids at this time. - - Returns: - recast_award_id, AwardIdType - If award_id is an internal award id, an - integer will be returned along with AwardIdType.internal. If award_id - is a generated award id, a string will be returned along with - AwardIdType.generated. Otherwise, the original award_id and - AwardIdType.unknown will be returned. - - Examples: - detect_award_id_type(123456) => 123456, AwardIdType.internal - detect_award_id_type("123456") => 123456, AwardIdType.internal - detect_award_id_type("CONT_AW_-NONE-") => "CONT_AW_-NONE-", AwardIdType.generated - detect_award_id_type(123.456) => 123.456, AwardIdType.unknown - - IMPORTANT THING 1: I highly recommend you use the return value in your - queries instead of the original award_id provided to this function since - it is being cast to an appropriate data type for you. This eliminates odd - situations where the award_id is successfully cast to an integer, yet the - original value is something that the database does not understand (for - example, there are certain unicode characters that are numbers but the - database will not see them as such). - - IMPORTANT THING A: We will NOT be querying the database here. We are - strictly attempting to discern the type of the id based on its value. - """ - if type(award_id) in (int, str): - try: - _award_id = int(award_id) - if MIN_INT <= _award_id <= MAX_INT: - return _award_id, AwardIdType.internal - except ValueError: - pass - return str(award_id).upper(), AwardIdType.generated - return award_id, AwardIdType.unknown # ¯\_(ツ)_/¯ diff --git a/usaspending_api/awards/tests/test_award_id_helper.py b/usaspending_api/awards/tests/test_award_id_helper.py deleted file mode 100644 index 8e9f0cb198..0000000000 --- a/usaspending_api/awards/tests/test_award_id_helper.py +++ /dev/null @@ -1,44 +0,0 @@ -from usaspending_api.awards.award_id_helper import AwardIdType, detect_award_id_type, MAX_INT - - -def test_acceptable_internal_award_ids(): - - assert detect_award_id_type(12345) == (12345, AwardIdType.internal) - assert detect_award_id_type(-12345) == (-12345, AwardIdType.internal) - assert detect_award_id_type(+12345) == (12345, AwardIdType.internal) - assert detect_award_id_type('12345') == (12345, AwardIdType.internal) - assert detect_award_id_type('-12345') == (-12345, AwardIdType.internal) - assert detect_award_id_type('+12345') == (12345, AwardIdType.internal) - - assert detect_award_id_type(0) == (0, AwardIdType.internal) - assert detect_award_id_type(-0) == (0, AwardIdType.internal) - assert detect_award_id_type(+0) == (0, AwardIdType.internal) - assert detect_award_id_type('0') == (0, AwardIdType.internal) - assert detect_award_id_type('-0') == (0, AwardIdType.internal) - assert detect_award_id_type('+0') == (0, AwardIdType.internal) - - # A unicode Arabic-Indic zero. - assert detect_award_id_type('\u0660') == (0, AwardIdType.internal) - - -def test_acceptable_generated_award_ids(): - - assert detect_award_id_type('1a') == ('1A', AwardIdType.generated) - assert detect_award_id_type('1.1') == ('1.1', AwardIdType.generated) - assert detect_award_id_type('CONT_AW_4732_-NONE-_47QRAA18D0081_-NONE-') == \ - ('CONT_AW_4732_-NONE-_47QRAA18D0081_-NONE-', AwardIdType.generated) - assert detect_award_id_type('') == ('', AwardIdType.generated) # Unfortunately - assert detect_award_id_type(' ') == (' ', AwardIdType.generated) # Also unfortunately - - # An integer that's too big for the database will be cast to a generated - # type. Is this right? Who's to say. - assert detect_award_id_type(MAX_INT + 1) == (str(MAX_INT + 1), AwardIdType.generated) - - -def test_invalid_award_ids(): - - assert detect_award_id_type(None) == (None, AwardIdType.unknown) - assert detect_award_id_type(1.0) == (1.0, AwardIdType.unknown) - assert detect_award_id_type([1, 2, 3]) == ([1, 2, 3], AwardIdType.unknown) - assert detect_award_id_type(('1', '2', '3')) == (('1', '2', '3'), AwardIdType.unknown) - assert detect_award_id_type({'a': 1, 'b': 2}) == ({'a': 1, 'b': 2}, AwardIdType.unknown) diff --git a/usaspending_api/awards/v2/views/idvs/amounts.py b/usaspending_api/awards/v2/views/idvs/amounts.py index baf488ee35..3929d0848e 100644 --- a/usaspending_api/awards/v2/views/idvs/amounts.py +++ b/usaspending_api/awards/v2/views/idvs/amounts.py @@ -6,16 +6,19 @@ from rest_framework.request import Request from rest_framework.response import Response -from usaspending_api.awards.award_id_helper import detect_award_id_type, AwardIdType from usaspending_api.awards.models import ParentAward from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.views import APIDocumentationView +from usaspending_api.core.validator.award import get_internal_or_generated_award_id_rule from usaspending_api.core.validator.tinyshield import TinyShield logger = logging.getLogger('console') +TINY_SHIELD_RULES = TinyShield([get_internal_or_generated_award_id_rule()]) + + class IDVAmountsViewSet(APIDocumentationView): """Returns counts and dollar figures for a specific IDV. endpoint_doc: /awards/idvs/amounts.md @@ -23,30 +26,18 @@ class IDVAmountsViewSet(APIDocumentationView): @staticmethod def _parse_and_validate_request(requested_award: str) -> dict: - award_id, award_id_type = detect_award_id_type(requested_award) - if award_id_type is AwardIdType.internal: - request_dict = {'award_id': award_id} - models = [{ - 'key': 'award_id', - 'name': 'award_id', - 'type': 'integer', - 'optional': False, - }] - else: - request_dict = {'generated_unique_award_id': award_id} - models = [{ - 'key': 'generated_unique_award_id', - 'name': 'generated_unique_award_id', - 'type': 'text', - 'text_type': 'search', - 'optional': False, - }] - return TinyShield(models).block(request_dict) + return TINY_SHIELD_RULES.block({'award_id': requested_award}) @staticmethod def _business_logic(request_data: dict) -> OrderedDict: + # By this point, our award_id has been validated and cleaned up by + # TinyShield. We will either have an internal award id that is an + # integer or a generated award id that is a string. + award_id = request_data['award_id'] + award_id_column = 'award_id' if type(award_id) is int else 'generated_unique_award_id' + try: - parent_award = ParentAward.objects.get(**request_data) + parent_award = ParentAward.objects.get(**{award_id_column: award_id}) return OrderedDict(( ('award_id', parent_award.award_id), ('generated_unique_award_id', parent_award.generated_unique_award_id), diff --git a/usaspending_api/awards/v2/views/idvs/awards.py b/usaspending_api/awards/v2/views/idvs/awards.py index 1d15b32ac9..60922a46bf 100644 --- a/usaspending_api/awards/v2/views/idvs/awards.py +++ b/usaspending_api/awards/v2/views/idvs/awards.py @@ -1,16 +1,15 @@ from collections import OrderedDict -from copy import deepcopy from django.db import connection from psycopg2.sql import Identifier, Literal, SQL from rest_framework.request import Request from rest_framework.response import Response -from usaspending_api.awards.award_id_helper import detect_award_id_type, AwardIdType from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata from usaspending_api.common.views import APIDocumentationView -from usaspending_api.core.validator.pagination import PAGINATION +from usaspending_api.core.validator.award import get_internal_or_generated_award_id_rule +from usaspending_api.core.validator.pagination import customize_pagination_with_sort_columns from usaspending_api.core.validator.tinyshield import TinyShield from usaspending_api.etl.broker_etl_helpers import dictfetchall @@ -84,28 +83,12 @@ def _prepare_tiny_shield_rules(): Our TinyShield rules never change. Encapsulate them here and store them once in TINY_SHIELD_RULES. """ - - # This endpoint supports paging. - models = deepcopy(PAGINATION) - - # Add the list of sortable columns for validation. - sort_rule = TinyShield.get_model_by_name(models, 'sort') - sort_rule['type'] = 'enum' - sort_rule['enum_values'] = SORTABLE_COLUMNS - sort_rule['default'] = DEFAULT_SORT_COLUMN - - # Add additional models for the award id and the idv filter. + models = customize_pagination_with_sort_columns(SORTABLE_COLUMNS, DEFAULT_SORT_COLUMN) models.extend([ - # Award id can be either an integer or a string depending upon whether - # it's an internal id or a unique generated id. - {'key': 'award_id', 'name': 'award_id', 'type': 'any', 'optional': False, 'models': [ - {'type': 'integer'}, - {'type': 'text', 'text_type': 'search'} - ]}, + get_internal_or_generated_award_id_rule(), {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True} ]) - - return models + return TinyShield(models) TINY_SHIELD_RULES = _prepare_tiny_shield_rules() @@ -118,12 +101,16 @@ class IDVAwardsViewSet(APIDocumentationView): @staticmethod def _parse_and_validate_request(request: Request) -> dict: - return TinyShield(deepcopy(TINY_SHIELD_RULES)).block(request) + return TINY_SHIELD_RULES.block(request) @staticmethod def _business_logic(request_data: dict) -> list: - award_id, award_id_type = detect_award_id_type(request_data['award_id']) - award_id_column = 'award_id' if award_id_type is AwardIdType.internal else 'generated_unique_award_id' + # By this point, our award_id has been validated and cleaned up by + # TinyShield. We will either have an internal award id that is an + # integer or a generated award id that is a string. + award_id = request_data['award_id'] + award_id_column = 'award_id' if type(award_id) is int else 'generated_unique_award_id' + sql = GET_IDVS_SQL if request_data['idv'] else GET_CONTRACTS_SQL sql = sql.format( award_id_column=Identifier(award_id_column), @@ -134,7 +121,10 @@ def _business_logic(request_data: dict) -> list: offset=Literal((request_data['page'] - 1) * request_data['limit']), ) with connection.cursor() as cursor: - cursor.execute(sql, [award_id]) + # We must convert this to an actual query string else + # django-debug-toolbar will blow up since it is assuming a string + # instead of a SQL object. + cursor.execute(sql.as_string(connection.connection), [award_id]) return dictfetchall(cursor) @cache_response() diff --git a/usaspending_api/awards/v2/views/transactions.py b/usaspending_api/awards/v2/views/transactions.py index 6a8be10472..1680939877 100644 --- a/usaspending_api/awards/v2/views/transactions.py +++ b/usaspending_api/awards/v2/views/transactions.py @@ -1,15 +1,13 @@ -from copy import deepcopy - from django.db.models import F from rest_framework.request import Request from rest_framework.response import Response -from usaspending_api.awards.award_id_helper import AwardIdType, detect_award_id_type from usaspending_api.awards.models import TransactionNormalized from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata from usaspending_api.common.views import APIDocumentationView -from usaspending_api.core.validator.pagination import PAGINATION +from usaspending_api.core.validator.award import get_internal_or_generated_award_id_rule +from usaspending_api.core.validator.pagination import customize_pagination_with_sort_columns from usaspending_api.core.validator.tinyshield import TinyShield @@ -35,31 +33,33 @@ class TransactionViewSet(APIDocumentationView): "is_fpds": "is_fpds", } - def _parse_and_validate_request(self, request_dict: dict) -> dict: - models = deepcopy(PAGINATION) - models.append({"key": "award_id", "name": "award_id", "type": "text", "text_type": "search", "optional": False}) - for model in models: - # Change sort to an enum of the desired values - if model["name"] == "sort": - model["type"] = "enum" - model["enum_values"] = list(self.transaction_lookup.keys()) - model["default"] = "action_date" + def __init__(self): + """ + Our TinyShield rules never change. Encapsulate them here and store + them once in TINY_SHIELD_RULES. + """ + models = customize_pagination_with_sort_columns(TransactionViewSet.transaction_lookup.keys(), 'action_date') + models.extend([ + get_internal_or_generated_award_id_rule(), + {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True} + ]) + self._tiny_shield_rules = TinyShield(models) + super(TransactionViewSet, self).__init__() - validated_request_data = TinyShield(models).block(request_dict) - return validated_request_data + def _parse_and_validate_request(self, request_dict: dict) -> dict: + return self._tiny_shield_rules.block(request_dict) def _business_logic(self, request_data: dict) -> list: + # By this point, our award_id has been validated and cleaned up by + # TinyShield. We will either have an internal award id that is an + # integer or a generated award id that is a string. + award_id = request_data['award_id'] + award_id_column = 'award_id' if type(award_id) is int else 'award__generated_unique_award_id' + filter = {award_id_column: award_id} + lower_limit = (request_data["page"] - 1) * request_data["limit"] upper_limit = request_data["page"] * request_data["limit"] - award_id, id_type = detect_award_id_type(request_data['award_id']) - if id_type is AwardIdType.internal: - # Award ID - filter = {'award_id': award_id} - else: - # Generated Award ID - filter = {'award__generated_unique_award_id': award_id} - queryset = (TransactionNormalized.objects.all() .values(*list(self.transaction_lookup.values())) .filter(**filter)) diff --git a/usaspending_api/core/validator/award.py b/usaspending_api/core/validator/award.py new file mode 100644 index 0000000000..f1ecb03ab6 --- /dev/null +++ b/usaspending_api/core/validator/award.py @@ -0,0 +1,18 @@ +""" +Some shortcuts for generating "standardized" basic award id TinyShield model strings. +""" + + +def get_generated_award_id_rule(key='award_id', name='award_id', optional=False): + return {'key': key, 'name': name, 'type': 'text', 'text_type': 'search', 'optional': optional} + + +def get_internal_award_id_rule(key='award_id', name='award_id', optional=False): + return {'key': key, 'name': name, 'type': 'integer', 'optional': optional} + + +def get_internal_or_generated_award_id_rule(key='award_id', name='award_id', optional=False): + return {'key': key, 'name': name, 'type': 'any', 'optional': optional, 'models': [ + {'type': 'integer'}, + {'type': 'text', 'text_type': 'search'} + ]} diff --git a/usaspending_api/core/validator/helpers.py b/usaspending_api/core/validator/helpers.py index 6a836a70f6..3499fd7dab 100644 --- a/usaspending_api/core/validator/helpers.py +++ b/usaspending_api/core/validator/helpers.py @@ -46,10 +46,11 @@ def _check_min(rule): def _verify_int_value(value): - try: - return int(value) - except Exception as e: - pass + if type(value) in (int, str): + try: + return int(value) + except Exception as e: + pass return None @@ -193,7 +194,7 @@ def validate_text(rule): if text_type in ('raw', 'sql', 'password'): # TODO: flesh out expectations and constraints for sql and password types if text_type in ('sql', 'password'): - logger.warn('Caution: text_type \'{}\' not yet fully implemented'.format(text_type)) + logger.warning('Caution: text_type \'{}\' not yet fully implemented'.format(text_type)) val = rule['value'] elif text_type == 'url': val = urllib.parse.quote_plus(rule['value']) diff --git a/usaspending_api/core/validator/pagination.py b/usaspending_api/core/validator/pagination.py index 7853afb0dc..595ff622c8 100644 --- a/usaspending_api/core/validator/pagination.py +++ b/usaspending_api/core/validator/pagination.py @@ -1,3 +1,8 @@ +from copy import deepcopy + +from .utils import get_model_by_name + + PAGINATION = [ {'name': 'page', 'type': 'integer', 'default': 1, 'min': 1}, {'name': 'limit', 'type': 'integer', 'default': 10, 'min': 1, 'max': 100}, @@ -5,6 +10,35 @@ {'name': 'order', 'type': 'enum', 'enum_values': ('asc', 'desc'), 'default': 'desc'}, ] + for p in PAGINATION: p['optional'] = p.get('optional', True) p['key'] = p['name'] + + +def customize_pagination_with_sort_columns(sortable_columns, default_sort_column=None): + """ + A common customization to TinyShield pagination rules is to enumerate the + actual sort columns for validation. + + sortable_columns - An iterable (list, tuple, set, whatever) of allowable + sort field names (strings). + default_sort_column - The sort field name to use if one isn't supplied in + the input. + """ + for s in sortable_columns: + if type(s) is not str: + raise TypeError('sortable_columns must be string field names') + + models = deepcopy(PAGINATION) + + # Add the list of sortable columns for validation. + sort_rule = get_model_by_name(models, 'sort') + sort_rule['type'] = 'enum' + sort_rule['enum_values'] = sortable_columns + if default_sort_column is not None: + if default_sort_column not in sortable_columns: + raise ValueError('default_sort_column "%s" not found in list of sortable_columns' % default_sort_column) + sort_rule['default'] = default_sort_column + + return models diff --git a/usaspending_api/core/validator/tests/unit/test_awards.py b/usaspending_api/core/validator/tests/unit/test_awards.py new file mode 100644 index 0000000000..d472db30f3 --- /dev/null +++ b/usaspending_api/core/validator/tests/unit/test_awards.py @@ -0,0 +1,176 @@ +""" +Some sanity checks for the pre-built TinyShield Awards rules. +""" +import pytest + +from usaspending_api.common.exceptions import InvalidParameterException, UnprocessableEntityException +from usaspending_api.core.validator.tinyshield import TinyShield +from usaspending_api.core.validator.award import get_generated_award_id_rule, get_internal_award_id_rule, \ + get_internal_or_generated_award_id_rule +from usaspending_api.core.validator.helpers import MAX_INT, MAX_ITEMS, MIN_INT + + +def test_get_generate_award_id_rule(): + models = [get_generated_award_id_rule()] + r = TinyShield(models).block({'award_id': 'abcd'}) + assert r == {'award_id': 'abcd'} + + models = [get_generated_award_id_rule()] + r = TinyShield(models).block({'award_id': 'A' * MAX_ITEMS}) + assert r == {'award_id': 'A' * MAX_ITEMS} + + models = [get_generated_award_id_rule(key='my_award_id')] + r = TinyShield(models).block({'my_award_id': 'abcd'}) + assert r == {'my_award_id': 'abcd'} + + models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id')] + r = TinyShield(models).block({'my_award_id': 'abcd'}) + assert r == {'my_award_id': 'abcd'} + + models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({'my_award_id': 'abcd'}) + assert r == {'my_award_id': 'abcd'} + + models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({}) + assert r == {} + + # Bad rules. + ts = TinyShield([get_generated_award_id_rule()]) + + with pytest.raises(UnprocessableEntityException): + ts.block({}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': 'A' * (MAX_ITEMS + 1)}) + with pytest.raises(InvalidParameterException): + ts.block({'award_id': 1.1}) + with pytest.raises(InvalidParameterException): + ts.block({'award_id': [1, 2]}) + with pytest.raises(UnprocessableEntityException): + ts.block({'id': 'abcd'}) + + +def test_get_internal_award_id_rule(): + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': 12345}) + assert r == {'award_id': 12345} + + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': '12345'}) + assert r == {'award_id': 12345} + + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': MAX_INT}) + assert r == {'award_id': MAX_INT} + + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': MIN_INT}) + assert r == {'award_id': MIN_INT} + + models = [get_internal_award_id_rule(key='my_award_id')] + r = TinyShield(models).block({'my_award_id': 12345}) + assert r == {'my_award_id': 12345} + + models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id')] + r = TinyShield(models).block({'my_award_id': 12345}) + assert r == {'my_award_id': 12345} + + models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({'my_award_id': 12345}) + assert r == {'my_award_id': 12345} + + models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({}) + assert r == {} + + # Bad rules. + ts = TinyShield([get_internal_award_id_rule()]) + + with pytest.raises(UnprocessableEntityException): + ts.block({}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': MAX_INT + 1}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': MIN_INT - 1}) + with pytest.raises(InvalidParameterException): + ts.block({'award_id': 1.1}) + with pytest.raises(InvalidParameterException): + ts.block({'award_id': [1, 2]}) + with pytest.raises(UnprocessableEntityException): + ts.block({'id': 'abcde'}) + + +def test_get_internal_or_generated_award_id_rule(): + models = [get_generated_award_id_rule()] + r = TinyShield(models).block({'award_id': 'abcd'}) + assert r == {'award_id': 'abcd'} + + models = [get_generated_award_id_rule()] + r = TinyShield(models).block({'award_id': 'A' * MAX_ITEMS}) + assert r == {'award_id': 'A' * MAX_ITEMS} + + models = [get_generated_award_id_rule(key='my_award_id')] + r = TinyShield(models).block({'my_award_id': 'abcd'}) + assert r == {'my_award_id': 'abcd'} + + models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id')] + r = TinyShield(models).block({'my_award_id': 'abcd'}) + assert r == {'my_award_id': 'abcd'} + + models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({'my_award_id': 'abcd'}) + assert r == {'my_award_id': 'abcd'} + + models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({}) + assert r == {} + + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': 12345}) + assert r == {'award_id': 12345} + + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': '12345'}) + assert r == {'award_id': 12345} + + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': MAX_INT}) + assert r == {'award_id': MAX_INT} + + models = [get_internal_award_id_rule()] + r = TinyShield(models).block({'award_id': MIN_INT}) + assert r == {'award_id': MIN_INT} + + models = [get_internal_award_id_rule(key='my_award_id')] + r = TinyShield(models).block({'my_award_id': 12345}) + assert r == {'my_award_id': 12345} + + models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id')] + r = TinyShield(models).block({'my_award_id': 12345}) + assert r == {'my_award_id': 12345} + + models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({'my_award_id': 12345}) + assert r == {'my_award_id': 12345} + + models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + r = TinyShield(models).block({}) + assert r == {} + + # Bad rules. + ts = TinyShield([get_internal_or_generated_award_id_rule()]) + + with pytest.raises(UnprocessableEntityException): + ts.block({}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': 'A' * (MAX_ITEMS + 1)}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': MAX_INT + 1}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': MIN_INT - 1}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': 1.1}) + with pytest.raises(UnprocessableEntityException): + ts.block({'award_id': [1, 2]}) + with pytest.raises(UnprocessableEntityException): + ts.block({'id': 'abcde'}) diff --git a/usaspending_api/core/validator/tests/unit/test_pagination.py b/usaspending_api/core/validator/tests/unit/test_pagination.py new file mode 100644 index 0000000000..a11839738a --- /dev/null +++ b/usaspending_api/core/validator/tests/unit/test_pagination.py @@ -0,0 +1,84 @@ +""" +Some sanity checks for the pre-built TinyShield PAGINATION rules. +""" +import pytest + +from copy import deepcopy + +from usaspending_api.common.exceptions import InvalidParameterException, UnprocessableEntityException +from usaspending_api.core.validator.pagination import customize_pagination_with_sort_columns, PAGINATION +from usaspending_api.core.validator.tinyshield import TinyShield + + +def test_default_pagination(): + pagination = deepcopy(PAGINATION) + + r = TinyShield(pagination).block({}) + assert r == {'page': 1, 'limit': 10, 'order': 'desc'} + + +def test_non_default_overridden_pagination(): + pagination = deepcopy(PAGINATION) + + r = TinyShield(pagination).block({'page': 2, 'limit': 11, 'sort': 'whatever', 'order': 'asc'}) + assert r == {'page': 2, 'limit': 11, 'sort': 'whatever', 'order': 'asc'} + + +def test_invalid_pagination_values(): + pagination = deepcopy(PAGINATION) + + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'page': 'minus one'}) + with pytest.raises(UnprocessableEntityException): + TinyShield(pagination).block({'page': -1}) + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'page': ['test']}) + + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'limit': 'minus one'}) + with pytest.raises(UnprocessableEntityException): + TinyShield(pagination).block({'limit': -1}) + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'limit': ['test']}) + + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'sort': -1}) + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'sort': ['test']}) + + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'order': 'whatever'}) + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'order': -1}) + with pytest.raises(InvalidParameterException): + TinyShield(pagination).block({'order': ['test']}) + + +def test_customized_pagination(): + pagination = customize_pagination_with_sort_columns(['field1', 'fieldA', 'fieldOne']) + + r = TinyShield(pagination).block({}) + assert r == {'page': 1, 'limit': 10, 'order': 'desc'} + + +def test_customized_pagination_with_a_default_sort_column(): + pagination = customize_pagination_with_sort_columns(['field1', 'fieldA', 'fieldOne'], 'fieldOne') + + r = TinyShield(pagination).block({}) + assert r == {'page': 1, 'limit': 10, 'order': 'desc', 'sort': 'fieldOne'} + + +def test_invalid_customized_pagination_values(): + with pytest.raises(TypeError): + customize_pagination_with_sort_columns(None) + with pytest.raises(TypeError): + customize_pagination_with_sort_columns(1) + with pytest.raises(TypeError): + customize_pagination_with_sort_columns([1, 2, 3]) + with pytest.raises(ValueError): + customize_pagination_with_sort_columns(['field1', 'fieldA', 'fieldOne'], 1) + with pytest.raises(ValueError): + customize_pagination_with_sort_columns(['field1', 'fieldA', 'fieldOne'], 'fieldTwo') + + # This should work. Last second sanity check. + customize_pagination_with_sort_columns(['field1', 'fieldA', 'fieldOne'], 'fieldOne') diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py index 6bf743b85b..cae5839b74 100644 --- a/usaspending_api/core/validator/tinyshield.py +++ b/usaspending_api/core/validator/tinyshield.py @@ -358,15 +358,3 @@ def recurse_append(self, struct, mydict, data): else: mydict[level] = {} self.recurse_append(struct, mydict[level], data) - - @staticmethod - def get_model_by_name(models, name): - """ - Little helper function to return a TinyShield model from a list of - models given the model's name. Returns None if the model was not - found. - """ - for model in models: - if model.get('name') == name: - return model - return None diff --git a/usaspending_api/core/validator/utils.py b/usaspending_api/core/validator/utils.py index 1b62df9382..c05b337909 100644 --- a/usaspending_api/core/validator/utils.py +++ b/usaspending_api/core/validator/utils.py @@ -1,3 +1,14 @@ +def get_model_by_name(models, name): + """ + Little helper function to return a TinyShield model from a list of models + given the model's name. Returns None if the model was not found. + """ + for model in models: + if model.get('name') == name: + return model + return None + + def update_model_in_list(model_list: list, model_name: str, new_dict: dict, replace: bool = False) -> list: """ This is a generic helper utility function which can update or fully From 9f8ce9409e1b400353196cc6b388cd07752f7b1e Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Wed, 6 Feb 2019 13:53:35 -0500 Subject: [PATCH 37/52] codeclimate... and a bug --- .../unit/{test_awards.py => test_award.py} | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) rename usaspending_api/core/validator/tests/unit/{test_awards.py => test_award.py} (83%) diff --git a/usaspending_api/core/validator/tests/unit/test_awards.py b/usaspending_api/core/validator/tests/unit/test_award.py similarity index 83% rename from usaspending_api/core/validator/tests/unit/test_awards.py rename to usaspending_api/core/validator/tests/unit/test_award.py index d472db30f3..04d607ae4f 100644 --- a/usaspending_api/core/validator/tests/unit/test_awards.py +++ b/usaspending_api/core/validator/tests/unit/test_award.py @@ -101,69 +101,68 @@ def test_get_internal_award_id_rule(): def test_get_internal_or_generated_award_id_rule(): - models = [get_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_rule()] r = TinyShield(models).block({'award_id': 'abcd'}) assert r == {'award_id': 'abcd'} - models = [get_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_rule()] r = TinyShield(models).block({'award_id': 'A' * MAX_ITEMS}) assert r == {'award_id': 'A' * MAX_ITEMS} - models = [get_generated_award_id_rule(key='my_award_id')] + models = [get_internal_or_generated_award_id_rule(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id')] + models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] - r = TinyShield(models).block({}) - assert r == {} - - models = [get_internal_award_id_rule()] + models = [get_internal_or_generated_award_id_rule()] r = TinyShield(models).block({'award_id': 12345}) assert r == {'award_id': 12345} - models = [get_internal_award_id_rule()] + models = [get_internal_or_generated_award_id_rule()] r = TinyShield(models).block({'award_id': '12345'}) assert r == {'award_id': 12345} - models = [get_internal_award_id_rule()] + models = [get_internal_or_generated_award_id_rule()] r = TinyShield(models).block({'award_id': MAX_INT}) assert r == {'award_id': MAX_INT} - models = [get_internal_award_id_rule()] + models = [get_internal_or_generated_award_id_rule()] r = TinyShield(models).block({'award_id': MIN_INT}) assert r == {'award_id': MIN_INT} - models = [get_internal_award_id_rule(key='my_award_id')] + models = [get_internal_or_generated_award_id_rule(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id')] + models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({}) assert r == {} + +def test_get_internal_or_generated_award_id_rule_bad(): + # Bad rules. ts = TinyShield([get_internal_or_generated_award_id_rule()]) with pytest.raises(UnprocessableEntityException): ts.block({}) with pytest.raises(UnprocessableEntityException): - ts.block({'award_id': 'A' * (MAX_ITEMS + 1)}) + ts.block({'award_id': 'B' * (MAX_ITEMS + 1)}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': MAX_INT + 1}) with pytest.raises(UnprocessableEntityException): From 3894dfee5e3662dba1cc1ef732b70ed9c52333c3 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Wed, 6 Feb 2019 14:40:00 -0500 Subject: [PATCH 38/52] minor fix for documentation --- usaspending_api/awards/v2/views/awards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/awards/v2/views/awards.py b/usaspending_api/awards/v2/views/awards.py index bafc712c22..be973cc474 100644 --- a/usaspending_api/awards/v2/views/awards.py +++ b/usaspending_api/awards/v2/views/awards.py @@ -38,7 +38,7 @@ def get(self, request): class AwardRetrieveViewSet(APIDocumentationView): """ - endpoint_doc: /awards.md + endpoint_doc: /awards/awards.md """ def _parse_and_validate_request(self, provided_award_id: str) -> dict: From 65ad37cac91ebe7dbd73f49c7d8fb50c13162dc8 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Thu, 7 Feb 2019 07:40:12 -0500 Subject: [PATCH 39/52] requested changes --- usaspending_api/awards/v2/views/idvs/awards.py | 6 +++--- usaspending_api/core/validator/pagination.py | 2 +- .../core/validator/tests/unit/test_award.py | 6 +++--- usaspending_api/core/validator/tinyshield.py | 10 ++++------ 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/usaspending_api/awards/v2/views/idvs/awards.py b/usaspending_api/awards/v2/views/idvs/awards.py index 60922a46bf..3bf4147d31 100644 --- a/usaspending_api/awards/v2/views/idvs/awards.py +++ b/usaspending_api/awards/v2/views/idvs/awards.py @@ -46,7 +46,7 @@ inner join awards ac on ac.id = pac.award_id inner join transaction_fpds tf on tf.transaction_id = ac.latest_transaction_id where - pap.{award_id_column} = %s + pap.{award_id_column} = {award_id} order by {sort_column} {sort_direction}, ac.id {sort_direction} limit {limit} offset {offset} @@ -71,7 +71,7 @@ ac.type not like 'IDV\_%%' inner join transaction_fpds tf on tf.transaction_id = ac.latest_transaction_id where - pap.{award_id_column} = %s + pap.{award_id_column} = {award_id} order by {sort_column} {sort_direction}, ac.id {sort_direction} limit {limit} offset {offset} @@ -124,7 +124,7 @@ def _business_logic(request_data: dict) -> list: # We must convert this to an actual query string else # django-debug-toolbar will blow up since it is assuming a string # instead of a SQL object. - cursor.execute(sql.as_string(connection.connection), [award_id]) + cursor.execute(sql.as_string(connection.connection)) return dictfetchall(cursor) @cache_response() diff --git a/usaspending_api/core/validator/pagination.py b/usaspending_api/core/validator/pagination.py index 595ff622c8..4840e98c9c 100644 --- a/usaspending_api/core/validator/pagination.py +++ b/usaspending_api/core/validator/pagination.py @@ -28,7 +28,7 @@ def customize_pagination_with_sort_columns(sortable_columns, default_sort_column """ for s in sortable_columns: if type(s) is not str: - raise TypeError('sortable_columns must be string field names') + raise TypeError('sortable_columns must be an iterable of string field names') models = deepcopy(PAGINATION) diff --git a/usaspending_api/core/validator/tests/unit/test_award.py b/usaspending_api/core/validator/tests/unit/test_award.py index 04d607ae4f..1fcc454ea5 100644 --- a/usaspending_api/core/validator/tests/unit/test_award.py +++ b/usaspending_api/core/validator/tests/unit/test_award.py @@ -35,7 +35,7 @@ def test_get_generate_award_id_rule(): r = TinyShield(models).block({}) assert r == {} - # Bad rules. + # Rule violations. ts = TinyShield([get_generated_award_id_rule()]) with pytest.raises(UnprocessableEntityException): @@ -83,7 +83,7 @@ def test_get_internal_award_id_rule(): r = TinyShield(models).block({}) assert r == {} - # Bad rules. + # Rule violations. ts = TinyShield([get_internal_award_id_rule()]) with pytest.raises(UnprocessableEntityException): @@ -156,7 +156,7 @@ def test_get_internal_or_generated_award_id_rule(): def test_get_internal_or_generated_award_id_rule_bad(): - # Bad rules. + # Rule violations. ts = TinyShield([get_internal_or_generated_award_id_rule()]) with pytest.raises(UnprocessableEntityException): diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py index cae5839b74..242cb3327f 100644 --- a/usaspending_api/core/validator/tinyshield.py +++ b/usaspending_api/core/validator/tinyshield.py @@ -185,8 +185,7 @@ def block(self, request): self.enforce_rules() return self.data - @staticmethod - def check_model(model, in_any=False): + def check_model(self, model, in_any=False): # Confirm required fields (both baseline and type-specific) are in the model base_minimum_fields = ('name', 'key', 'type') @@ -220,15 +219,14 @@ def check_model(model, in_any=False): sub_model['name'] = model['name'] sub_model['key'] = model['key'] sub_model['optional'] = True - TinyShield.check_model(sub_model, True) + self.check_model(sub_model, True) return model - @staticmethod - def check_models(models): + def check_models(self, models): for model in models: - TinyShield.check_model(model) + self.check_model(model) # Check to ensure unique names for destination dictionary keys = [x['name'] for x in models if x['type'] != 'schema'] # ignore schema as they are schema-only From 51fbf7d075c1bce371235ae77d02b0fc867c5372 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Thu, 7 Feb 2019 13:52:48 -0500 Subject: [PATCH 40/52] updates as requested --- usaspending_api/awards/v2/data_layer/orm.py | 13 ++++++++++++- usaspending_api/awards/v2/data_layer/orm_mappers.py | 13 +++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index c2aab1f18c..60b8d6ac75 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -2,6 +2,7 @@ import logging from collections import OrderedDict +from django.db.models import Sum from usaspending_api.awards.v2.data_layer.orm_mappers import ( FABS_AWARD_FIELDS, @@ -10,7 +11,9 @@ FPDS_AWARD_FIELDS, FABS_ASSISTANCE_FIELDS, ) -from usaspending_api.awards.models import Award, TransactionFABS, TransactionFPDS, ParentAward +from usaspending_api.awards.models import ( + Award, FinancialAccountsByAwards, TransactionFABS, TransactionFPDS, ParentAward +) from usaspending_api.common.helpers.date_helper import get_date_from_datetime from usaspending_api.recipient.models import RecipientLookup from usaspending_api.references.models import Agency, LegalEntity, LegalEntityOfficers, Cfda @@ -40,6 +43,7 @@ def construct_assistance_response(requested_award_dict): response["cfda_number"] = transaction["cfda_number"] response["cfda_title"] = transaction["cfda_title"] response["cfda_objectives"] = cfda_info.get("objectives") + response["transaction_obligated_amount"] = fetch_transaction_obligated_amount_by_internal_award_id(award["id"]) response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) @@ -332,3 +336,10 @@ def fetch_cfda_details_using_cfda_number(cfda): if not c: return {} return c + + +def fetch_transaction_obligated_amount_by_internal_award_id(internal_award_id): + _sum = FinancialAccountsByAwards.objects.filter( + award_id=internal_award_id).aggregate(Sum('transaction_obligated_amount')) + if _sum: + return _sum.get('transaction_obligated_amount__sum') diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index fab4fe0575..5cc9d113a4 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -23,7 +23,7 @@ ("total_subsidy_cost", "total_subsidy_cost"), ("total_loan_value", "total_loan_value"), ("total_obligation", "total_obligation"), - ("base_and_all_options_value", "base_and_all_options_value"), + ("base_and_all_options_value", "base_and_all_options"), ("base_exercised_options_val", "base_exercised_options"), ("non_federal_funding_amount", "non_federal_funding"), ("total_funding_amount", "total_funding"), @@ -34,6 +34,7 @@ ("funding_agency_id", "_funding_agency"), ("period_of_performance_start_date", "_start_date"), ("period_of_performance_current_end_date", "_end_date"), + ("date_signed", "date_signed"), ] ) @@ -50,7 +51,7 @@ ("description", "description"), ("total_obligation", "total_obligation"), ("base_exercised_options_val", "base_exercised_options_val"), - ("base_and_all_options_value", "base_and_all_options_value"), + ("base_and_all_options_value", "base_and_all_options"), ("subaward_count", "subaward_count"), ("total_subaward_amount", "total_subaward_amount"), # extra fields @@ -60,6 +61,7 @@ ("funding_agency_id", "_funding_agency"), ("period_of_performance_start_date", "_start_date"), ("period_of_performance_current_end_date", "_end_date"), + ("date_signed", "date_signed"), ] ) @@ -120,11 +122,14 @@ ("small_business_competitive", "small_business_competitive"), ("fair_opportunity_limi_desc", "fair_opportunity_limi_desc"), ("product_or_service_code", "product_or_service_code"), - ("product_or_service_co_desc", "product_or_service_co_desc"), + ("product_or_service_co_desc", "product_or_service_desc"), ("naics", "naics"), ("naics_description", "naics_description"), ("dod_claimant_program_code", "dod_claimant_program_code"), - ("program_system_or_equipmen", "program_system_or_equipmen"), + ("program_system_or_equipmen", "dod_acquisition_program_code"), + ("program_system_or_equ_desc", "dod_acquisition_program_description"), + ("information_technology_com", "information_technology_commercial_item_category_code"), + ("information_technolog_desc", "information_technology_commercial_item_category"), ("information_technolog_desc", "information_technolog_desc"), ("sea_transportation_desc", "sea_transportation_desc"), ("clinger_cohen_act_pla_desc", "clinger_cohen_act_pla_desc"), From 23be0e68074bf8148abc3e46540493bf54d4289f Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Thu, 7 Feb 2019 16:12:55 -0500 Subject: [PATCH 41/52] updates to tests --- .../awards/tests/test_awards_idv_v2.py | 13 +++++++++---- .../awards/tests/test_awards_v2.py | 19 +++++++++++++------ .../awards/v2/data_layer/orm_mappers.py | 1 - 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index 25cceb9430..6647b35dfe 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -155,6 +155,7 @@ def awards_and_transactions(db): "recipient": LegalEntity.objects.get(pk=1), "place_of_performance": Location.objects.get(pk=1), "latest_transaction": TransactionNormalized.objects.get(pk=1), + "date_signed": "2005-04-03", } award_2_model = { @@ -174,6 +175,7 @@ def awards_and_transactions(db): "latest_transaction": TransactionNormalized.objects.get(pk=2), "total_subaward_amount": 12345.00, "subaward_count": 10, + "date_signed": "2004-03-02", } mommy.make("awards.Award", **award_1_model) mommy.make("awards.Award", **award_2_model) @@ -256,7 +258,7 @@ def test_award_endpoint_different_ids(client, awards_and_transactions): "parent_recipient_name": None, }, "total_obligation": 1000.0, - "base_and_all_options_value": 2000.0, + "base_and_all_options": 2000.0, "base_exercised_options_val": None, "place_of_performance": { "address_line1": None, @@ -291,12 +293,14 @@ def test_award_endpoint_different_ids(client, awards_and_transactions): "small_business_competitive": False, "fair_opportunity_limi_desc": None, "product_or_service_code": "4730", - "product_or_service_co_desc": None, + "product_or_service_desc": None, "naics": "333911", "naics_description": "PUMP AND PUMPING EQUIPMENT MANUFACTURING", "dod_claimant_program_code": "C9E", - "program_system_or_equipmen": "000", - "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", + "dod_acquisition_program_code": "000", + "dod_acquisition_program_description": None, + "information_technology_commercial_item_category": "NOT IT PRODUCTS OR SERVICES", + "information_technology_commercial_item_category_code": None, "sea_transportation_desc": "NO", "clinger_cohen_act_pla_desc": "NO", "construction_wage_rat_desc": "NO", @@ -318,4 +322,5 @@ def test_award_endpoint_different_ids(client, awards_and_transactions): "subaward_count": 10, "total_subaward_amount": 12345.0, "executive_details": {"officers": []}, + "date_signed": "2004-03-02", } diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index d1c786d042..e10a0cbb41 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -85,7 +85,7 @@ def awards_and_transactions(db): "place_of_perform_zip_last4": "2135", "place_of_performance_zip5": "40221", "place_of_performance_forei": None, - "modified_at": "2000-01-02T00:00:00Z" + "modified_at": "2000-01-02T00:00:00Z", } cont_data = { "awardee_or_recipient_legal": "John's Pizza", @@ -175,6 +175,7 @@ def awards_and_transactions(db): "recipient": LegalEntity.objects.get(pk=1), "place_of_performance": Location.objects.get(pk=1), "latest_transaction": TransactionNormalized.objects.get(pk=1), + "date_signed": "2005-04-03", } award_2_model = { @@ -197,6 +198,7 @@ def awards_and_transactions(db): "latest_transaction": TransactionNormalized.objects.get(pk=2), "total_subaward_amount": 12345.00, "subaward_count": 10, + "date_signed": "2004-03-02", } mommy.make("awards.Award", **award_1_model) @@ -250,13 +252,14 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "cfda_objectives": None, "cfda_number": "1234", "cfda_title": "Shiloh", - "base_and_all_options_value": None, + "base_and_all_options": None, "base_exercised_options": None, "non_federal_funding": None, "total_funding": None, "total_loan_value": None, "total_obligation": None, "total_subsidy_cost": None, + "transaction_obligated_amount": None, "awarding_agency": { "id": 1, "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, @@ -314,6 +317,7 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "location_country_code": "PDA", "congressional_code": "-0-", }, + "date_signed": "2005-04-03", } @@ -362,7 +366,7 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "parent_recipient_name": None, }, "total_obligation": 1000.0, - "base_and_all_options_value": 2000.0, + "base_and_all_options": 2000.0, "base_exercised_options_val": None, "period_of_performance": { "period_of_performance_start_date": "2004-02-04", @@ -400,7 +404,6 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "fed_biz_opps_description": "YES", "foreign_funding_desc": "NOT APPLICABLE", "idv_type_description": None, - "information_technolog_desc": "NOT IT PRODUCTS OR SERVICES", "interagency_contract_desc": "NOT APPLICABLE", "labor_standards_descrip": "NO", "major_program": None, @@ -412,10 +415,12 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "number_of_offers_received": None, "other_than_full_and_o_desc": None, "price_evaluation_adjustmen": None, - "product_or_service_co_desc": None, + "dod_acquisition_program_code": "000", + "dod_acquisition_program_description": None, + "information_technology_commercial_item_category_code": None, + "information_technology_commercial_item_category": "NOT IT PRODUCTS OR SERVICES", "product_or_service_code": "4730", "program_acronym": None, - "program_system_or_equipmen": "000", "purchase_card_as_paym_desc": "NO", "referenced_idv_agency_iden": "9700", "sea_transportation_desc": "NO", @@ -426,8 +431,10 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "type_of_contract_pric_desc": "FIRM FIXED PRICE", "type_of_idc_description": None, "type_set_aside_description": None, + "product_or_service_desc": None, }, "subaward_count": 10, "total_subaward_amount": 12345.0, "executive_details": {"officers": []}, + "date_signed": "2004-03-02", } diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index 5cc9d113a4..8503a388ec 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -130,7 +130,6 @@ ("program_system_or_equ_desc", "dod_acquisition_program_description"), ("information_technology_com", "information_technology_commercial_item_category_code"), ("information_technolog_desc", "information_technology_commercial_item_category"), - ("information_technolog_desc", "information_technolog_desc"), ("sea_transportation_desc", "sea_transportation_desc"), ("clinger_cohen_act_pla_desc", "clinger_cohen_act_pla_desc"), ("construction_wage_rat_desc", "construction_wage_rat_desc"), From fc5ecd901606e98a1872528bbe3e85967db5d696 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Fri, 8 Feb 2019 10:24:57 -0500 Subject: [PATCH 42/52] requested updates --- usaspending_api/awards/tests/test_awards_idv_v2.py | 12 ++++++++++-- usaspending_api/awards/tests/test_awards_v2.py | 10 +++++----- usaspending_api/awards/v2/data_layer/orm.py | 13 +++++++------ usaspending_api/awards/v2/data_layer/orm_mappers.py | 2 +- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index 6647b35dfe..a69251453c 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -120,6 +120,7 @@ def awards_and_transactions(db): "place_of_performance_state": "TX", "place_of_perform_zip_last4": "2135", "place_of_performance_zip5": "40221", + "period_of_perf_potential_e": "2003-04-05", "price_evaluation_adjustmen": None, "product_or_service_code": "4730", "program_acronym": None, @@ -171,6 +172,8 @@ def awards_and_transactions(db): "recipient": LegalEntity.objects.get(pk=1), "total_obligation": 1000, "base_and_all_options_value": 2000, + "period_of_performance_start_date": "2004-02-04", + "period_of_performance_current_end_date": "2005-02-04", "generated_unique_award_id": "CONT_AW_9700_9700_03VD_SPM30012D3486", "latest_transaction": TransactionNormalized.objects.get(pk=2), "total_subaward_amount": 12345.00, @@ -221,7 +224,12 @@ def test_award_endpoint_different_ids(client, awards_and_transactions): "parent_award_piid": "1234", "parent_award": None, "description": "lorem ipsum", - "idv_dates": {"end_date": "2025-06-30", "last_modified_date": "2018-08-24", "start_date": None}, + "period_of_performance": { + "start_date": "2004-02-04", + "end_date": "2025-06-30", + "last_modified_date": "2018-08-24", + "potential_end_date": "2003-04-05", + }, "awarding_agency": { "id": 1, "toptier_agency": {"name": "agency name", "abbreviation": "some other stuff", "code": None}, @@ -259,7 +267,7 @@ def test_award_endpoint_different_ids(client, awards_and_transactions): }, "total_obligation": 1000.0, "base_and_all_options": 2000.0, - "base_exercised_options_val": None, + "base_exercised_options": None, "place_of_performance": { "address_line1": None, "address_line2": None, diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index e10a0cbb41..4682d1a7a9 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -298,8 +298,8 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "subaward_count": 10, "total_subaward_amount": 12345.0, "period_of_performance": { - "period_of_performance_current_end_date": "2005-02-04", - "period_of_performance_start_date": "2004-02-04", + "start_date": "2004-02-04", + "end_date": "2005-02-04", "last_modified_date": "2000-01-02", }, "place_of_performance": { @@ -367,10 +367,10 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): }, "total_obligation": 1000.0, "base_and_all_options": 2000.0, - "base_exercised_options_val": None, + "base_exercised_options": None, "period_of_performance": { - "period_of_performance_start_date": "2004-02-04", - "period_of_performance_current_end_date": "2005-02-04", + "start_date": "2004-02-04", + "end_date": "2005-02-04", "last_modified_date": "2001-02-03", "potential_end_date": "2003-04-05", }, diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 60b8d6ac75..8cc84171d8 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -49,8 +49,8 @@ def construct_assistance_response(requested_award_dict): response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) response["period_of_performance"] = OrderedDict( [ - ("period_of_performance_start_date", award["_start_date"]), - ("period_of_performance_current_end_date", award["_end_date"]), + ("start_date", award["_start_date"]), + ("end_date", award["_end_date"]), ("last_modified_date", get_date_from_datetime(transaction["_modified_at"])), ] ) @@ -83,8 +83,8 @@ def construct_contract_response(requested_award_dict): response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) response["period_of_performance"] = OrderedDict( [ - ("period_of_performance_start_date", award["_start_date"]), - ("period_of_performance_current_end_date", award["_end_date"]), + ("start_date", award["_start_date"]), + ("end_date", award["_end_date"]), ("last_modified_date", transaction["_last_modified"]), ("potential_end_date", transaction["_period_of_perf_potential_e"]), ] @@ -130,11 +130,12 @@ def construct_idv_response(requested_award_dict): response["latest_transaction_contract_data"] = transaction response["funding_agency"] = fetch_agency_details(response["_funding_agency"]) response["awarding_agency"] = fetch_agency_details(response["_awarding_agency"]) - response["idv_dates"] = OrderedDict( + response["period_of_performance"] = OrderedDict( [ ("start_date", award["_start_date"]), - ("last_modified_date", transaction["_last_modified_date"]), ("end_date", transaction["_end_date"]), + ("last_modified_date", transaction["_last_modified_date"]), + ("potential_end_date", transaction["_period_of_perf_potential_e"]), ] ) transaction["_lei"] = award["_lei"] diff --git a/usaspending_api/awards/v2/data_layer/orm_mappers.py b/usaspending_api/awards/v2/data_layer/orm_mappers.py index 8503a388ec..118879662c 100644 --- a/usaspending_api/awards/v2/data_layer/orm_mappers.py +++ b/usaspending_api/awards/v2/data_layer/orm_mappers.py @@ -50,7 +50,7 @@ ("type_description", "type_description"), ("description", "description"), ("total_obligation", "total_obligation"), - ("base_exercised_options_val", "base_exercised_options_val"), + ("base_exercised_options_val", "base_exercised_options"), ("base_and_all_options_value", "base_and_all_options"), ("subaward_count", "subaward_count"), ("total_subaward_amount", "total_subaward_amount"), From 36f81d7c1a9db317c0e75823ec3cd871457a12da Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Mon, 11 Feb 2019 13:14:23 -0500 Subject: [PATCH 43/52] fix for multi-db connection issue --- usaspending_api/awards/v2/views/idvs/awards.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usaspending_api/awards/v2/views/idvs/awards.py b/usaspending_api/awards/v2/views/idvs/awards.py index 3bf4147d31..1e0dcc58ad 100644 --- a/usaspending_api/awards/v2/views/idvs/awards.py +++ b/usaspending_api/awards/v2/views/idvs/awards.py @@ -1,10 +1,11 @@ from collections import OrderedDict -from django.db import connection +from django.db import connections, router from psycopg2.sql import Identifier, Literal, SQL from rest_framework.request import Request from rest_framework.response import Response +from usaspending_api.awards.models import Award # We only use this model to get a connection from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata from usaspending_api.common.views import APIDocumentationView @@ -120,6 +121,10 @@ def _business_logic(request_data: dict) -> list: limit=Literal(request_data['limit'] + 1), offset=Literal((request_data['page'] - 1) * request_data['limit']), ) + # Because we use multiple read connections, we need to actually choose + # a connection. Using the default connection won't work in production. + # A model is a required parameter for db_for_read. + connection = connections[router.db_for_read(Award)] with connection.cursor() as cursor: # We must convert this to an actual query string else # django-debug-toolbar will blow up since it is assuming a string From 65f72791ea5a8113722043534fae3a89dfbe51af Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Tue, 19 Feb 2019 10:21:14 -0500 Subject: [PATCH 44/52] fixed Recipient Hash issue --- usaspending_api/awards/v2/data_layer/orm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 8cc84171d8..ed01a98d56 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -15,7 +15,7 @@ Award, FinancialAccountsByAwards, TransactionFABS, TransactionFPDS, ParentAward ) from usaspending_api.common.helpers.date_helper import get_date_from_datetime -from usaspending_api.recipient.models import RecipientLookup +from usaspending_api.recipient.models import RecipientProfile from usaspending_api.references.models import Agency, LegalEntity, LegalEntityOfficers, Cfda from usaspending_api.awards.v2.data_layer.orm_utils import delete_keys_from_dict, split_mapper_into_qs @@ -320,7 +320,7 @@ def fetch_officers_by_legal_entity_id(legal_entity_id): def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id): recipient = None if recipient_unique_id: - recipient = RecipientLookup.objects.filter(duns=recipient_unique_id).values("recipient_hash").first() + recipient = RecipientProfile.objects.filter(recipient_unique_id=recipient_unique_id).values("recipient_hash", "recipient_level").first() if not recipient: # SQL: MD5(UPPER(CONCAT(awardee_or_recipient_uniqu, legal_business_name)))::uuid @@ -328,8 +328,8 @@ def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id import uuid h = hashlib.md5("{}{}".format(recipient_unique_id, recipient_name).upper().encode("utf-8")).hexdigest() - return str(uuid.UUID(h)) - return recipient["recipient_hash"] + return "{}-R".format(uuid.UUID(h)) + return "{recipient_hash}-{recipient_level}".format(**recipient) def fetch_cfda_details_using_cfda_number(cfda): From cd293e69cb906242ff1fde7ac396dfaa6e056139 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Tue, 19 Feb 2019 10:25:31 -0500 Subject: [PATCH 45/52] code style --- usaspending_api/awards/v2/data_layer/orm.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index ed01a98d56..047c38e268 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -320,7 +320,11 @@ def fetch_officers_by_legal_entity_id(legal_entity_id): def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id): recipient = None if recipient_unique_id: - recipient = RecipientProfile.objects.filter(recipient_unique_id=recipient_unique_id).values("recipient_hash", "recipient_level").first() + recipient = ( + RecipientProfile.objects.filter(recipient_unique_id=recipient_unique_id) + .values("recipient_hash", "recipient_level") + .first() + ) if not recipient: # SQL: MD5(UPPER(CONCAT(awardee_or_recipient_uniqu, legal_business_name)))::uuid @@ -340,7 +344,9 @@ def fetch_cfda_details_using_cfda_number(cfda): def fetch_transaction_obligated_amount_by_internal_award_id(internal_award_id): - _sum = FinancialAccountsByAwards.objects.filter( - award_id=internal_award_id).aggregate(Sum('transaction_obligated_amount')) + _sum = ( + FinancialAccountsByAwards.objects.filter(award_id=internal_award_id) + .aggregate(Sum("transaction_obligated_amount")) + ) if _sum: - return _sum.get('transaction_obligated_amount__sum') + return _sum.get("transaction_obligated_amount__sum") From c2665c992e9730b7ea8a3adbb43763203a3fb8cd Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Tue, 19 Feb 2019 13:51:24 -0500 Subject: [PATCH 46/52] added new recipient uri logic in common location --- usaspending_api/awards/v2/data_layer/orm.py | 30 +++-------- usaspending_api/common/recipient_lookups.py | 59 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 usaspending_api/common/recipient_lookups.py diff --git a/usaspending_api/awards/v2/data_layer/orm.py b/usaspending_api/awards/v2/data_layer/orm.py index 047c38e268..d814794a8c 100644 --- a/usaspending_api/awards/v2/data_layer/orm.py +++ b/usaspending_api/awards/v2/data_layer/orm.py @@ -14,10 +14,10 @@ from usaspending_api.awards.models import ( Award, FinancialAccountsByAwards, TransactionFABS, TransactionFPDS, ParentAward ) +from usaspending_api.awards.v2.data_layer.orm_utils import delete_keys_from_dict, split_mapper_into_qs from usaspending_api.common.helpers.date_helper import get_date_from_datetime -from usaspending_api.recipient.models import RecipientProfile +from usaspending_api.common.recipient_lookups import obtain_recipient_uri from usaspending_api.references.models import Agency, LegalEntity, LegalEntityOfficers, Cfda -from usaspending_api.awards.v2.data_layer.orm_utils import delete_keys_from_dict, split_mapper_into_qs logger = logging.getLogger("console") @@ -149,9 +149,10 @@ def create_recipient_object(db_row_dict): return OrderedDict( [ ( - "recipient_hash", - fetch_recipient_hash_using_name_and_duns( - db_row_dict["_recipient_name"], db_row_dict["_recipient_unique_id"] + "recipient_hash", obtain_recipient_uri( + db_row_dict["_recipient_name"], + db_row_dict["_recipient_unique_id"], + db_row_dict["_parent_recipient_unique_id"], ), ), ("recipient_name", db_row_dict["_recipient_name"]), @@ -317,25 +318,6 @@ def fetch_officers_by_legal_entity_id(legal_entity_id): return {"officers": officers} -def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id): - recipient = None - if recipient_unique_id: - recipient = ( - RecipientProfile.objects.filter(recipient_unique_id=recipient_unique_id) - .values("recipient_hash", "recipient_level") - .first() - ) - - if not recipient: - # SQL: MD5(UPPER(CONCAT(awardee_or_recipient_uniqu, legal_business_name)))::uuid - import hashlib - import uuid - - h = hashlib.md5("{}{}".format(recipient_unique_id, recipient_name).upper().encode("utf-8")).hexdigest() - return "{}-R".format(uuid.UUID(h)) - return "{recipient_hash}-{recipient_level}".format(**recipient) - - def fetch_cfda_details_using_cfda_number(cfda): c = Cfda.objects.filter(program_number=cfda).values("program_title", "objectives").first() if not c: diff --git a/usaspending_api/common/recipient_lookups.py b/usaspending_api/common/recipient_lookups.py new file mode 100644 index 0000000000..d60c0fdd97 --- /dev/null +++ b/usaspending_api/common/recipient_lookups.py @@ -0,0 +1,59 @@ +from usaspending_api.recipient.models import RecipientLookup + + +def obtain_recipient_uri(recipient_name, recipient_unique_id, parent_recipient_unique_id=None): + if not recipient_unique_id: + return create_recipient_uri_without_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id) + + return fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id) + + +def create_recipient_uri_without_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id=None): + recipient_hash = generate_missing_recipient_hash(recipient_name, recipient_unique_id) + recipient_level = obtain_recipient_level({"duns": recipient_unique_id, "parent_duns": parent_recipient_unique_id}) + return combine_recipient_hash_and_level(recipient_hash, recipient_level) + + +def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id): + recipient_hash = fetch_recipient_hash_using_duns(recipient_unique_id) + recipient_level = obtain_recipient_level({"duns": recipient_unique_id, "parent_duns": parent_recipient_unique_id}) + return combine_recipient_hash_and_level(recipient_hash, recipient_level) + + +def generate_missing_recipient_hash(recipient_unique_id, recipient_name): + # SQL: MD5(UPPER(CONCAT(awardee_or_recipient_uniqu, legal_business_name)))::uuid + import hashlib + import uuid + + h = hashlib.md5("{}{}".format(recipient_unique_id, recipient_name).upper().encode("utf-8")).hexdigest() + return str(uuid.UUID(h)) + + +def fetch_recipient_hash_using_duns(recipient_unique_id): + recipient = ( + RecipientLookup.objects.filter(duns=recipient_unique_id) + .values("recipient_hash") + .first() + ) + + return recipient["recipient_hash"] if recipient else None + + +def obtain_recipient_level(recipient_record: dict) -> str: + if recipient_is_standalone(recipient_record): + level = "R" + elif recipient_is_child(recipient_record): + level = "C" + return level + + +def recipient_is_standalone(recipient_record: dict) -> bool: + return recipient_record["parent_duns"] is None + + +def recipient_is_child(recipient_record: dict) -> bool: + return recipient_record["parent_duns"] is not None + + +def combine_recipient_hash_and_level(recipient_hash, recipient_level): + return "{}-{}".format(recipient_hash, recipient_level.upper()) From 940fd2722ab45c3bd5fda37492935cfc606b7f97 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Tue, 19 Feb 2019 14:37:55 -0500 Subject: [PATCH 47/52] Corrected tests expected values for recipient hash --- usaspending_api/awards/tests/test_awards_idv_v2.py | 10 ++++++---- usaspending_api/awards/tests/test_awards_v2.py | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/usaspending_api/awards/tests/test_awards_idv_v2.py b/usaspending_api/awards/tests/test_awards_idv_v2.py index a69251453c..edaa7f68d6 100644 --- a/usaspending_api/awards/tests/test_awards_idv_v2.py +++ b/usaspending_api/awards/tests/test_awards_idv_v2.py @@ -30,10 +30,12 @@ def awards_and_transactions(db): trans_asst = {"pk": 1} trans_cont = {"pk": 2} - duns = {"awardee_or_recipient_uniqu": 123, "legal_business_name": "Sams Club"} + duns = {"awardee_or_recipient_uniqu": "123", "legal_business_name": "Sams Club"} + recipient_lookup = {"duns": "456", "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367"} mommy.make("references.Cfda", program_number=1234) mommy.make("references.Location", **loc) mommy.make("recipient.DUNS", **duns) + mommy.make("recipient.RecipientLookup", **recipient_lookup) mommy.make("references.SubtierAgency", **subag) mommy.make("references.ToptierAgency", **subag) mommy.make("references.OfficeAgency", name="office_agency") @@ -41,8 +43,8 @@ def awards_and_transactions(db): le = { "pk": 1, "recipient_name": "John's Pizza", - "recipient_unique_id": 456, - "parent_recipient_unique_id": 123, + "recipient_unique_id": "456", + "parent_recipient_unique_id": "123", "business_categories": ["small_business"], "location": Location.objects.get(pk=1), } @@ -243,7 +245,7 @@ def test_award_endpoint_different_ids(client, awards_and_transactions): "office_agency_name": "office_agency", }, "recipient": { - "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367", + "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367-C", "recipient_name": "John's Pizza", "recipient_unique_id": "456", "parent_recipient_unique_id": "123", diff --git a/usaspending_api/awards/tests/test_awards_v2.py b/usaspending_api/awards/tests/test_awards_v2.py index 4682d1a7a9..0979630c24 100644 --- a/usaspending_api/awards/tests/test_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_v2.py @@ -31,10 +31,12 @@ def awards_and_transactions(db): sub_agency = {"pk": 1, "name": "agency name", "abbreviation": "some other stuff"} trans_asst = {"pk": 1} trans_cont = {"pk": 2} - duns = {"awardee_or_recipient_uniqu": 123, "legal_business_name": "Sams Club"} + duns = {"awardee_or_recipient_uniqu": "123", "legal_business_name": "Sams Club"} + recipient_lookup = {"duns": "456", "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367"} mommy.make("references.Cfda", program_number=1234) mommy.make("references.Location", **loc) mommy.make("recipient.DUNS", **duns) + mommy.make("recipient.RecipientLookup", **recipient_lookup) mommy.make("references.SubtierAgency", **sub_agency) mommy.make("references.ToptierAgency", **sub_agency) mommy.make("references.OfficeAgency", name="office_agency", office_agency_id=1) @@ -42,7 +44,6 @@ def awards_and_transactions(db): le = { "pk": 1, "business_categories": ["small_business"], - # "location": Location.objects.get(pk=1), } ag = { @@ -273,7 +274,7 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "office_agency_name": "office_agency", }, "recipient": { - "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367", + "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367-C", "recipient_name": "John's Pizza", "recipient_unique_id": "456", "parent_recipient_unique_id": "123", @@ -343,7 +344,7 @@ def test_award_endpoint_generated_id(client, awards_and_transactions): "office_agency_name": "office_agency", }, "recipient": { - "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367", + "recipient_hash": "f989e299-1f50-2600-f2f7-b6a45d11f367-C", "recipient_name": "John's Pizza", "recipient_unique_id": "456", "parent_recipient_unique_id": "123", From eb433d210bc6d5dea1d9a6eca548cceebe466535 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Tue, 19 Feb 2019 17:39:45 -0500 Subject: [PATCH 48/52] fixes and changes from PR review --- usaspending_api/common/recipient_lookups.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/usaspending_api/common/recipient_lookups.py b/usaspending_api/common/recipient_lookups.py index d60c0fdd97..019ba54afa 100644 --- a/usaspending_api/common/recipient_lookups.py +++ b/usaspending_api/common/recipient_lookups.py @@ -4,8 +4,7 @@ def obtain_recipient_uri(recipient_name, recipient_unique_id, parent_recipient_unique_id=None): if not recipient_unique_id: return create_recipient_uri_without_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id) - - return fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id) + return fetch_recipient_uri_with_duns(recipient_unique_id, parent_recipient_unique_id) def create_recipient_uri_without_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id=None): @@ -14,13 +13,13 @@ def create_recipient_uri_without_duns(recipient_name, recipient_unique_id, paren return combine_recipient_hash_and_level(recipient_hash, recipient_level) -def fetch_recipient_hash_using_name_and_duns(recipient_name, recipient_unique_id, parent_recipient_unique_id): +def fetch_recipient_uri_with_duns(recipient_unique_id, parent_recipient_unique_id): recipient_hash = fetch_recipient_hash_using_duns(recipient_unique_id) recipient_level = obtain_recipient_level({"duns": recipient_unique_id, "parent_duns": parent_recipient_unique_id}) return combine_recipient_hash_and_level(recipient_hash, recipient_level) -def generate_missing_recipient_hash(recipient_unique_id, recipient_name): +def generate_missing_recipient_hash(recipient_name, recipient_unique_id): # SQL: MD5(UPPER(CONCAT(awardee_or_recipient_uniqu, legal_business_name)))::uuid import hashlib import uuid @@ -40,10 +39,12 @@ def fetch_recipient_hash_using_duns(recipient_unique_id): def obtain_recipient_level(recipient_record: dict) -> str: + level = None if recipient_is_standalone(recipient_record): level = "R" elif recipient_is_child(recipient_record): level = "C" + # Can never be associated with a "parent" recipient profile level return level From 8e4a327573786fbf5aef92711a3e7b06bd676cff Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Thu, 21 Feb 2019 06:38:09 -0500 Subject: [PATCH 49/52] correction to TinyShield usage --- usaspending_api/awards/v2/views/idvs/amounts.py | 4 ++-- usaspending_api/awards/v2/views/idvs/awards.py | 16 ++++++---------- usaspending_api/awards/v2/views/transactions.py | 12 ++++-------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/usaspending_api/awards/v2/views/idvs/amounts.py b/usaspending_api/awards/v2/views/idvs/amounts.py index 3929d0848e..d59f51cecc 100644 --- a/usaspending_api/awards/v2/views/idvs/amounts.py +++ b/usaspending_api/awards/v2/views/idvs/amounts.py @@ -16,7 +16,7 @@ logger = logging.getLogger('console') -TINY_SHIELD_RULES = TinyShield([get_internal_or_generated_award_id_rule()]) +TINY_SHIELD_MODEL = [get_internal_or_generated_award_id_rule()] class IDVAmountsViewSet(APIDocumentationView): @@ -26,7 +26,7 @@ class IDVAmountsViewSet(APIDocumentationView): @staticmethod def _parse_and_validate_request(requested_award: str) -> dict: - return TINY_SHIELD_RULES.block({'award_id': requested_award}) + return TinyShield(TINY_SHIELD_MODEL).block({'award_id': requested_award}) @staticmethod def _business_logic(request_data: dict) -> OrderedDict: diff --git a/usaspending_api/awards/v2/views/idvs/awards.py b/usaspending_api/awards/v2/views/idvs/awards.py index 1e0dcc58ad..ad627d6e9b 100644 --- a/usaspending_api/awards/v2/views/idvs/awards.py +++ b/usaspending_api/awards/v2/views/idvs/awards.py @@ -79,20 +79,16 @@ """) -def _prepare_tiny_shield_rules(): - """ - Our TinyShield rules never change. Encapsulate them here and store them - once in TINY_SHIELD_RULES. - """ - models = customize_pagination_with_sort_columns(SORTABLE_COLUMNS, DEFAULT_SORT_COLUMN) - models.extend([ +def _prepare_tiny_shield_model(): + model = customize_pagination_with_sort_columns(SORTABLE_COLUMNS, DEFAULT_SORT_COLUMN) + model.extend([ get_internal_or_generated_award_id_rule(), {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True} ]) - return TinyShield(models) + return model -TINY_SHIELD_RULES = _prepare_tiny_shield_rules() +TINY_SHIELD_MODEL = _prepare_tiny_shield_model() class IDVAwardsViewSet(APIDocumentationView): @@ -102,7 +98,7 @@ class IDVAwardsViewSet(APIDocumentationView): @staticmethod def _parse_and_validate_request(request: Request) -> dict: - return TINY_SHIELD_RULES.block(request) + return TinyShield(TINY_SHIELD_MODEL).block(request) @staticmethod def _business_logic(request_data: dict) -> list: diff --git a/usaspending_api/awards/v2/views/transactions.py b/usaspending_api/awards/v2/views/transactions.py index 1680939877..d9ed1e4ba4 100644 --- a/usaspending_api/awards/v2/views/transactions.py +++ b/usaspending_api/awards/v2/views/transactions.py @@ -34,20 +34,16 @@ class TransactionViewSet(APIDocumentationView): } def __init__(self): - """ - Our TinyShield rules never change. Encapsulate them here and store - them once in TINY_SHIELD_RULES. - """ - models = customize_pagination_with_sort_columns(TransactionViewSet.transaction_lookup.keys(), 'action_date') - models.extend([ + model = customize_pagination_with_sort_columns(TransactionViewSet.transaction_lookup.keys(), 'action_date') + model.extend([ get_internal_or_generated_award_id_rule(), {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True} ]) - self._tiny_shield_rules = TinyShield(models) + self._tiny_shield_model = model super(TransactionViewSet, self).__init__() def _parse_and_validate_request(self, request_dict: dict) -> dict: - return self._tiny_shield_rules.block(request_dict) + return TinyShield(self._tiny_shield_model).block(request_dict) def _business_logic(self, request_data: dict) -> list: # By this point, our award_id has been validated and cleaned up by From e1f50ab786326207e438585b230b095a47747269 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Thu, 21 Feb 2019 07:28:35 -0500 Subject: [PATCH 50/52] add funding_agency_id by request --- .../api_docs/api_documentation/awards/idvs/awards.md | 2 ++ .../awards/tests/test_awards_idvs_awards_v2.py | 8 +++++++- usaspending_api/awards/v2/views/idvs/awards.py | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/usaspending_api/api_docs/api_documentation/awards/idvs/awards.md b/usaspending_api/api_docs/api_documentation/awards/idvs/awards.md index 840f173e74..afb5af0cb8 100644 --- a/usaspending_api/api_docs/api_documentation/awards/idvs/awards.md +++ b/usaspending_api/api_docs/api_documentation/awards/idvs/awards.md @@ -24,6 +24,7 @@ Returns IDVs or contracts related to the requested Indefinite Delivery Vehicle a "award_type": "DO", "description": "4524345064!OTHER GROCERY AND R", "funding_agency": "DEPARTMENT OF DEFENSE (DOD)", + "funding_agency_id": 1219, "generated_unique_award_id": "CONT_AW_9700_9700_71T0_SPM30008D3155", "last_date_to_order": null, "obligated_amount": 4080.71, @@ -48,6 +49,7 @@ Returns IDVs or contracts related to the requested Indefinite Delivery Vehicle a - `award_type`: Type of the award. See https://fedspendingtransparency.github.io/whitepapers/types/ for a better description. - `description`: Description of the award as provided by the funding agency. - `funding_agency`: Name of the agency that paid/is paying for the award. +- `funding_agency_id`: Internal surrogate key of the agency. - `generated_unique_award_id`: Natural key of Award. - `last_date_to_order`: The date after which no more orders may be placed against the award. - `obligated_amount`: The amount of money agreed upon for this award. diff --git a/usaspending_api/awards/tests/test_awards_idvs_awards_v2.py b/usaspending_api/awards/tests/test_awards_idvs_awards_v2.py index 74badb2cd9..c38b43b94f 100644 --- a/usaspending_api/awards/tests/test_awards_idvs_awards_v2.py +++ b/usaspending_api/awards/tests/test_awards_idvs_awards_v2.py @@ -45,9 +45,13 @@ def setUpTestData(cls): ordering_period_end_date='2018-01-%02d' % transaction_id ) - # We'll need some awards. + # We'll need some awards (and agencies). for award_id in range(1, AWARD_COUNT + 1): parent_n = PARENTS.get(award_id) + mommy.make( + 'references.Agency', + id=award_id * 12 + ) mommy.make( 'awards.Award', id=award_id, @@ -63,6 +67,7 @@ def setUpTestData(cls): description='description_%s' % award_id, period_of_performance_current_end_date='2018-03-%02d' % award_id, period_of_performance_start_date='2018-02-%02d' % award_id, + funding_agency_id=award_id * 12, ) # We'll need some parent_awards. @@ -94,6 +99,7 @@ def _generate_expected_response(previous, next, page, has_previous, has_next, *a 'award_type': 'type_description_%s' % award_id, 'description': 'description_%s' % award_id, 'funding_agency': 'funding_agency_name_%s' % award_id, + 'funding_agency_id': award_id * 12, 'generated_unique_award_id': 'GENERATED_UNIQUE_AWARD_ID_%s' % award_id, 'last_date_to_order': '2018-01-%02d' % award_id, 'obligated_amount': float(award_id * (1000 if award_id in IDVS else 1)), diff --git a/usaspending_api/awards/v2/views/idvs/awards.py b/usaspending_api/awards/v2/views/idvs/awards.py index 1e0dcc58ad..a7cf6b5364 100644 --- a/usaspending_api/awards/v2/views/idvs/awards.py +++ b/usaspending_api/awards/v2/views/idvs/awards.py @@ -35,6 +35,7 @@ ac.type_description award_type, ac.description, tf.funding_agency_name funding_agency, + ac.funding_agency_id, ac.generated_unique_award_id, tf.ordering_period_end_date last_date_to_order, pac.rollup_total_obligation obligated_amount, @@ -59,6 +60,7 @@ ac.type_description award_type, ac.description, tf.funding_agency_name funding_agency, + ac.funding_agency_id, ac.generated_unique_award_id, tf.ordering_period_end_date last_date_to_order, ac.total_obligation obligated_amount, From b01d8b6338f891abb26546c4013b09f4a495d189 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Thu, 21 Feb 2019 12:46:41 -0500 Subject: [PATCH 51/52] requested changes --- .../awards/v2/views/idvs/amounts.py | 7 +- .../awards/v2/views/idvs/awards.py | 17 ++--- .../awards/v2/views/transactions.py | 14 ++-- usaspending_api/core/validator/award.py | 8 +-- .../core/validator/tests/unit/test_award.py | 64 +++++++++---------- 5 files changed, 57 insertions(+), 53 deletions(-) diff --git a/usaspending_api/awards/v2/views/idvs/amounts.py b/usaspending_api/awards/v2/views/idvs/amounts.py index d59f51cecc..2d8d964d9f 100644 --- a/usaspending_api/awards/v2/views/idvs/amounts.py +++ b/usaspending_api/awards/v2/views/idvs/amounts.py @@ -1,6 +1,7 @@ import logging from collections import OrderedDict +from copy import deepcopy from rest_framework.exceptions import NotFound from rest_framework.request import Request @@ -9,14 +10,14 @@ from usaspending_api.awards.models import ParentAward from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.views import APIDocumentationView -from usaspending_api.core.validator.award import get_internal_or_generated_award_id_rule +from usaspending_api.core.validator.award import get_internal_or_generated_award_id_model from usaspending_api.core.validator.tinyshield import TinyShield logger = logging.getLogger('console') -TINY_SHIELD_MODEL = [get_internal_or_generated_award_id_rule()] +TINY_SHIELD_MODELS = [get_internal_or_generated_award_id_model()] class IDVAmountsViewSet(APIDocumentationView): @@ -26,7 +27,7 @@ class IDVAmountsViewSet(APIDocumentationView): @staticmethod def _parse_and_validate_request(requested_award: str) -> dict: - return TinyShield(TINY_SHIELD_MODEL).block({'award_id': requested_award}) + return TinyShield(deepcopy(TINY_SHIELD_MODELS)).block({'award_id': requested_award}) @staticmethod def _business_logic(request_data: dict) -> OrderedDict: diff --git a/usaspending_api/awards/v2/views/idvs/awards.py b/usaspending_api/awards/v2/views/idvs/awards.py index ad627d6e9b..be5d88cd34 100644 --- a/usaspending_api/awards/v2/views/idvs/awards.py +++ b/usaspending_api/awards/v2/views/idvs/awards.py @@ -1,4 +1,5 @@ from collections import OrderedDict +from copy import deepcopy from django.db import connections, router from psycopg2.sql import Identifier, Literal, SQL @@ -9,7 +10,7 @@ from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata from usaspending_api.common.views import APIDocumentationView -from usaspending_api.core.validator.award import get_internal_or_generated_award_id_rule +from usaspending_api.core.validator.award import get_internal_or_generated_award_id_model from usaspending_api.core.validator.pagination import customize_pagination_with_sort_columns from usaspending_api.core.validator.tinyshield import TinyShield from usaspending_api.etl.broker_etl_helpers import dictfetchall @@ -79,16 +80,16 @@ """) -def _prepare_tiny_shield_model(): - model = customize_pagination_with_sort_columns(SORTABLE_COLUMNS, DEFAULT_SORT_COLUMN) - model.extend([ - get_internal_or_generated_award_id_rule(), +def _prepare_tiny_shield_models(): + models = customize_pagination_with_sort_columns(SORTABLE_COLUMNS, DEFAULT_SORT_COLUMN) + models.extend([ + get_internal_or_generated_award_id_model(), {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True} ]) - return model + return models -TINY_SHIELD_MODEL = _prepare_tiny_shield_model() +TINY_SHIELD_MODELS = _prepare_tiny_shield_models() class IDVAwardsViewSet(APIDocumentationView): @@ -98,7 +99,7 @@ class IDVAwardsViewSet(APIDocumentationView): @staticmethod def _parse_and_validate_request(request: Request) -> dict: - return TinyShield(TINY_SHIELD_MODEL).block(request) + return TinyShield(deepcopy(TINY_SHIELD_MODELS)).block(request) @staticmethod def _business_logic(request_data: dict) -> list: diff --git a/usaspending_api/awards/v2/views/transactions.py b/usaspending_api/awards/v2/views/transactions.py index d9ed1e4ba4..a07535cfda 100644 --- a/usaspending_api/awards/v2/views/transactions.py +++ b/usaspending_api/awards/v2/views/transactions.py @@ -1,3 +1,5 @@ +from copy import deepcopy + from django.db.models import F from rest_framework.request import Request from rest_framework.response import Response @@ -6,7 +8,7 @@ from usaspending_api.common.cache_decorator import cache_response from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata from usaspending_api.common.views import APIDocumentationView -from usaspending_api.core.validator.award import get_internal_or_generated_award_id_rule +from usaspending_api.core.validator.award import get_internal_or_generated_award_id_model from usaspending_api.core.validator.pagination import customize_pagination_with_sort_columns from usaspending_api.core.validator.tinyshield import TinyShield @@ -34,16 +36,16 @@ class TransactionViewSet(APIDocumentationView): } def __init__(self): - model = customize_pagination_with_sort_columns(TransactionViewSet.transaction_lookup.keys(), 'action_date') - model.extend([ - get_internal_or_generated_award_id_rule(), + models = customize_pagination_with_sort_columns(TransactionViewSet.transaction_lookup.keys(), 'action_date') + models.extend([ + get_internal_or_generated_award_id_model(), {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True} ]) - self._tiny_shield_model = model + self._tiny_shield_models = models super(TransactionViewSet, self).__init__() def _parse_and_validate_request(self, request_dict: dict) -> dict: - return TinyShield(self._tiny_shield_model).block(request_dict) + return TinyShield(deepcopy(self._tiny_shield_models)).block(request_dict) def _business_logic(self, request_data: dict) -> list: # By this point, our award_id has been validated and cleaned up by diff --git a/usaspending_api/core/validator/award.py b/usaspending_api/core/validator/award.py index f1ecb03ab6..867a811f49 100644 --- a/usaspending_api/core/validator/award.py +++ b/usaspending_api/core/validator/award.py @@ -1,17 +1,17 @@ """ -Some shortcuts for generating "standardized" basic award id TinyShield model strings. +Some shortcuts for generating "standardized" basic award id TinyShield models. """ -def get_generated_award_id_rule(key='award_id', name='award_id', optional=False): +def get_generated_award_id_model(key='award_id', name='award_id', optional=False): return {'key': key, 'name': name, 'type': 'text', 'text_type': 'search', 'optional': optional} -def get_internal_award_id_rule(key='award_id', name='award_id', optional=False): +def get_internal_award_id_model(key='award_id', name='award_id', optional=False): return {'key': key, 'name': name, 'type': 'integer', 'optional': optional} -def get_internal_or_generated_award_id_rule(key='award_id', name='award_id', optional=False): +def get_internal_or_generated_award_id_model(key='award_id', name='award_id', optional=False): return {'key': key, 'name': name, 'type': 'any', 'optional': optional, 'models': [ {'type': 'integer'}, {'type': 'text', 'text_type': 'search'} diff --git a/usaspending_api/core/validator/tests/unit/test_award.py b/usaspending_api/core/validator/tests/unit/test_award.py index 1fcc454ea5..b12aa27670 100644 --- a/usaspending_api/core/validator/tests/unit/test_award.py +++ b/usaspending_api/core/validator/tests/unit/test_award.py @@ -5,38 +5,38 @@ from usaspending_api.common.exceptions import InvalidParameterException, UnprocessableEntityException from usaspending_api.core.validator.tinyshield import TinyShield -from usaspending_api.core.validator.award import get_generated_award_id_rule, get_internal_award_id_rule, \ - get_internal_or_generated_award_id_rule +from usaspending_api.core.validator.award import get_generated_award_id_model, get_internal_award_id_model, \ + get_internal_or_generated_award_id_model from usaspending_api.core.validator.helpers import MAX_INT, MAX_ITEMS, MIN_INT def test_get_generate_award_id_rule(): - models = [get_generated_award_id_rule()] + models = [get_generated_award_id_model()] r = TinyShield(models).block({'award_id': 'abcd'}) assert r == {'award_id': 'abcd'} - models = [get_generated_award_id_rule()] + models = [get_generated_award_id_model()] r = TinyShield(models).block({'award_id': 'A' * MAX_ITEMS}) assert r == {'award_id': 'A' * MAX_ITEMS} - models = [get_generated_award_id_rule(key='my_award_id')] + models = [get_generated_award_id_model(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id')] + models = [get_generated_award_id_model(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({}) assert r == {} # Rule violations. - ts = TinyShield([get_generated_award_id_rule()]) + ts = TinyShield([get_generated_award_id_model()]) with pytest.raises(UnprocessableEntityException): ts.block({}) @@ -51,40 +51,40 @@ def test_get_generate_award_id_rule(): def test_get_internal_award_id_rule(): - models = [get_internal_award_id_rule()] + models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': 12345}) assert r == {'award_id': 12345} - models = [get_internal_award_id_rule()] + models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': '12345'}) assert r == {'award_id': 12345} - models = [get_internal_award_id_rule()] + models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': MAX_INT}) assert r == {'award_id': MAX_INT} - models = [get_internal_award_id_rule()] + models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': MIN_INT}) assert r == {'award_id': MIN_INT} - models = [get_internal_award_id_rule(key='my_award_id')] + models = [get_internal_award_id_model(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id')] + models = [get_internal_award_id_model(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({}) assert r == {} # Rule violations. - ts = TinyShield([get_internal_award_id_rule()]) + ts = TinyShield([get_internal_award_id_model()]) with pytest.raises(UnprocessableEntityException): ts.block({}) @@ -101,55 +101,55 @@ def test_get_internal_award_id_rule(): def test_get_internal_or_generated_award_id_rule(): - models = [get_internal_or_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_model()] r = TinyShield(models).block({'award_id': 'abcd'}) assert r == {'award_id': 'abcd'} - models = [get_internal_or_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_model()] r = TinyShield(models).block({'award_id': 'A' * MAX_ITEMS}) assert r == {'award_id': 'A' * MAX_ITEMS} - models = [get_internal_or_generated_award_id_rule(key='my_award_id')] + models = [get_internal_or_generated_award_id_model(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id')] + models = [get_internal_or_generated_award_id_model(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_or_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} - models = [get_internal_or_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_model()] r = TinyShield(models).block({'award_id': 12345}) assert r == {'award_id': 12345} - models = [get_internal_or_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_model()] r = TinyShield(models).block({'award_id': '12345'}) assert r == {'award_id': 12345} - models = [get_internal_or_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_model()] r = TinyShield(models).block({'award_id': MAX_INT}) assert r == {'award_id': MAX_INT} - models = [get_internal_or_generated_award_id_rule()] + models = [get_internal_or_generated_award_id_model()] r = TinyShield(models).block({'award_id': MIN_INT}) assert r == {'award_id': MIN_INT} - models = [get_internal_or_generated_award_id_rule(key='my_award_id')] + models = [get_internal_or_generated_award_id_model(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id')] + models = [get_internal_or_generated_award_id_model(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_or_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} - models = [get_internal_or_generated_award_id_rule(key='my_award_id', name='your_award_id', optional=True)] + models = [get_internal_or_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({}) assert r == {} @@ -157,7 +157,7 @@ def test_get_internal_or_generated_award_id_rule(): def test_get_internal_or_generated_award_id_rule_bad(): # Rule violations. - ts = TinyShield([get_internal_or_generated_award_id_rule()]) + ts = TinyShield([get_internal_or_generated_award_id_model()]) with pytest.raises(UnprocessableEntityException): ts.block({}) From 60133161b53ce2c332cc550b7d3cc8a33acdb647 Mon Sep 17 00:00:00 2001 From: Kirk Barden Date: Thu, 21 Feb 2019 13:19:33 -0500 Subject: [PATCH 52/52] deep copy doesn't work with dict_keys --- usaspending_api/awards/v2/views/transactions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/usaspending_api/awards/v2/views/transactions.py b/usaspending_api/awards/v2/views/transactions.py index a07535cfda..7060960079 100644 --- a/usaspending_api/awards/v2/views/transactions.py +++ b/usaspending_api/awards/v2/views/transactions.py @@ -36,7 +36,10 @@ class TransactionViewSet(APIDocumentationView): } def __init__(self): - models = customize_pagination_with_sort_columns(TransactionViewSet.transaction_lookup.keys(), 'action_date') + models = customize_pagination_with_sort_columns( + list(TransactionViewSet.transaction_lookup.keys()), + 'action_date' + ) models.extend([ get_internal_or_generated_award_id_model(), {'key': 'idv', 'name': 'idv', 'type': 'boolean', 'default': True, 'optional': True}