Merge pull request #3779 from GSA-TTS/main

backend/audit/intakelib/checks/check_finding_prior_references_pattern.py

@@ -14,7 +14,7 @@
 logger = logging.getLogger(__name__)
 
 # A version of these regexes also exists in Base.libsonnet
-PRIOR_REFERENCES_REGEX = r"^20[1-9][0-9]-[0-9]{3}(,\s*20[1-9][0-9]-[0-9]{3})*$"
+PRIOR_REFERENCES_REGEX = r"^[1-2][0-9]{3}-[0-9]{3}(,\s*[1-2][0-9]{3}-[0-9]{3})*$"
 
 
 # DESCRIPTION

backend/audit/intakelib/checks/check_finding_reference_pattern.py

@@ -13,12 +13,12 @@
 logger = logging.getLogger(__name__)
 
 # A version of these regexes also exists in Base.libsonnet
-FINDING_REFERENCE_REGEX = r"^20[1-9][0-9]-[0-9]{3}(,\s*20[1-9][0-9]-[0-9]{3})*$"
+FINDING_REFERENCE_REGEX = r"^[1-2][0-9]{3}-[0-9]{3}(,\s*[1-2][0-9]{3}-[0-9]{3})*$"
 
 
 # DESCRIPTION
 # Finding references should be in 20##-### format where the first four
-# digits are a year >= 2010.
+# digits are a year >= 1900.
 # TESTED BY
 # has_bad_references.xlsx
 def finding_reference_pattern(ir):

backend/audit/intakelib/checks/check_version_number.py

@@ -7,7 +7,16 @@
 
 logger = logging.getLogger(__name__)
 
-AUTHORIZED_VERSIONS = {"1.0.0", "1.0.1", "1.0.2", "1.0.3", "1.0.4", "1.0.5", "1.1.0"}
+AUTHORIZED_VERSIONS = {
+    "1.0.0",
+    "1.0.1",
+    "1.0.2",
+    "1.0.3",
+    "1.0.4",
+    "1.0.5",
+    "1.1.0",
+    "1.1.1",
+}
 
 
 # DESCRIPTION

backend/audit/intakelib/checks/runners.py

@@ -129,13 +129,15 @@
     has_all_the_named_ranges(FORM_SECTIONS.FINDINGS_TEXT),
     has_all_required_fields(FORM_SECTIONS.FINDINGS_TEXT),
     has_invalid_yorn_field(FORM_SECTIONS.FINDINGS_TEXT),
+    finding_reference_pattern,
 ]
 
 corrective_action_plan_checks = general_checks + [
     is_right_workbook(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
     has_all_the_named_ranges(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
     has_all_required_fields(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
     has_invalid_yorn_field(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
+    finding_reference_pattern,
 ]
 
 secondary_auditors_checks = general_checks + [

backend/census_historical_migration/sac_general_lib/general_information.py

@@ -160,6 +160,25 @@ def xform_update_multiple_ueis_flag(audit_header):
         audit_header.MULTIPLEUEIS = "Y" if ueis else "N"
 
 
+def xform_update_entity_type(audit_header):
+    """Updates ENTITY_TYPE.
+    This updates does not propagate to the database, it only updates the object.
+    """
+    if string_to_string(audit_header.ENTITY_TYPE) == "":
+        audit_header.ENTITY_TYPE = (
+            "tribal"
+            if string_to_string(audit_header.SUPPRESSION_CODE).upper() == "IT"
+            else "unknown"
+        )
+        track_transformations(
+            "ENTITY_TYPE",
+            "",
+            "entity_type",
+            audit_header.ENTITY_TYPE,
+            "xform_update_entity_type",
+        )
+
+
 def _period_covered(s):
     """Helper to transform the period covered from Census format to FAC format."""
     if s not in PERIOD_DICT:
@@ -383,6 +402,7 @@ def general_information(audit_header):
     """Generates general information JSON."""
     xform_update_multiple_eins_flag(audit_header)
     xform_update_multiple_ueis_flag(audit_header)
+    xform_update_entity_type(audit_header)
     general_information = create_json_from_db_object(audit_header, mappings)
     transformations = [
         xform_auditee_fiscal_period_start,

backend/census_historical_migration/test_notes_to_sefa_xforms.py

@@ -2,7 +2,10 @@
 from django.test import SimpleTestCase
 
 from .exception_utils import DataMigrationError
-from .workbooklib.notes_to_sefa import xform_is_minimis_rate_used
+from .workbooklib.notes_to_sefa import (
+    xform_is_minimis_rate_used,
+    xform_missing_note_title_and_content,
+)
 
 
 class TestXformIsMinimisRateUsed(SimpleTestCase):
@@ -143,3 +146,83 @@ def test_ambiguous_or_unclear_raises_exception(self):
     def test_empty_string(self):
         """Test that the function returns GSA MIGRATION keyword when the input is an empty string."""
         self.assertEqual(xform_is_minimis_rate_used(""), settings.GSA_MIGRATION)
+
+
+class TestXformMissingNoteTitleAndContent(SimpleTestCase):
+    class MockNote:
+        def __init__(
+            self,
+            TITLE,
+            CONTENT,
+        ):
+            self.TITLE = TITLE
+            self.CONTENT = CONTENT
+
+    def _mock_notes_no_title(self):
+        notes = []
+        notes.append(
+            self.MockNote(
+                TITLE="",
+                CONTENT="SUPPORTIVE HOUSING FOR THE ELDERLY (14.157) - Balances outstanding at the end of the audit period were 3356.",
+            )
+        )
+        notes.append(
+            self.MockNote(
+                TITLE="",
+                CONTENT="MORTGAGE INSURANCE FOR THE PURCHASE OR REFINANCING OF EXISTING MULTIFAMILY HOUSING PROJECTS (14.155) - Balances outstanding at the end of the audit period were 4040.",
+            )
+        )
+        return notes
+
+    def _mock_notes_no_content(self):
+        notes = []
+        notes.append(
+            self.MockNote(
+                TITLE="Loan/loan guarantee outstanding balances",
+                CONTENT="",
+            )
+        )
+        notes.append(
+            self.MockNote(
+                TITLE="Federally Funded Insured Mortgages and Capital Advances",
+                CONTENT="",
+            )
+        )
+        return notes
+
+    def _mock_notes_with_title_content(self):
+        notes = []
+        notes.append(
+            self.MockNote(
+                TITLE="Loan/loan guarantee outstanding balances",
+                CONTENT="SUPPORTIVE HOUSING FOR THE ELDERLY (14.157) - Balances outstanding at the end of the audit period were 4000.",
+            )
+        )
+        notes.append(
+            self.MockNote(
+                TITLE="Federally Funded Insured Mortgages and Capital Advances",
+                CONTENT="MORTGAGE INSURANCE FOR THE PURCHASE OR REFINANCING OF EXISTING MULTIFAMILY HOUSING PROJECTS (14.155) - Balances outstanding at the end of the audit period were 5000.",
+            )
+        )
+        return notes
+
+    def test_note_w_no_title(self):
+        notes = self._mock_notes_no_title()
+        result = xform_missing_note_title_and_content(notes)
+        for note in result:
+            self.assertIn(settings.GSA_MIGRATION, note.TITLE)
+            self.assertNotIn(settings.GSA_MIGRATION, note.CONTENT)
+
+    def test_note_w_no_content(self):
+        notes = self._mock_notes_no_content()
+        result = xform_missing_note_title_and_content(notes)
+        for note in result:
+            self.assertNotIn(settings.GSA_MIGRATION, note.TITLE)
+            self.assertIn(settings.GSA_MIGRATION, note.CONTENT)
+
+    def test_note_with_title_content(self):
+        notes = self._mock_notes_with_title_content()
+        result = xform_missing_note_title_and_content(notes)
+        for note in result:
+            self.assertNotIn(settings.GSA_MIGRATION, note.TITLE)
+            self.assertNotIn(settings.GSA_MIGRATION, note.CONTENT)

backend/census_historical_migration/workbooklib/notes_to_sefa.py

@@ -167,28 +167,63 @@ def _get_notes(dbkey, year):
     return sort_by_field(results, "SEQ_NUMBER")
 
 
-def xform_missing_notes_records(audit_header, policies_content, rate_content):
-    """Transforms missing notes records for 2016, 2017, and 2018 audits."""
-    if string_to_string(audit_header.AUDITYEAR) in ["2018", "2017", "2016"] and not (
-        policies_content or rate_content
-    ):
-        policies_content = settings.GSA_MIGRATION
-        rate_content = settings.GSA_MIGRATION
+def xform_missing_notes_records_v2(audit_header, policies_content, rate_content):
+    """Transforms missing notes records for 2022, 2021, 2020, 2019, 2016, 2017 and 2018 audits.
+    Note:
+    This function replaces xform_missing_notes_records function.
+    This function covers all years 2016 through 2022.
+    This function tracks census data for policies_content and rate_content."""
+
+    if string_to_string(audit_header.AUDITYEAR) in [
+        "2022",
+        "2021",
+        "2020",
+        "2019",
+        "2018",
+        "2017",
+        "2016",
+    ] and not (policies_content or rate_content):
         track_data_transformation(
             policies_content,
             settings.GSA_MIGRATION,
-            "xform_missing_notes_records",
+            "xform_missing_notes_records_v2",
             "accounting_policies",
         )
+        policies_content = settings.GSA_MIGRATION
         track_data_transformation(
             rate_content,
             settings.GSA_MIGRATION,
-            "xform_missing_notes_records",
+            "xform_missing_notes_records_v2",
             "rate_explained",
         )
+        rate_content = settings.GSA_MIGRATION
     return policies_content, rate_content
 
 
+def xform_missing_note_title_and_content(notes):
+    """Transforms missing note title and note content."""
+    for note in notes:
+        if string_to_string(note.TITLE) == "" and string_to_string(note.CONTENT) != "":
+            track_data_transformation(
+                note.TITLE,
+                settings.GSA_MIGRATION,
+                "xform_missing_note_title_and_content",
+                "note_title",
+            )
+            note.TITLE = settings.GSA_MIGRATION
+
+        if string_to_string(note.CONTENT) == "" and string_to_string(note.TITLE) != "":
+            track_data_transformation(
+                note.CONTENT,
+                settings.GSA_MIGRATION,
+                "xform_missing_note_title_and_content",
+                "content",
+            )
+            note.CONTENT = settings.GSA_MIGRATION
+
+    return notes
+
+
 def generate_notes_to_sefa(audit_header, outfile):
     """
     Generates notes to SEFA workbook for a given audit header.
@@ -201,21 +236,27 @@ def generate_notes_to_sefa(audit_header, outfile):
     uei = xform_retrieve_uei(audit_header.UEI)
     set_workbook_uei(wb, uei)
     notes = _get_notes(audit_header.DBKEY, audit_header.AUDITYEAR)
+    notes = xform_missing_note_title_and_content(notes)
     rate_content, index = _get_minimis_cost_rate(
         audit_header.DBKEY, audit_header.AUDITYEAR
     )
     policies_content = _get_accounting_policies(
         audit_header.DBKEY, audit_header.AUDITYEAR
     )
     is_minimis_rate_used = xform_is_minimis_rate_used(rate_content, index)
-    policies_content, rate_content = xform_missing_notes_records(
+    policies_content, rate_content = xform_missing_notes_records_v2(
         audit_header, policies_content, rate_content
     )
     set_range(wb, "accounting_policies", [policies_content])
     set_range(wb, "is_minimis_rate_used", [is_minimis_rate_used])
     set_range(wb, "rate_explained", [rate_content])
 
-    contains_chart_or_tables = [settings.GSA_MIGRATION] * len(notes)
+    contains_chart_or_tables = []
+    for note in notes:
+        if string_to_string(note.TITLE) == "" and string_to_string(note.CONTENT) == "":
+            contains_chart_or_tables.append("")
+        else:
+            contains_chart_or_tables.append(settings.GSA_MIGRATION)
 
     # Map the rest as notes.
     map_simple_columns(wb, mappings, notes)

backend/schemas/output/excel/json/additional-eins-workbook.json

@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

backend/schemas/output/excel/json/additional-ueis-workbook.json

@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

backend/schemas/output/excel/json/audit-findings-text-workbook.json

@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {
@@ -102,9 +102,9 @@
                "title_cell": "A1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year after 2010 and NNN a three digit number (e.g. 2023-001)",
+                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year and NNN a three digit number (e.g. 2023-001)",
                   "custom_title": "Reference number",
-                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, LEFT(FIRSTCELLREF, 2) = \"20\", (MID(FIRSTCELLREF, 3, 1) * 1) > 0, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
+                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, (LEFT(FIRSTCELLREF, 2)*1) > 18, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
                   "type": "custom"
                },
                "width": 36

backend/schemas/output/excel/json/corrective-action-plan-workbook.json

@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {
@@ -102,9 +102,9 @@
                "title_cell": "A1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year after 2010 and NNN a three digit number (e.g. 2023-001)",
+                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year and NNN a three digit number (e.g. 2023-001)",
                   "custom_title": "Reference number",
-                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, LEFT(FIRSTCELLREF, 2) = \"20\", (MID(FIRSTCELLREF, 3, 1) * 1) > 0, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
+                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, (LEFT(FIRSTCELLREF, 2)*1) > 18, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
                   "type": "custom"
                },
                "width": 36

backend/schemas/output/excel/json/federal-awards-audit-findings-workbook.json

@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {
@@ -118,9 +118,9 @@
                "title_cell": "B1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year after 2010 and NNN a three digit number (e.g. 2023-001)",
+                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year and NNN a three digit number (e.g. 2023-001)",
                   "custom_title": "Reference number",
-                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, LEFT(FIRSTCELLREF, 2) = \"20\", (MID(FIRSTCELLREF, 3, 1) * 1) > 0, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
+                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, (LEFT(FIRSTCELLREF, 2)*1) > 18, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
                   "type": "custom"
                },
                "width": 18

backend/schemas/output/excel/json/federal-awards-workbook.json

@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

backend/schemas/output/excel/json/notes-to-sefa-workbook.json

@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

backend/schemas/output/excel/json/secondary-auditors-workbook.json

@@ -42,7 +42,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {