From 9c61eb5c9c5d4ef144a7d7d52caa0b70416282d2 Mon Sep 17 00:00:00 2001 From: victorjz <2616689+victorjz@users.noreply.github.com> Date: Sun, 28 Jun 2020 15:57:28 -0700 Subject: [PATCH] [WIP] revised issue #1063 strictly for parsing district attorney number made the new mebmer district_attorney_number a 'proper' member ...meaning it's no longer optional with a default value ...so constructors across the project have been revised, including in tests did my best guessing when tests should have a blank DA number vs some value like 555 made other changes as suggested --- src/backend/expungeservice/crawler/crawler.py | 3 +- .../crawler/parsers/case_parser.py | 17 +++++----- .../crawler/parsers/record_parser.py | 1 + src/backend/expungeservice/models/case.py | 8 ++--- src/backend/tests/factories/case_factory.py | 6 ++-- src/backend/tests/models/test_case.py | 2 +- src/backend/tests/parser/test_case_parser.py | 32 +++++++++---------- src/backend/tests/test_edit_results.py | 2 ++ 8 files changed, 39 insertions(+), 32 deletions(-) diff --git a/src/backend/expungeservice/crawler/crawler.py b/src/backend/expungeservice/crawler/crawler.py index 0e884537a..86041a02b 100644 --- a/src/backend/expungeservice/crawler/crawler.py +++ b/src/backend/expungeservice/crawler/crawler.py @@ -86,6 +86,7 @@ def _search_record(session: Session, node_response, search_url, first_name, last @staticmethod def _read_case(session: Session, case_summary: CaseSummary) -> OeciCase: case_parser_data = Crawler._parse_case(session, case_summary) + district_attorney_number = case_parser_data.district_attorney_number balance_due_in_cents = CaseCreator.compute_balance_due_in_cents(case_parser_data.balance_due) charges: List[OeciCharge] = [] for charge_id, charge_dict in case_parser_data.hashed_charge_data.items(): @@ -95,7 +96,7 @@ def _read_case(session: Session, case_summary: CaseSummary) -> OeciCase: ) charges.append(charge) updated_case_summary = replace( - case_summary, balance_due_in_cents=balance_due_in_cents, edit_status=EditStatus.UNCHANGED + case_summary, district_attorney_number=district_attorney_number, balance_due_in_cents=balance_due_in_cents, edit_status=EditStatus.UNCHANGED ) return OeciCase(updated_case_summary, charges=tuple(charges)) diff --git a/src/backend/expungeservice/crawler/parsers/case_parser.py b/src/backend/expungeservice/crawler/parsers/case_parser.py index 36595ee5e..3fcbb3b85 100644 --- a/src/backend/expungeservice/crawler/parsers/case_parser.py +++ b/src/backend/expungeservice/crawler/parsers/case_parser.py @@ -17,18 +17,18 @@ @dataclass class CaseParserData: + district_attorney_number: str hashed_charge_data: Dict[int, Dict[str, str]] hashed_dispo_data: Dict[int, Dict[str, str]] balance_due: str probation_revoked: Optional[date] - distr_atty: str class CaseParser: @staticmethod def feed(data) -> CaseParserData: soup = BeautifulSoup(data, "html.parser") - distr_atty = CaseParser.__parse_distr_atty(soup) + district_attorney_number = CaseParser.__parse_district_attorney_number(soup) hashed_charge_data = CaseParser.__build_charge_table_data(soup) ( hashed_dispo_data, @@ -39,18 +39,19 @@ def feed(data) -> CaseParserData: probation_revoked = date.fromdatetime(datetime.strptime(probation_revoked_date_string, "%m/%d/%Y")) else: probation_revoked = None # type: ignore - return CaseParserData(hashed_charge_data, hashed_dispo_data, balance_due, probation_revoked, distr_atty) + return CaseParserData(district_attorney_number, hashed_charge_data, hashed_dispo_data, balance_due, probation_revoked) @staticmethod - def __parse_distr_atty(soup) -> str: - distr_atty_key = "District Attorney Number:" + def __parse_district_attorney_number(soup) -> str: + district_attorney_key = "District Attorney Number:" labels = soup.find_all("th", "ssTableHeaderLabel", limit=10) table = {} for tag in labels: table[tag.string] = tag.parent.find("td").string - if distr_atty_key in table: - return table[distr_atty_key] - return "" + if district_attorney_key in table: + return table[district_attorney_key] + else: + return "" @staticmethod def __build_charge_table_data(soup) -> Dict[int, Dict[str, str]]: diff --git a/src/backend/expungeservice/crawler/parsers/record_parser.py b/src/backend/expungeservice/crawler/parsers/record_parser.py index 7104f5a3c..77049f000 100644 --- a/src/backend/expungeservice/crawler/parsers/record_parser.py +++ b/src/backend/expungeservice/crawler/parsers/record_parser.py @@ -77,6 +77,7 @@ def __record_case(self): CaseCreator.create( self.info, self.case_number, + "", # district_attorney_number self.citation_number, self.date_location, self.type_status, diff --git a/src/backend/expungeservice/models/case.py b/src/backend/expungeservice/models/case.py index ee83bb8bc..1e9716b59 100644 --- a/src/backend/expungeservice/models/case.py +++ b/src/backend/expungeservice/models/case.py @@ -11,6 +11,7 @@ class CaseSummary: name: str birth_year: Optional[int] case_number: str + district_attorney_number: str citation_number: str location: str date: date_class @@ -19,7 +20,6 @@ class CaseSummary: case_detail_link: str balance_due_in_cents: int edit_status: EditStatus - distr_atty: str="" def get_balance_due(self): return self.balance_due_in_cents / 100 @@ -47,8 +47,8 @@ def empty(case_number: str): CaseSummary( name="", birth_year=1900, - # distr_atty="", case_number=case_number, + district_attorney_number="", citation_number="", location="", date=date_class.today(), @@ -71,7 +71,7 @@ class Case(OeciCase): class CaseCreator: @staticmethod def create( - info, case_number, citation_number, date_location, type_status, case_detail_link, balance="0" + info, case_number, district_attorney_number, citation_number, date_location, type_status, case_detail_link, balance="0" ) -> CaseSummary: name = info[0] birth_year = CaseSummary._parse_birth_year(info) @@ -84,6 +84,7 @@ def create( name, birth_year, case_number, + district_attorney_number, citation_number, location, date, @@ -92,7 +93,6 @@ def create( case_detail_link, balance_due_in_cents, EditStatus.UNCHANGED, - "", # distr_atty ) @staticmethod diff --git a/src/backend/tests/factories/case_factory.py b/src/backend/tests/factories/case_factory.py index ee10c063e..bfaea335f 100644 --- a/src/backend/tests/factories/case_factory.py +++ b/src/backend/tests/factories/case_factory.py @@ -6,6 +6,7 @@ class CaseSummaryFactory: def create( info=["John Doe", "1990"], case_number="1", + district_attorney_number="", citation_number=None, date_location=["1/1/1995", "Multnomah"], type_status=["Offense Misdemeanor", "Closed"], @@ -13,7 +14,7 @@ def create( balance="0", ) -> CaseSummary: return CaseCreator.create( - info, case_number, citation_number, date_location, type_status, case_detail_link, balance + info, case_number, district_attorney_number, citation_number, date_location, type_status, case_detail_link, balance ) @@ -22,6 +23,7 @@ class CaseFactory: def create( info=["John Doe", "1990"], case_number="1", + district_attorney_number="", citation_number=None, date_location=["1/1/1995", "Multnomah"], type_status=["Offense Misdemeanor", "Closed"], @@ -30,6 +32,6 @@ def create( balance="0", ) -> Case: case_summary = CaseSummaryFactory.create( - info, case_number, citation_number, date_location, type_status, case_detail_link, balance + info, case_number, district_attorney_number, citation_number, date_location, type_status, case_detail_link, balance ) return Case(case_summary, tuple(charges)) diff --git a/src/backend/tests/models/test_case.py b/src/backend/tests/models/test_case.py index cb957ac62..c2b86924e 100644 --- a/src/backend/tests/models/test_case.py +++ b/src/backend/tests/models/test_case.py @@ -6,7 +6,7 @@ class TestCaseBalanceDue(unittest.TestCase): def test_balance_due_getter_setter(self): - case_args = [("John Doe", "1990"), "", "", ("1/1/2019", ""), ("", ""), ""] + case_args = [("John Doe", "1990"), "", "", "", ("1/1/2019", ""), ("", ""), ""] case_1 = CaseCreator.create(*case_args, "123.45") # type: ignore assert case_1.get_balance_due() == 123.45 diff --git a/src/backend/tests/parser/test_case_parser.py b/src/backend/tests/parser/test_case_parser.py index f1d5e234c..a2a225c6f 100644 --- a/src/backend/tests/parser/test_case_parser.py +++ b/src/backend/tests/parser/test_case_parser.py @@ -49,8 +49,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert not self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "" class TestCaseWithoutFinancialTable(unittest.TestCase): @@ -79,8 +79,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert not self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "01234567" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "01234567" @@ -125,8 +125,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert not self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "555555-1" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "555555-1" class TestCaseWithoutDisposition(unittest.TestCase): @@ -163,8 +163,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert not self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "2377315-1" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "2377315-1" class TestParkingViolationCase(unittest.TestCase): @@ -190,8 +190,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert not self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "" class TestCaseWithRelatedCases(unittest.TestCase): @@ -220,8 +220,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert not self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "555555-B" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "555555-B" class TestFelicia(unittest.TestCase): @@ -257,8 +257,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert not self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "555555" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "555555" class TestRevokedProbation(unittest.TestCase): @@ -294,8 +294,8 @@ def test_charge_data_is_formatted(self): def test_probation_revoked_is_parsed(self): assert self.parser.probation_revoked - def test_distr_atty_is_parsed(self): - assert self.parser.distr_atty == "55555555" + def test_district_attorney_number_is_parsed(self): + assert self.parser.district_attorney_number == "55555555" class TestSpacesExistingInChargeInfoCells(unittest.TestCase): diff --git a/src/backend/tests/test_edit_results.py b/src/backend/tests/test_edit_results.py index d9702b10c..cc4d80daa 100644 --- a/src/backend/tests/test_edit_results.py +++ b/src/backend/tests/test_edit_results.py @@ -15,6 +15,7 @@ name="John Doe", birth_year=1980, case_number="X0001", + district_attorney_number="555", citation_number="X0001", location="earth", date=date(2001, 1, 1), @@ -58,6 +59,7 @@ name="John Albert Doe", birth_year=1970, case_number="X0002", + district_attorney_number="555", citation_number="X0002", location="america", date=date(1981, 1, 1),