Merge pull request #1378 from bcgov/feature/ALCS-1610

Insert date_acknowledged_incomplete

bin/migrate-oats-data/applications/application_updates/__init__.py

@@ -0,0 +1,2 @@
+from .update_app_created_date import update_application_created_date
+from .update_app_ack_incomplete import update_application_incomplete_date

bin/migrate-oats-data/applications/application_updates/update_app_ack_incomplete.py

@@ -0,0 +1,111 @@
+from common import (
+    setup_and_get_logger,
+    BATCH_UPLOAD_SIZE
+)
+from db import inject_conn_pool
+from psycopg2.extras import RealDictCursor, execute_batch
+
+etl_name = "update_application_incomplete_date"
+logger = setup_and_get_logger(etl_name)
+
+
+@inject_conn_pool
+def update_application_incomplete_date(conn=None, batch_size=BATCH_UPLOAD_SIZE):
+    """
+    This function is responsible for updating existing the application acknowledged_incomplete_date in ALCS.
+
+    Args:
+    conn (psycopg2.extensions.connection): PostgreSQL database connection. Provided by the decorator.
+    batch_size (int): The number of items to process at once. Defaults to BATCH_UPLOAD_SIZE.
+    """
+
+    logger.info(f"Start {etl_name}")
+    with conn.cursor(cursor_factory=RealDictCursor) as cursor:
+        with open(
+            "applications/sql/application_update_incomplete_date_count.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            count_query = sql_file.read()
+            cursor.execute(count_query)
+            count_total = dict(cursor.fetchone())["count"]
+        logger.info(f"Total Application data to be updated: {count_total}")
+
+        failed_updates_count = 0
+        successful_updates_count = 0
+        last_file_number = 0
+
+        with open(
+            "applications/sql/application_update_incomplete_date.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            query = sql_file.read()
+            while True:
+                cursor.execute(
+                    f"""{query}
+                      AND as2.file_number::bigint > '{last_file_number}' ORDER BY as2.file_number::bigint;"""
+                )
+
+                rows = cursor.fetchmany(batch_size)
+
+                if not rows:
+                    break
+                try:
+                    conditions_to_be_updated_count = len(rows)
+
+                    _update_application_incomplete_date(conn, batch_size, cursor, rows)
+
+                    successful_updates_count = (
+                        successful_updates_count + conditions_to_be_updated_count
+                    )
+                    last_file_number = dict(rows[-1])["file_number"]
+
+                    logger.debug(
+                        f"retrieved/updated items count: {conditions_to_be_updated_count}; total successfully updated applications so far {successful_updates_count}; last updated application: {last_file_number}"
+                    )
+                except Exception as err:
+                    logger.exception(err)
+                    conn.rollback()
+                    failed_updates_count = count_total - successful_updates_count
+                    last_file_number = last_file_number + 1
+
+    logger.info(
+        f"Finished {etl_name}: total amount of successful updates {successful_updates_count}, total failed updates {failed_updates_count}"
+    )
+
+
+def _update_application_incomplete_date(conn, batch_size, cursor, rows):
+    data = _prepare_applications_data(rows)
+
+    if len(data) > 0:
+        execute_batch(
+            cursor,
+            _get_update_query(),
+            data,
+            page_size=batch_size,
+        )
+
+    conn.commit()
+
+
+def _get_update_query():
+    query = f"""
+                 UPDATE alcs.application
+                    SET date_acknowledged_incomplete = %(date_to_insert)s
+                    WHERE alcs.application.file_number = %(file_number)s;
+    """
+    return query
+
+
+def _prepare_applications_data(row_data_list):
+    data_list = []
+    date_to_insert = None
+    for row in row_data_list:
+        mapped_row = {
+            "date_to_insert": row.get("effective_date"),
+            "file_number": row.get("file_number"),
+        }
+        data_list.append(mapped_row)
+
+    return data_list

bin/migrate-oats-data/applications/migrate_application.py

@@ -52,7 +52,7 @@
 
 from .set_application_visibility import set_application_visibility
 
-from .update_app_created_date import update_application_created_date
+from .application_updates import update_application_created_date, update_application_incomplete_date
 
 from .application_decision_date import process_alcs_application_decision_date
 
@@ -66,6 +66,7 @@ def process_application_etl(batch_size):
     insert_application_submission_review(batch_size)
     process_alcs_application_decision_date(batch_size)
     process_application_statuses(batch_size)
+    update_application_incomplete_date(batch_size)
     process_application_parcels(batch_size)
     process_application_owners(batch_size)
     process_application_applicant_on_submissions(batch_size)

bin/migrate-oats-data/applications/sql/application_update_incomplete_date.sql

@@ -0,0 +1,9 @@
+SELECT
+    astss.effective_date,
+    as2.file_number
+FROM
+    alcs.application_submission as2
+    JOIN alcs.application_submission_to_submission_status astss ON as2."uuid" = astss.submission_uuid
+WHERE
+    astss.status_type_code = 'SUIN'
+    AND astss.effective_date IS NOT NULL

bin/migrate-oats-data/applications/sql/application_update_incomplete_date_count.sql

@@ -0,0 +1,8 @@
+SELECT
+    COUNT(*)
+FROM
+    alcs.application_submission as2
+    JOIN alcs.application_submission_to_submission_status astss ON as2."uuid" = astss.submission_uuid
+WHERE
+    astss.status_type_code = 'SUIN'
+    AND astss.effective_date IS NOT NULL;

bin/migrate-oats-data/noi/notice_of_intent_migration.py

@@ -31,6 +31,8 @@
 )
 from .oats_to_alcs_notice_of_intent_table_etl.update_notice_of_intent_created_date import update_noi_created_date
 
+from .oats_to_alcs_notice_of_intent_table_etl.notice_of_intent_update_ack_incomplete import update_notice_of_intent_incomplete_date
+
 from .notice_of_intent_submissions.parcels import (
     init_notice_of_intent_parcels,
     clean_parcels,
@@ -161,6 +163,8 @@ def process_notice_of_intent(batch_size):
 
     process_alcs_notice_of_intent_decision_released_status()
 
+    update_notice_of_intent_incomplete_date(batch_size)
+
     init_notice_of_intent_parcels(batch_size)
 
     process_notice_of_intent_certificate_of_title(batch_size)

bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/__init__.py

@@ -2,4 +2,5 @@
     process_alcs_notice_of_intent_base_fields,
 )
 from .notice_of_intent_rx_all_items import update_notice_of_intent_date_rx_all_items
-from .update_notice_of_intent_created_date import update_noi_created_date
+from .update_notice_of_intent_created_date import update_noi_created_date
+from .notice_of_intent_update_ack_incomplete import update_notice_of_intent_incomplete_date

bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/notice_of_intent_update_ack_incomplete.py

@@ -0,0 +1,111 @@
+from common import (
+    setup_and_get_logger,
+    BATCH_UPLOAD_SIZE
+)
+from db import inject_conn_pool
+from psycopg2.extras import RealDictCursor, execute_batch
+
+etl_name = "update_notice_of_intent_incomplete_date"
+logger = setup_and_get_logger(etl_name)
+
+
+@inject_conn_pool
+def update_notice_of_intent_incomplete_date(conn=None, batch_size=BATCH_UPLOAD_SIZE):
+    """
+    This function is responsible for updating existing the notice of intent acknowledged_incomplete_date in ALCS.
+
+    Args:
+    conn (psycopg2.extensions.connection): PostgreSQL database connection. Provided by the decorator.
+    batch_size (int): The number of items to process at once. Defaults to BATCH_UPLOAD_SIZE.
+    """
+
+    logger.info(f"Start {etl_name}")
+    with conn.cursor(cursor_factory=RealDictCursor) as cursor:
+        with open(
+            "noi/sql/notice_of_intent_base/notice_of_intent_update_ack_incomplete_count.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            count_query = sql_file.read()
+            cursor.execute(count_query)
+            count_total = dict(cursor.fetchone())["count"]
+        logger.info(f"Total Notice of intent data to be updated: {count_total}")
+
+        failed_updates_count = 0
+        successful_updates_count = 0
+        last_file_number = 0
+
+        with open(
+            "noi/sql/notice_of_intent_base/notice_of_intent_update_ack_incomplete.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            query = sql_file.read()
+            while True:
+                cursor.execute(
+                    f"""{query}
+                      AND nois.file_number::bigint > '{last_file_number}' ORDER BY nois.file_number::bigint;"""
+                )
+
+                rows = cursor.fetchmany(batch_size)
+
+                if not rows:
+                    break
+                try:
+                    conditions_to_be_updated_count = len(rows)
+
+                    _update_notice_of_intent_incomplete_date(conn, batch_size, cursor, rows)
+
+                    successful_updates_count = (
+                        successful_updates_count + conditions_to_be_updated_count
+                    )
+                    last_file_number = dict(rows[-1])["file_number"]
+
+                    logger.debug(
+                        f"retrieved/updated items count: {conditions_to_be_updated_count}; total successfully updated notice_of_intents so far {successful_updates_count}; last updated notice_of_intent: {last_file_number}"
+                    )
+                except Exception as err:
+                    logger.exception(err)
+                    conn.rollback()
+                    failed_updates_count = count_total - successful_updates_count
+                    last_file_number = last_file_number + 1
+
+    logger.info(
+        f"Finished {etl_name}: total amount of successful updates {successful_updates_count}, total failed updates {failed_updates_count}"
+    )
+
+
+def _update_notice_of_intent_incomplete_date(conn, batch_size, cursor, rows):
+    data = _prepare_notice_of_intents_data(rows)
+
+    if len(data) > 0:
+        execute_batch(
+            cursor,
+            _get_update_query(),
+            data,
+            page_size=batch_size,
+        )
+
+    conn.commit()
+
+
+def _get_update_query():
+    query = f"""
+                 UPDATE alcs.notice_of_intent
+                    SET date_acknowledged_incomplete = %(date_to_insert)s
+                    WHERE alcs.notice_of_intent.file_number = %(file_number)s;
+    """
+    return query
+
+
+def _prepare_notice_of_intents_data(row_data_list):
+    data_list = []
+    date_to_insert = None
+    for row in row_data_list:
+        mapped_row = {
+            "date_to_insert": row.get("effective_date"),
+            "file_number": row.get("file_number"),
+        }
+        data_list.append(mapped_row)
+
+    return data_list

bin/migrate-oats-data/noi/sql/notice_of_intent_base/notice_of_intent_update_ack_incomplete.sql

@@ -0,0 +1,9 @@
+SELECT
+    noitss.effective_date,
+    nois.file_number
+FROM
+    alcs.notice_of_intent_submission nois
+    JOIN alcs.notice_of_intent_submission_to_submission_status noitss ON nois."uuid" = noitss.submission_uuid
+WHERE
+    noitss.status_type_code = 'SUIN'
+    AND noitss.effective_date IS NOT NULL

bin/migrate-oats-data/noi/sql/notice_of_intent_base/notice_of_intent_update_ack_incomplete_count.sql

@@ -0,0 +1,8 @@
+SELECT
+    COUNT(*)
+FROM
+    alcs.notice_of_intent_submission nois
+    JOIN alcs.notice_of_intent_submission_to_submission_status noitss ON nois."uuid" = noitss.submission_uuid
+WHERE
+    noitss.status_type_code = 'SUIN'
+    AND noitss.effective_date IS NOT NULL