fix batch for: soil, adjacent_land,

bcgov · Nov 1, 2023 · 02487ad · 02487ad
1 parent f463746
commit 02487ad
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 38 deletions.
diff --git a/...sions/notice_of_intent_adjacent_land_use/notice_of_intent_submission_adjacent_land_use.py b/...sions/notice_of_intent_adjacent_land_use/notice_of_intent_submission_adjacent_land_use.py
@@ -38,7 +38,7 @@ def process_notice_of_intent_adjacent_land_use(conn=None, batch_size=BATCH_UPLOA
 
         failed_updates = 0
         successful_updates_count = 0
-        last_submission_id = 0
+        last_land_use_id = 0
 
         with open(
             "noi/sql/notice_of_intent_submission/adjacent_land_use/notice_of_intent_adjacent_land_use.sql",
@@ -48,7 +48,7 @@ def process_notice_of_intent_adjacent_land_use(conn=None, batch_size=BATCH_UPLOA
             submission_sql = sql_file.read()
             while True:
                 cursor.execute(
-                    f"{submission_sql} WHERE oalu.alr_application_id > {last_submission_id} ORDER BY oalu.alr_application_id;"
+                    f"{submission_sql} WHERE oalu.adjacent_land_use_id > {last_land_use_id} ORDER BY oalu.adjacent_land_use_id;"
                 )
 
                 rows = cursor.fetchmany(batch_size)
@@ -58,23 +58,24 @@ def process_notice_of_intent_adjacent_land_use(conn=None, batch_size=BATCH_UPLOA
                 try:
                     submissions_to_be_updated_count = len(rows)
 
-                    (parsed_data, raw_data) = _update_notice_of_intent_submissions(
+                    total_updated_items = _update_notice_of_intent_submissions(
                         conn, batch_size, cursor, rows
                     )
 
-                    successful_updates_count = successful_updates_count + len(
-                        parsed_data
+                    successful_updates_count = (
+                        successful_updates_count + total_updated_items
                     )
-                    last_submission_id = dict(rows[-1])["alr_application_id"]
+
+                    last_land_use_id = dict(rows[-1])["adjacent_land_use_id"]
 
                     logger.debug(
-                        f"retrieved/updated items count: {submissions_to_be_updated_count}; total successfully processed submissions so far {successful_updates_count}; last update alr_application_id: {last_submission_id}"
+                        f"retrieved/updated items count: {submissions_to_be_updated_count}; total successfully processed submissions so far {successful_updates_count}; last update alr_application_id: {last_land_use_id}"
                     )
                 except Exception as err:
                     logger.exception()
                     conn.rollback()
                     failed_updates = count_total - successful_updates_count
-                    last_submission_id = last_submission_id + 1
+                    last_land_use_id = last_land_use_id + 1
 
     logger.info(
         f"Finished {etl_name}: total amount of successfully processed adjacent land uses {successful_updates_count}, total failed updates {failed_updates}"
@@ -101,7 +102,11 @@ def _update_notice_of_intent_submissions(conn, batch_size, cursor, rows):
 
     conn.commit()
 
-    return land_use_data, rows
+    total_items = 0
+    for direction in land_use_data:
+        total_items += len(land_use_data[direction])
+
+    return total_items
 
 
 _land_use_update_queries = {

diff --git a/bin/migrate-oats-data/noi/notice_of_intent_submissions/notice_of_intent_soil_fields.py b/bin/migrate-oats-data/noi/notice_of_intent_submissions/notice_of_intent_soil_fields.py
@@ -35,7 +35,7 @@ def process_alcs_notice_of_intent_soil_fields(conn=None, batch_size=BATCH_UPLOAD
 
         failed_inserts = 0
         successful_updates_count = 0
-        last_application_id = 0
+        last_soil_change_element_id = 0
 
         with open(
             "noi/sql/notice_of_intent_submission/soil_fields/notice_of_intent_soil_fields.sql",
@@ -45,7 +45,7 @@ def process_alcs_notice_of_intent_soil_fields(conn=None, batch_size=BATCH_UPLOAD
             application_sql = sql_file.read()
             while True:
                 cursor.execute(
-                    f"{application_sql} WHERE oaac.alr_application_id > {last_application_id} ORDER BY oaac.alr_application_id;"
+                    f"{application_sql} WHERE osce.soil_change_element_id > {last_soil_change_element_id} ORDER BY osce.soil_change_element_id;"
                 )
 
                 rows = cursor.fetchmany(batch_size)
@@ -60,17 +60,17 @@ def process_alcs_notice_of_intent_soil_fields(conn=None, batch_size=BATCH_UPLOAD
                     successful_updates_count = (
                         successful_updates_count + records_to_be_updated_count
                     )
-                    last_application_id = dict(rows[-1])["alr_application_id"]
+                    last_soil_change_element_id = dict(rows[-1])["soil_change_element_id"]
 
                     logger.debug(
-                        f"Retrieved/updated items count: {records_to_be_updated_count}; total successfully updated notice of intents so far {successful_updates_count}; last updated alr_application_id: {last_application_id}"
+                        f"Retrieved/updated items count: {records_to_be_updated_count}; total successfully updated notice of intents so far {successful_updates_count}; last updated alr_application_id: {last_soil_change_element_id}"
                     )
                 except Exception as err:
                     # this is NOT going to be caused by actual data update failure. This code is only executed when the code error appears or connection to DB is lost
                     logger.exception()
                     conn.rollback()
                     failed_inserts = count_total - successful_updates_count
-                    last_application_id = last_application_id + 1
+                    last_soil_change_element_id = last_soil_change_element_id + 1
 
     logger.info(
         f"Finished {etl_name}: total amount of successful updates {successful_updates_count}, total failed updates {failed_inserts}"

diff --git a/...otice_of_intent_submission/adjacent_land_use/notice_of_intent_adjacent_land_use_count.sql b/...otice_of_intent_submission/adjacent_land_use/notice_of_intent_adjacent_land_use_count.sql
@@ -6,12 +6,7 @@ WITH nois_with_one_or_zero_component_only AS (
         AND oaa.application_class_code = 'NOI'
     GROUP BY oaac.alr_application_id
     HAVING count(oaac.alr_application_id) < 2
-),
-grouped_adjacent_land_uses_by_application_id AS (
-    SELECT count(oalu.alr_application_id)
-    FROM oats.oats_adjacent_land_uses oalu
-        JOIN nois_with_one_or_zero_component_only ON oalu.alr_application_id = nois_with_one_or_zero_component_only.alr_application_id
-    GROUP BY oalu.alr_application_id
 )
 SELECT count(*)
-FROM grouped_adjacent_land_uses_by_application_id;
+FROM oats.oats_adjacent_land_uses oalu
+    JOIN nois_with_one_or_zero_component_only ON oalu.alr_application_id = nois_with_one_or_zero_component_only.alr_application_id
diff --git a/...ats-data/noi/sql/notice_of_intent_submission/soil_fields/notice_of_intent_soil_fields.sql b/...ats-data/noi/sql/notice_of_intent_submission/soil_fields/notice_of_intent_soil_fields.sql
@@ -1,17 +1,12 @@
-
-
-SELECT
-oaac.alr_application_id,
-osce.soil_change_code,
-osce.material_desc,
-osce.material_origin_desc,
-osce.volume,
-osce.project_area ,
-osce."depth",
-osce.project_duration
-FROM
-    oats.oats_soil_change_elements osce
-JOIN oats.oats_alr_appl_components oaac ON oaac.alr_appl_component_id = osce.alr_appl_component_id 
-JOIN alcs.notice_of_intent_submission nois ON nois.file_number = oaac.alr_application_id::text
-
-
+SELECT oaac.alr_application_id,
+    osce.soil_change_code,
+    osce.material_desc,
+    osce.material_origin_desc,
+    osce.volume,
+    osce.project_area,
+    osce."depth",
+    osce.project_duration,
+    osce.soil_change_element_id
+FROM oats.oats_soil_change_elements osce
+    JOIN oats.oats_alr_appl_components oaac ON oaac.alr_appl_component_id = osce.alr_appl_component_id
+    JOIN alcs.notice_of_intent_submission nois ON nois.file_number = oaac.alr_application_id::text