added pnp and berd seperation in config and subsiquent operations

src/dev_config.yaml

@@ -73,7 +73,8 @@ network_paths:
   # updated_snapshot_path: "R:/BERD Results System Development 2023/DAP_emulation/spp_snapshots/2022_snapshots/snapshot-202212-002-83b5bacd-7c99-45cf-b989-d43d762dd054.json"
   # Freezing data paths
   frozen_data_staged_output_path: "02_freezing/frozen_data_staged/"
-  frozen_data_staged_path: "02_freezing/frozen_data_staged/2023_FROZEN_staged_full_responses_25-01-21_v397.csv"
+  berd_frozen_data_staged_path: "02_freezing/frozen_data_staged/2023_FROZEN_staged_full_responses_25-01-21_v397.csv"
+  pnp_frozen_data_staged_path: "02_freezing/frozen_data_staged/PNP_2023_FROZEN_staged_full_responses_25-01-21_v397.csv"
   freezing_changes_to_review_path: "02_freezing/changes_to_review/"
   freezing_additions_path: "R:/BERD Results System Development 2023/DAP_emulation/2023_surveys/BERD/02_freezing/freezing_updates/2023_freezing_additions_to_review_24-09-12_v1324.csv"
   freezing_amendments_path: "R:/BERD Results System Development 2023/DAP_emulation/2023_surveys/BERD/02_freezing/freezing_updates/2023_freezing_amendments_to_review_24-09-12_v1324.csv"

src/freezing/freezing_main.py

@@ -106,7 +106,15 @@ def read_frozen_csv(config: dict, read_csv: Callable) -> pd.DataFrame:
     Returns:
         pd.DataFrame: The frozen data csv.
     """
-    frozen_data_staged_path = config["freezing_paths"]["frozen_data_staged_path"]
+    if config["survey"]["survey_type"] == "BERD":
+        frozen_data_staged_path = config["freezing_paths"][
+            "berd_frozen_data_staged_path"
+        ]
+    elif config["survey"]["survey_type"] == "PNP":
+        frozen_data_staged_path = config["freezing_paths"][
+            "pnp_frozen_data_staged_path"
+        ]
+
     FreezingLogger.info("Loading frozen data...")
     frozen_csv = read_csv(frozen_data_staged_path)
     validate_data_with_schema(frozen_csv, "./config/frozen_data_staged_schema.toml")

src/user_config.yaml

@@ -2,7 +2,7 @@ config_validation:
   validate: True
   path: src/user_config_schema.yaml
 survey:
-  survey_type: "BERD"
+  survey_type: "PNP"
   survey_year: 2023
 global:
   # Staging and validation settings

src/utils/config.py

@@ -332,11 +332,20 @@ def validate_freezing_config_settings(config: dict):
     ) = validate_freezing_run_config(config)
 
     snapshot_path = config["staging_paths"]["snapshot_path"]
-    frozen_data_staged_path = config["freezing_paths"]["frozen_data_staged_path"]
     updated_snapshot_path = config["staging_paths"]["updated_snapshot_path"]
     freezing_additions_path = config["freezing_paths"]["freezing_additions_path"]
     freezing_amendments_path = config["freezing_paths"]["freezing_amendments_path"]
 
+    # Frozen data path is dependent on the survey type
+    if config["survey"]["survey_type"] == "BERD":
+        frozen_data_staged_path = config["freezing_paths"][
+            "berd_frozen_data_staged_path"
+        ]
+    elif config["survey"]["survey_type"] == "PNP":
+        frozen_data_staged_path = config["freezing_paths"][
+            "pnp_frozen_data_staged_path"
+        ]
+
     if run_with_snapshot:
         if snapshot_path is None:
             raise ValueError(

src/utils/path_helpers.py

@@ -1,6 +1,5 @@
 """This module contains helper functions for creating paths."""
 
-import re
 import os
 import logging
 
@@ -48,6 +47,8 @@ def get_paths(config: dict) -> dict:
     platform = config["global"]["platform"]
     survey = config["survey"]["survey_type"]
 
+    print(platform)
+
     # select either network_paths or s3_paths from the config, depending on platform,
     paths = config[f"{platform}_paths"]
     paths["year"] = config["survey"]["survey_year"]
@@ -218,18 +219,14 @@ def create_freezing_config(config: dict) -> dict:
     paths = get_paths(config)
     survey_path = paths["survey_path"]
 
-    if (
-        config["survey"]["survey_type"] == "PNP"
-        and "PNP_" not in paths["frozen_data_staged_path"]
-    ):
-        pattern = "frozen_data_staged/"
-        insert_string = "frozen_data_staged/PNP_"
-        paths["frozen_data_staged_path"] = re.sub(
-            pattern, insert_string, paths["frozen_data_staged_path"]
+    if config["survey"]["survey_type"] == "BERD":
+        freezing_dict["frozen_data_staged_path"] = os.path.join(
+            survey_path, paths["berd_frozen_data_staged_path"]
+        )
+    elif config["survey"]["survey_type"] == "PNP":
+        freezing_dict["frozen_data_staged_path"] = os.path.join(
+            survey_path, paths["pnp_frozen_data_staged_path"]
         )
-    freezing_dict["frozen_data_staged_path"] = os.path.join(
-        survey_path, paths["frozen_data_staged_path"]
-    )
     freezing_dict["freezing_changes_to_review_path"] = os.path.join(
         survey_path, paths["freezing_changes_to_review_path"]
     )